Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Numeric iterator (#1709)
Harpreet Sangar43a9bb75d1a0
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex 0e7e0467..da27131a 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -216,10 +216,12 @@ struct filter_result_t {constexpr uint16_t function_call_modulo = 10;constexpr uint16_t string_filter_ids_threshold = 3;constexpr uint16_t bool_filter_ids_threshold = 3;+ constexpr uint16_t numeric_filter_ids_threshold = 3;#elseconstexpr uint16_t function_call_modulo = 16'384;constexpr uint16_t string_filter_ids_threshold = 20'000;constexpr uint16_t bool_filter_ids_threshold = 20'000;+ constexpr uint16_t numeric_filter_ids_threshold = 20'000;#endifstruct filter_result_iterator_timeout_info {@@ -262,6 +264,22 @@ private:/// Used in case of a single boolean filter matching more than `bool_filter_ids_threshold` ids.num_tree_t::iterator_t bool_iterator = num_tree_t::iterator_t(nullptr, NUM_COMPARATOR::EQUALS, 0);+ /// Initialized in case of filter on a numeric field.+ /// Sample filter: [10..100, 150]. Operators other than `=` and `!` can match more than one values. We get id list+ /// iterator for each value.+ ///+ /// Multiple filters: Multiple values: id list iterator+ std::vector<std::vector<id_list_t*>> id_lists;+ std::vector<std::vector<id_list_t::iterator_t>> id_list_iterators;+ std::vector<id_list_t*> expanded_id_lists;++ /// Stores the the current seq_id of filter values.+ std::vector<uint32_t> seq_ids;++ /// Numerical filters can have `!` operator individually.+ /// Sample filter: [>10, !15].+ std::unordered_set<uint32_t> numerical_not_iterator_index;+bool delete_filter_node = false;std::unique_ptr<filter_result_iterator_timeout_info> timeout_info;@@ -275,12 +293,18 @@ private:/// Performs OR on the subtrees of operator.void or_filter_iterators();- /// Advances all the token iterators that are at seq_id and finds the next intersection.+ /// Advances all the token iterators that are at seq_id.void advance_string_filter_token_iterators();/// Finds the next match for a filter on string field.void get_string_filter_next_match(const bool& field_is_array);+ /// Advances all the iterators that are at seq_id.+ void advance_numeric_filter_iterators();++ /// Computes the match for a filter on numeric field.+ void get_numeric_filter_match(const bool init = false);+explicit filter_result_iterator_t(uint32_t approx_filter_ids_length);/// Collects n doc ids while advancing the iterator. The iterator may become invalid during this operation.diff --git a/include/num_tree.h b/include/num_tree.hindex b0b1a951..97715012 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -42,6 +42,9 @@ public:void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len);+ std::vector<void*> search(const NUM_COMPARATOR& comparator, const int64_t& value,+ const int64_t& range_end_value = 0) const;+uint32_t approx_search_count(NUM_COMPARATOR comparator, int64_t value);void remove(uint64_t value, uint32_t id);diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 393e623a..05fc4fd1 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -453,6 +453,183 @@ void filter_result_iterator_t::get_string_filter_next_match(const bool& field_isvalidity = one_is_valid || is_not_equals_iterator ? valid : invalid;}+void filter_result_iterator_t::advance_numeric_filter_iterators() {+ auto one_is_valid = false;+ for (uint32_t i = 0; i < id_list_iterators.size(); i++) {+ if (seq_ids[i] > seq_id || numerical_not_iterator_index.count(i) > 0) {+ one_is_valid = true;+ continue;+ }++ auto& its = id_list_iterators[i];+ // Iterators get ORed, so we only advance the iterator that is at seq_id.+ switch (its.size()) {+ case 0:+ continue;++ case 1:+ if (!its[0].valid()) {+ its.clear();+ continue;+ }++ if (its[0].id() == seq_id) {+ its[0].next();+ }++ if (its[0].valid()) {+ seq_ids[i] = its[0].id();+ one_is_valid = true;+ }+ continue;++ case 2:+ if (!its[0].valid() && !its[1].valid()) {+ its.clear();+ continue;+ }++ if (its[0].valid() && its[0].id() == seq_id) {+ its[0].next();+ }+ if (its[1].valid() && its[1].id() == seq_id) {+ its[1].next();+ }++ if (its[0].valid() && its[1].valid()) {+ seq_ids[i] = its[0].id() < its[1].id() ? its[0].id() : its[1].id();+ one_is_valid = true;+ } else if (its[0].valid()) {+ seq_ids[i] = its[0].id();+ one_is_valid = true;+ } else if (its[1].valid()) {+ seq_ids[i] = its[1].id();+ one_is_valid = true;+ }+ continue;++ default:+ auto is_valid = false;+ auto lowest_id = UINT32_MAX;++ for (auto it = its.begin(); it != its.end();) {+ if (!it->valid()) {+ it = its.erase(it);+ continue;+ }++ if (it->id() == seq_id) {+ it->next();+ }++ if (it->valid()) {+ lowest_id = std::min(it->id(), lowest_id);+ is_valid = true;+ it++;+ } else {+ it = its.erase(it);+ }+ }++ if (is_valid) {+ seq_ids[i] = lowest_id;+ one_is_valid = true;+ }+ }+ }++ is_equals_iterator_valid = one_is_valid;+ validity = one_is_valid ? valid : invalid;+}++void filter_result_iterator_t::get_numeric_filter_match(const bool init) {+ if (init) {+ seq_id = 0;+ // Initialize seq_ids and get the first match.+ auto one_is_valid = false;+ for (uint32_t i = 0; i < id_list_iterators.size(); i++) {+ if (numerical_not_iterator_index.count(i) > 0) {+ seq_ids[i] = 0;+ one_is_valid = true;+ continue;+ }++ // Iterators get ORed, so the lowest id is the match.+ auto& its = id_list_iterators[i];+ switch (its.size()) {+ case 0:+ continue;++ case 1:+ if (its[0].valid()) {+ seq_ids[i] = its[0].id();+ one_is_valid = true;+ } else {+ its.clear();+ }+ continue;++ case 2:+ if (its[0].valid() && its[1].valid()) {+ seq_ids[i] = its[0].id() < its[1].id() ? its[0].id() : its[1].id();+ one_is_valid = true;+ } else if (its[0].valid()) {+ seq_ids[i] = its[0].id();+ one_is_valid = true;+ } else if (its[1].valid()) {+ seq_ids[i] = its[1].id();+ one_is_valid = true;+ } else {+ its.clear();+ }+ continue;++ default:+ auto is_valid = false;+ auto lowest_id = UINT32_MAX;++ for (auto it = its.begin(); it != its.end();) {+ if (it->valid()) {+ lowest_id = std::min(it->id(), lowest_id);+ is_valid = true;+ it++;+ } else {+ it = its.erase(it);+ }+ }++ if (is_valid) {+ seq_ids[i] = lowest_id;+ one_is_valid = true;+ }+ }+ }++ if (!one_is_valid) {+ validity = invalid;+ return;+ }+ }++ // Multiple filter values get ORed, so the lowest id is the match.+ auto one_is_valid = false;+ auto lowest_id = UINT32_MAX;+ for (const auto& id: seq_ids) {+ if (id < seq_id || (!init && id == seq_id)) {+ continue;+ }++ lowest_id = std::min(id, lowest_id);+ one_is_valid = true;+ }++ if (one_is_valid) {+ equals_iterator_id = seq_id = lowest_id;+ }++ validity = one_is_valid ? valid : invalid;+}+void filter_result_iterator_t::next() {if (validity != valid) {return;@@ -514,14 +691,18 @@ void filter_result_iterator_t::next() {return;}- if (f.is_bool()) {+ if (f.is_integer() || f.is_float()) {+ advance_numeric_filter_iterators();+ get_numeric_filter_match();+ return;+ } else if (f.is_bool()) {bool_iterator.next();if (!bool_iterator.is_valid) {validity = invalid;return;}- seq_id = bool_iterator.seq_id;+ equals_iterator_id = seq_id = bool_iterator.seq_id;return;} else if (f.is_string()) {advance_string_filter_token_iterators();@@ -742,44 +923,96 @@ void filter_result_iterator_t::init() {filter_result.docs, filter_result.count);}}++ if (a_filter.apply_not_equals) {+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),+ filter_result.docs, filter_result.count);+ }++ if (filter_result.count == 0) {+ validity = invalid;+ return;+ }++ seq_id = filter_result.docs[result_index];+ is_filter_result_initialized = true;+ approx_filter_ids_length = filter_result.count;} else {- auto num_tree = index->numerical_index.at(a_filter.field_name);+ auto const& filter_values_count = a_filter.values.size();- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {+ auto const& num_tree = index->numerical_index.at(a_filter.field_name);+ size_t i = 0;+ for (size_t fi = 0; fi < filter_values_count; fi++, i++) {const std::string& filter_value = a_filter.values[fi];- int64_t value = (int64_t)std::stol(filter_value);+ auto const value = (int64_t)std::stol(filter_value);+ auto const& comparator = a_filter.comparators[fi];- size_t result_size = filter_result.count;- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {+ if (comparator == NOT_EQUALS) {+ numerical_not_iterator_index.emplace(i);+ }++ std::vector<void*> raw_id_lists;+ if (comparator == RANGE_INCLUSIVE && fi+1 < filter_values_count) {const std::string& next_filter_value = a_filter.values[fi + 1];auto const range_end_value = (int64_t)std::stol(next_filter_value);- num_tree->range_inclusive_search(value, range_end_value, &filter_result.docs, result_size);++ raw_id_lists = num_tree->search(comparator, value, range_end_value);fi++;- } else if (a_filter.comparators[fi] == NOT_EQUALS) {- numeric_not_equals_filter(num_tree, value,- index->seq_ids->uncompress(), index->seq_ids->num_ids(),- filter_result.docs, result_size);} else {- num_tree->search(a_filter.comparators[fi], value, &filter_result.docs, result_size);+ raw_id_lists = num_tree->search(comparator, value);}- filter_result.count = result_size;+ std::vector<id_list_t*> lists;+ ids_t::to_expanded_id_lists(raw_id_lists, lists, expanded_id_lists);++ std::vector<id_list_t::iterator_t> iters;+ for (const auto& id_list: lists) {+ iters.emplace_back(id_list->new_iterator());++ if (comparator == NOT_EQUALS) {+ auto const& filter_ids_length = id_list->num_ids();+ auto const& num_ids = index->seq_ids->num_ids();++ approx_filter_ids_length += (num_ids - filter_ids_length);+ } else {+ approx_filter_ids_length += id_list->num_ids();+ }+ }++ id_lists.emplace_back(std::move(lists));+ id_list_iterators.emplace_back(std::move(iters));}- }- if (a_filter.apply_not_equals) {- apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),- filter_result.docs, filter_result.count);- }+ seq_ids = std::vector<uint32_t>(id_lists.size(), UINT32_MAX);- if (filter_result.count == 0) {- validity = invalid;- return;+ if (a_filter.values.size() > 10) {+ compute_iterators();+ return;+ }++ if (a_filter.apply_not_equals) {+ auto const& num_ids = index->seq_ids->num_ids();+ approx_filter_ids_length = approx_filter_ids_length >= num_ids ? num_ids : (num_ids - approx_filter_ids_length);++ if (approx_filter_ids_length < numeric_filter_ids_threshold) {+ // Since there are very few matches, and we have to apply not equals, iteration will be inefficient.+ compute_iterators();+ return;+ } else {+ is_not_equals_iterator = true;+ }+ } else if (approx_filter_ids_length < numeric_filter_ids_threshold) {+ compute_iterators();+ return;+ }++ get_numeric_filter_match(true);+ if (is_not_equals_iterator) {+ seq_id = 0;+ }+ last_valid_id = index->seq_ids->last_id();}- seq_id = filter_result.docs[result_index];- is_filter_result_initialized = true;- approx_filter_ids_length = filter_result.count;return;} else if (f.is_float()) {if (f.range_index) {@@ -816,45 +1049,97 @@ void filter_result_iterator_t::init() {filter_result.docs, filter_result.count);}}++ if (a_filter.apply_not_equals) {+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),+ filter_result.docs, filter_result.count);+ }++ if (filter_result.count == 0) {+ validity = invalid;+ return;+ }++ seq_id = filter_result.docs[result_index];+ is_filter_result_initialized = true;+ approx_filter_ids_length = filter_result.count;} else {- auto num_tree = index->numerical_index.at(a_filter.field_name);+ auto const& filter_values_count = a_filter.values.size();- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {+ auto num_tree = index->numerical_index.at(a_filter.field_name);+ size_t i = 0;+ for (size_t fi = 0; fi < filter_values_count; fi++, i++) {const std::string& filter_value = a_filter.values[fi];float value = (float)std::atof(filter_value.c_str());int64_t float_int64 = Index::float_to_int64_t(value);+ auto const& comparator = a_filter.comparators[fi];- size_t result_size = filter_result.count;- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {- const std::string& next_filter_value = a_filter.values[fi+1];+ if (comparator == NOT_EQUALS) {+ numerical_not_iterator_index.emplace(i);+ }++ std::vector<void*> raw_id_lists;+ if (comparator == RANGE_INCLUSIVE && fi+1 < filter_values_count) {+ const std::string& next_filter_value = a_filter.values[fi + 1];int64_t range_end_value = Index::float_to_int64_t((float) std::atof(next_filter_value.c_str()));- num_tree->range_inclusive_search(float_int64, range_end_value, &filter_result.docs, result_size);++ raw_id_lists = num_tree->search(comparator, float_int64, range_end_value);fi++;- } else if (a_filter.comparators[fi] == NOT_EQUALS) {- numeric_not_equals_filter(num_tree, float_int64,- index->seq_ids->uncompress(), index->seq_ids->num_ids(),- filter_result.docs, result_size);} else {- num_tree->search(a_filter.comparators[fi], float_int64, &filter_result.docs, result_size);+ raw_id_lists = num_tree->search(comparator, float_int64);}- filter_result.count = result_size;+ std::vector<id_list_t*> lists;+ ids_t::to_expanded_id_lists(raw_id_lists, lists, expanded_id_lists);++ std::vector<id_list_t::iterator_t> iters;+ for (const auto& id_list: lists) {+ iters.emplace_back(id_list->new_iterator());++ if (comparator == NOT_EQUALS) {+ auto const& filter_ids_length = id_list->num_ids();+ auto const& num_ids = index->seq_ids->num_ids();++ approx_filter_ids_length += (num_ids - filter_ids_length);+ } else {+ approx_filter_ids_length += id_list->num_ids();+ }+ }++ id_lists.emplace_back(std::move(lists));+ id_list_iterators.emplace_back(std::move(iters));}- }- if (a_filter.apply_not_equals) {- apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),- filter_result.docs, filter_result.count);- }+ seq_ids = std::vector<uint32_t>(id_lists.size(), UINT32_MAX);- if (filter_result.count == 0) {- validity = invalid;- return;+ if (a_filter.values.size() > 10) {+ compute_iterators();+ return;+ }++ if (a_filter.apply_not_equals) {+ auto const& num_ids = index->seq_ids->num_ids();+ approx_filter_ids_length = approx_filter_ids_length >= num_ids ? num_ids : (num_ids - approx_filter_ids_length);++ if (approx_filter_ids_length < numeric_filter_ids_threshold) {+ // Since there are very few matches, and we have to apply not equals, iteration will be inefficient.+ compute_iterators();+ return;+ } else {+ is_not_equals_iterator = true;+ }+ } else if (approx_filter_ids_length < numeric_filter_ids_threshold) {+ compute_iterators();+ return;+ }++ get_numeric_filter_match(true);+ if (is_not_equals_iterator) {+ seq_id = 0;+ }+ last_valid_id = index->seq_ids->last_id();}- seq_id = filter_result.docs[result_index];- is_filter_result_initialized = true;- approx_filter_ids_length = filter_result.count;return;} else if (f.is_bool()) {if (f.range_index) {@@ -1285,14 +1570,158 @@ void filter_result_iterator_t::skip_to(uint32_t id) {field f = index->search_schema.at(a_filter.field_name);- if (f.is_bool()) {+ if (f.is_integer() || f.is_float()) {+ // Skip all the iterators and find a new match.+ auto one_is_valid = false;+ for (uint32_t i = 0; i < id_list_iterators.size(); i++) {+ auto& its = id_list_iterators[i];++ if (numerical_not_iterator_index.count(i) > 0) {+ if (id > last_valid_id) {+ continue;+ }++ one_is_valid = true;+ if (!its[0].valid() || id < its[0].id()) {+ seq_ids[i] = id;+ continue;+ } else if (id == its[0].id()) {+ seq_ids[i] = id + 1;+ continue;+ }++ its[0].skip_to(id);++ if (!its[0].valid() || id < its[0].id()) {+ seq_ids[i] = id;+ continue;+ } else if (id == its[0].id()) {+ seq_ids[i] = id + 1;+ continue;+ }++ continue;+ }++ switch (its.size()) {+ case 0:+ continue;++ case 1:+ if (!its[0].valid()) {+ its.clear();+ continue;+ }+ if (its[0].id() >= id) {+ one_is_valid = true;+ continue;+ }++ its[0].skip_to(id);+ if (its[0].valid()) {+ seq_ids[i] = its[0].id();+ one_is_valid = true;+ }+ continue;++ case 2:+ if (!its[0].valid() && !its[1].valid()) {+ its.clear();+ continue;+ }++ if (its[0].valid() && its[0].id() < id) {+ its[0].skip_to(id);+ }+ if (its[1].valid() && its[1].id() < id) {+ its[1].skip_to(id);+ }++ if (its[0].valid() && its[1].valid()) {+ seq_ids[i] = its[0].id() < its[1].id() ? its[0].id() : its[1].id();+ one_is_valid = true;+ } else if (its[0].valid()) {+ seq_ids[i] = its[0].id();+ one_is_valid = true;+ } else if (its[1].valid()) {+ seq_ids[i] = its[1].id();+ one_is_valid = true;+ }+ continue;++ default:+ auto is_valid = false;+ auto lowest_id = UINT32_MAX;++ for (auto it = its.begin(); it != its.end();) {+ if (!it->valid()) {+ it = its.erase(it);+ continue;+ }++ it->skip_to(id);+ if (it->valid()) {+ lowest_id = std::min(it->id(), lowest_id);+ is_valid = true;+ it++;+ } else {+ it = its.erase(it);+ }+ }++ if (is_valid) {+ seq_ids[i] = lowest_id;+ one_is_valid = true;+ }+ }+ }++ if (!one_is_valid) {+ is_equals_iterator_valid = false;+ validity = invalid;+ return;+ }++ auto lowest_id = UINT32_MAX;+ for (uint32_t i = 0; i < seq_ids.size(); i++) {+ if (numerical_not_iterator_index.count(i) == 0) {+ if (id > seq_ids[i]) {+ continue;+ }++ lowest_id = std::min(seq_ids[i], lowest_id);+ one_is_valid = true;+ continue;+ }++ // NOT_EQUALS comparator+ if (id == seq_ids[i]) {+ // In case id is match for not equals iterator, we need to set seq_id to id + 1.+ seq_ids[i]++;+ lowest_id = id;+ one_is_valid = true;+ } else if (id < seq_ids[i]) {+ lowest_id = std::min(seq_ids[i], lowest_id);+ one_is_valid = true;+ continue;+ }+ }++ if (one_is_valid) {+ equals_iterator_id = seq_id = lowest_id;+ }++ is_equals_iterator_valid = one_is_valid;+ validity = one_is_valid || is_not_equals_iterator ? valid : invalid;+ return;+ } else if (f.is_bool()) {bool_iterator.skip_to(id);if (!bool_iterator.is_valid) {validity = invalid;return;}- seq_id = bool_iterator.seq_id;+ equals_iterator_id = seq_id = bool_iterator.seq_id;return;} else if (f.is_string()) {// Skip all the token iterators and find a new match.@@ -1412,7 +1841,7 @@ int filter_result_iterator_t::is_valid(uint32_t id, const bool& override_timeoutreturn 1;}- return validity ? (seq_id == id ? 1 : 0) : -1;+ return validity ? (id == equals_iterator_id ? 1 : 0) : -1;}Option<bool> filter_result_iterator_t::init_status() {@@ -1547,16 +1976,28 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {field f = index->search_schema.at(a_filter.field_name);- if (f.is_bool()) {- auto num_tree = index->numerical_index.at(a_filter.field_name);- bool_iterator = num_tree_t::iterator_t(num_tree, EQUALS, (a_filter.values[0] == "1" ? 1 : 0));- if (!bool_iterator.is_valid) {- validity = invalid;- return;+ if (f.is_integer() || f.is_float()) {+ for (uint32_t i = 0; i < id_lists.size(); i++) {+ auto const& lists = id_lists[i];++ id_list_iterators[i].clear();+ for (auto const& list: lists) {+ id_list_iterators[i].emplace_back(list->new_iterator());+ }+ }++ seq_ids = std::vector<uint32_t>(id_lists.size(), UINT32_MAX);++ get_numeric_filter_match(true);+ if (is_not_equals_iterator) {+ seq_id = 0;}+ return;+ } else if (f.is_bool()) {+ bool_iterator.reset();seq_id = bool_iterator.seq_id;- validity = valid;+ validity = bool_iterator.is_valid ? valid : invalid;return;} else if (f.is_string()) {for (uint32_t i = 0; i < posting_lists.size(); i++) {@@ -1724,6 +2165,11 @@ filter_result_iterator_t::~filter_result_iterator_t() {delete expanded_plist;}+ // In case the filter was on int/float field.+ for (auto item: expanded_id_lists) {+ delete item;+ }+if (delete_filter_node) {delete filter_node;}@@ -1970,7 +2416,65 @@ void filter_result_iterator_t::compute_iterators() {const filter a_filter = filter_node->filter_exp;field f = index->search_schema.at(a_filter.field_name);- if (f.is_bool()) {+ if (f.is_integer() || f.is_float()) {+ uint32_t* filter_ids = nullptr;+ size_t filter_ids_len = 0;++ // aggregates IDs to reduce excessive ORing+ std::vector<uint32_t> f_id_buff;++ for (uint32_t i = 0; i < id_lists.size(); i++) {+ auto& lists = id_lists[i];+ for (const auto& list: lists) {+ if (numerical_not_iterator_index.count(i) == 0) {+ list->uncompress(f_id_buff);+ continue;+ }++ // NOT_EQUALS comparator+ std::vector<uint32_t> equals_ids;+ list->uncompress(equals_ids);++ uint32_t* not_equals_ids = nullptr;+ auto const not_equals_ids_len = ArrayUtils::exclude_scalar(index->seq_ids->uncompress(), index->seq_ids->num_ids(),+ &equals_ids[0], equals_ids.size(),+ ¬_equals_ids);++ std::copy(not_equals_ids, not_equals_ids + not_equals_ids_len, std::back_inserter(f_id_buff));+ }++ if (f_id_buff.size() >= 100'000) {+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());++ uint32_t* out = nullptr;+ filter_ids_len = ArrayUtils::or_scalar(filter_ids, filter_ids_len, f_id_buff.data(), f_id_buff.size(),+ &out);+ delete[] filter_ids;+ filter_ids = out;+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory+ }+ }++ if (!f_id_buff.empty()) {+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());++ uint32_t* out = nullptr;+ filter_ids_len = ArrayUtils::or_scalar(filter_ids, filter_ids_len, f_id_buff.data(), f_id_buff.size(),+ &out);+ delete[] filter_ids;+ filter_ids = out;+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory+ }++ filter_result.docs = filter_ids;+ filter_result.count = filter_ids_len;++ if (a_filter.apply_not_equals) {+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(), filter_result.docs, filter_result.count);+ }+ } else if (f.is_bool()) {auto num_tree = index->numerical_index.at(a_filter.field_name);int64_t bool_int64 = (a_filter.values[0] == "1") ? 1 : 0;diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 8e41511d..c04de4d9 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -187,6 +187,61 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids}}+std::vector<void*> num_tree_t::search(const NUM_COMPARATOR& comparator, const int64_t& value,+ const int64_t& range_end_value) const {+ if (int64map.empty()) {+ return {};+ }++ std::vector<void*> raw_id_lists;+ auto const& range_start_value = value;+ if (comparator == EQUALS || comparator == NOT_EQUALS) {+ const auto& it = int64map.find(value);+ if (it == int64map.end()) {+ return {};+ }++ raw_id_lists.emplace_back(it->second);+ } else if (comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {+ // iter entries will be >= value, or end() if all entries are before value+ auto iter_ge_value = int64map.lower_bound(value);+ if (iter_ge_value == int64map.end()) {+ return {};+ }++ if (comparator == GREATER_THAN && iter_ge_value->first == value) {+ iter_ge_value++;+ }++ while (iter_ge_value != int64map.end()) {+ raw_id_lists.emplace_back(iter_ge_value->second);+ iter_ge_value++;+ }+ } else if (comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {+ // iter entries will be >= value, or end() if all entries are before value+ auto iter_ge_value = int64map.lower_bound(value);+ auto it = int64map.begin();++ while (it != iter_ge_value) {+ raw_id_lists.emplace_back(it->second);+ it++;+ }++ // for LESS_THAN_EQUALS, check if last iter entry is equal to value+ if (it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {+ raw_id_lists.emplace_back(it->second);+ }+ } else if (comparator == RANGE_INCLUSIVE) {+ auto it_start = int64map.lower_bound(range_start_value); // iter values will be >= range_start_value+ while (it_start != int64map.end() && it_start->first <= range_end_value) {+ raw_id_lists.emplace_back(it_start->second);+ it_start++;+ }+ }++ return raw_id_lists;+}+uint32_t num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value) {if (int64map.empty()) {return 0;diff --git a/test/filter_test.cpp b/test/filter_test.cppindex e5479101..14b6e7f0 100644--- a/test/filter_test.cpp+++ b/test/filter_test.cpp@@ -1074,3 +1074,242 @@ TEST_F(FilterTest, NotEqualsStringFilter) {delete filter_tree_root;}++TEST_F(FilterTest, NumericFilterIterator) {+ nlohmann::json schema =+ R"({+ "name": "Collection",+ "fields": [+ {"name": "rating", "type": "float"},+ {"name": "age", "type": "int32"},+ {"name": "years", "type": "int32[]"},+ {"name": "timestamps", "type": "int64[]"}+ ]+ })"_json;++ Collection* coll = collectionManager.create_collection(schema).get();++ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");+ std::string json_line;+ while (std::getline(infile, json_line)) {+ auto add_op = coll->add(json_line);+ ASSERT_TRUE(add_op.ok());+ }+ infile.close();++ const std::string doc_id_prefix = std::to_string(coll->get_collection_id()) + "_" + Collection::DOC_ID_PREFIX + "_";+ filter_node_t* filter_tree_root = nullptr;++ Option<bool> filter_op = filter::parse_filter_query("age: > 32", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto computed_greater_than_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(computed_greater_than_test.init_status().ok());+ ASSERT_TRUE(computed_greater_than_test._get_is_filter_result_initialized());++ std::vector<int> expected = {1, 3};+ for (auto const& i : expected) {+ ASSERT_EQ(filter_result_iterator_t::valid, computed_greater_than_test.validity);+ ASSERT_EQ(i, computed_greater_than_test.seq_id);+ computed_greater_than_test.next();+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, computed_greater_than_test.validity);++ delete filter_tree_root;+ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("age: >= 32", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_greater_than_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_greater_than_test.init_status().ok());+ ASSERT_FALSE(iter_greater_than_test._get_is_filter_result_initialized());++ std::vector<uint32_t> validate_ids = {0, 1, 2, 3, 4, 5};+ std::vector<uint32_t> seq_ids = {1, 3, 3, 4, 4, 4};+ expected = {0, 1, 0, 1, 1, -1};+ for (uint32_t i = 0; i < validate_ids.size(); i++) {+ if (i < 5) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_greater_than_test.validity);+ } else {+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);+ }+ ASSERT_EQ(expected[i], iter_greater_than_test.is_valid(validate_ids[i]));++ if (expected[i] == 1) {+ iter_greater_than_test.next();+ }+ ASSERT_EQ(seq_ids[i], iter_greater_than_test.seq_id);+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);++ iter_greater_than_test.reset();+ validate_ids = {0, 1, 3, 5};+ seq_ids = {1, 3, 4, 4};+ expected = {0, 1, 1, -1};+ for (uint32_t i = 0; i < validate_ids.size(); i++) {+ if (i < 5) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_greater_than_test.validity);+ } else {+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);+ }+ ASSERT_EQ(expected[i], iter_greater_than_test.is_valid(validate_ids[i]));++ if (expected[i] == 1) {+ iter_greater_than_test.next();+ }+ ASSERT_EQ(seq_ids[i], iter_greater_than_test.seq_id);+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);++ delete filter_tree_root;+ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("age: != 21", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_not_equals_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_not_equals_test.init_status().ok());+ ASSERT_FALSE(iter_not_equals_test._get_is_filter_result_initialized());++ validate_ids = {0, 1, 2, 3, 4, 5};+ seq_ids = {1, 2, 3, 4, 5, 5};+ expected = {1, 1, 0, 1, 1, -1};++ for (uint32_t i = 0; i < validate_ids.size(); i++) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_not_equals_test.validity);+ ASSERT_EQ(expected[i], iter_not_equals_test.is_valid(validate_ids[i]));++ if (expected[i] == 1) {+ iter_not_equals_test.next();+ }+ ASSERT_EQ(seq_ids[i], iter_not_equals_test.seq_id);+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_not_equals_test.validity);++ delete filter_tree_root;+ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("age: != [21]", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_not_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_not_equals_test_2.init_status().ok());+ ASSERT_FALSE(iter_not_equals_test_2._get_is_filter_result_initialized());++ validate_ids = {0, 1, 2, 3, 4, 5};+ seq_ids = {1, 2, 3, 4, 5, 5};+ expected = {1, 1, 0, 1, 1, -1};+ std::vector<bool> equals_iterator_valid = {true, true, true, false, false, false};+ std::vector<uint32_t> equals_match_seq_ids = {2, 2, 2, 2, 2, 2};++ for (uint32_t i = 0; i < validate_ids.size(); i++) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_not_equals_test_2.validity);+ ASSERT_EQ(expected[i], iter_not_equals_test_2.is_valid(validate_ids[i]));++ ASSERT_EQ(equals_iterator_valid[i], iter_not_equals_test_2._get_is_equals_iterator_valid());+ ASSERT_EQ(equals_match_seq_ids[i], iter_not_equals_test_2._get_equals_iterator_id());++ if (expected[i] == 1) {+ iter_not_equals_test_2.next();+ }+ ASSERT_EQ(seq_ids[i], iter_not_equals_test_2.seq_id);+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_not_equals_test_2.validity);++ delete filter_tree_root;+ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("age: [<=21, >32]", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_multivalue_filter = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_multivalue_filter.init_status().ok());+ ASSERT_FALSE(iter_multivalue_filter._get_is_filter_result_initialized());++ validate_ids = {0, 1, 2, 3, 4, 5};+ seq_ids = {1, 2, 3, 3, 3, 3};+ expected = {0, 1, 1, 1, -1, -1};++ for (uint32_t i = 0; i < validate_ids.size(); i++) {+ if (i < 4) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_multivalue_filter.validity);+ } else {+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter.validity);+ }+ ASSERT_EQ(expected[i], iter_multivalue_filter.is_valid(validate_ids[i]));++ if (expected[i] == 1) {+ iter_multivalue_filter.next();+ }+ ASSERT_EQ(seq_ids[i], iter_multivalue_filter.seq_id);+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter.validity);++ delete filter_tree_root;+ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("age: != [<24, >44]", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_multivalue_filter_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_multivalue_filter_2.init_status().ok());+ ASSERT_FALSE(iter_multivalue_filter_2._get_is_filter_result_initialized());++ validate_ids = {0, 1, 2, 3, 4, 5};+ seq_ids = {1, 2, 3, 4, 5, 5};+ expected = {1, 1, 0, 0, 1, -1};+ equals_iterator_valid = {true, true, true, true, false, false};+ equals_match_seq_ids = {2, 2, 2, 3, 3, 3};++ for (uint32_t i = 0; i < validate_ids.size(); i++) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_multivalue_filter_2.validity);+ ASSERT_EQ(expected[i], iter_multivalue_filter_2.is_valid(validate_ids[i]));++ ASSERT_EQ(equals_iterator_valid[i], iter_multivalue_filter_2._get_is_equals_iterator_valid());+ ASSERT_EQ(equals_match_seq_ids[i], iter_multivalue_filter_2._get_equals_iterator_id());++ if (expected[i] == 1) {+ iter_multivalue_filter_2.next();+ }+ ASSERT_EQ(seq_ids[i], iter_multivalue_filter_2.seq_id);+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter_2.validity);++ delete filter_tree_root;+ filter_tree_root = nullptr;+ filter_op = filter::parse_filter_query("age: [21..32, >44]", coll->get_schema(), store, doc_id_prefix,+ filter_tree_root);+ ASSERT_TRUE(filter_op.ok());++ auto iter_multivalue_filter_3 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);+ ASSERT_TRUE(iter_multivalue_filter_3.init_status().ok());+ ASSERT_FALSE(iter_multivalue_filter_3._get_is_filter_result_initialized());++ validate_ids = {0, 1, 2, 3, 4, 5};+ seq_ids = {2, 2, 3, 4, 4, 4};+ expected = {1, 0, 1, 1, 1, -1};+ equals_iterator_valid = {true, true, true, true, true, false};+ equals_match_seq_ids = {0, 2, 2, 3, 4, 4};++ for (uint32_t i = 0; i < validate_ids.size(); i++) {+ if (i < 5) {+ ASSERT_EQ(filter_result_iterator_t::valid, iter_multivalue_filter_3.validity);+ } else {+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter_3.validity);+ }+ ASSERT_EQ(expected[i], iter_multivalue_filter_3.is_valid(validate_ids[i]));++ ASSERT_EQ(equals_iterator_valid[i], iter_multivalue_filter_3._get_is_equals_iterator_valid());+ ASSERT_EQ(equals_match_seq_ids[i], iter_multivalue_filter_3._get_equals_iterator_id());++ if (expected[i] == 1) {+ iter_multivalue_filter_3.next();+ }+ ASSERT_EQ(seq_ids[i], iter_multivalue_filter_3.seq_id);+ }+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter_3.validity);++ delete filter_tree_root;+}