Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Numeric iterator (#1709)

Harpreet Sangar43a9bb75d1a0
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index 0e7e0467..da27131a 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -216,10 +216,12 @@ struct filter_result_t {
constexpr uint16_t function_call_modulo = 10;
constexpr uint16_t string_filter_ids_threshold = 3;
constexpr uint16_t bool_filter_ids_threshold = 3;
+ constexpr uint16_t numeric_filter_ids_threshold = 3;
#else
constexpr uint16_t function_call_modulo = 16'384;
constexpr uint16_t string_filter_ids_threshold = 20'000;
constexpr uint16_t bool_filter_ids_threshold = 20'000;
+ constexpr uint16_t numeric_filter_ids_threshold = 20'000;
#endif
struct filter_result_iterator_timeout_info {
@@ -262,6 +264,22 @@ private:
/// Used in case of a single boolean filter matching more than `bool_filter_ids_threshold` ids.
num_tree_t::iterator_t bool_iterator = num_tree_t::iterator_t(nullptr, NUM_COMPARATOR::EQUALS, 0);
+ /// Initialized in case of filter on a numeric field.
+ /// Sample filter: [10..100, 150]. Operators other than `=` and `!` can match more than one values. We get id list
+ /// iterator for each value.
+ ///
+ /// Multiple filters: Multiple values: id list iterator
+ std::vector<std::vector<id_list_t*>> id_lists;
+ std::vector<std::vector<id_list_t::iterator_t>> id_list_iterators;
+ std::vector<id_list_t*> expanded_id_lists;
+
+ /// Stores the the current seq_id of filter values.
+ std::vector<uint32_t> seq_ids;
+
+ /// Numerical filters can have `!` operator individually.
+ /// Sample filter: [>10, !15].
+ std::unordered_set<uint32_t> numerical_not_iterator_index;
+
bool delete_filter_node = false;
std::unique_ptr<filter_result_iterator_timeout_info> timeout_info;
@@ -275,12 +293,18 @@ private:
/// Performs OR on the subtrees of operator.
void or_filter_iterators();
- /// Advances all the token iterators that are at seq_id and finds the next intersection.
+ /// Advances all the token iterators that are at seq_id.
void advance_string_filter_token_iterators();
/// Finds the next match for a filter on string field.
void get_string_filter_next_match(const bool& field_is_array);
+ /// Advances all the iterators that are at seq_id.
+ void advance_numeric_filter_iterators();
+
+ /// Computes the match for a filter on numeric field.
+ void get_numeric_filter_match(const bool init = false);
+
explicit filter_result_iterator_t(uint32_t approx_filter_ids_length);
/// Collects n doc ids while advancing the iterator. The iterator may become invalid during this operation.
diff --git a/include/num_tree.h b/include/num_tree.h
index b0b1a951..97715012 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -42,6 +42,9 @@ public:
void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len);
+ std::vector<void*> search(const NUM_COMPARATOR& comparator, const int64_t& value,
+ const int64_t& range_end_value = 0) const;
+
uint32_t approx_search_count(NUM_COMPARATOR comparator, int64_t value);
void remove(uint64_t value, uint32_t id);
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 393e623a..05fc4fd1 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -453,6 +453,183 @@ void filter_result_iterator_t::get_string_filter_next_match(const bool& field_is
validity = one_is_valid || is_not_equals_iterator ? valid : invalid;
}
+void filter_result_iterator_t::advance_numeric_filter_iterators() {
+ auto one_is_valid = false;
+ for (uint32_t i = 0; i < id_list_iterators.size(); i++) {
+ if (seq_ids[i] > seq_id || numerical_not_iterator_index.count(i) > 0) {
+ one_is_valid = true;
+ continue;
+ }
+
+ auto& its = id_list_iterators[i];
+ // Iterators get ORed, so we only advance the iterator that is at seq_id.
+ switch (its.size()) {
+ case 0:
+ continue;
+
+ case 1:
+ if (!its[0].valid()) {
+ its.clear();
+ continue;
+ }
+
+ if (its[0].id() == seq_id) {
+ its[0].next();
+ }
+
+ if (its[0].valid()) {
+ seq_ids[i] = its[0].id();
+ one_is_valid = true;
+ }
+ continue;
+
+ case 2:
+ if (!its[0].valid() && !its[1].valid()) {
+ its.clear();
+ continue;
+ }
+
+ if (its[0].valid() && its[0].id() == seq_id) {
+ its[0].next();
+ }
+ if (its[1].valid() && its[1].id() == seq_id) {
+ its[1].next();
+ }
+
+ if (its[0].valid() && its[1].valid()) {
+ seq_ids[i] = its[0].id() < its[1].id() ? its[0].id() : its[1].id();
+ one_is_valid = true;
+ } else if (its[0].valid()) {
+ seq_ids[i] = its[0].id();
+ one_is_valid = true;
+ } else if (its[1].valid()) {
+ seq_ids[i] = its[1].id();
+ one_is_valid = true;
+ }
+ continue;
+
+ default:
+ auto is_valid = false;
+ auto lowest_id = UINT32_MAX;
+
+ for (auto it = its.begin(); it != its.end();) {
+ if (!it->valid()) {
+ it = its.erase(it);
+ continue;
+ }
+
+ if (it->id() == seq_id) {
+ it->next();
+ }
+
+ if (it->valid()) {
+ lowest_id = std::min(it->id(), lowest_id);
+ is_valid = true;
+ it++;
+ } else {
+ it = its.erase(it);
+ }
+ }
+
+ if (is_valid) {
+ seq_ids[i] = lowest_id;
+ one_is_valid = true;
+ }
+ }
+ }
+
+ is_equals_iterator_valid = one_is_valid;
+ validity = one_is_valid ? valid : invalid;
+}
+
+void filter_result_iterator_t::get_numeric_filter_match(const bool init) {
+ if (init) {
+ seq_id = 0;
+ // Initialize seq_ids and get the first match.
+ auto one_is_valid = false;
+ for (uint32_t i = 0; i < id_list_iterators.size(); i++) {
+ if (numerical_not_iterator_index.count(i) > 0) {
+ seq_ids[i] = 0;
+ one_is_valid = true;
+ continue;
+ }
+
+ // Iterators get ORed, so the lowest id is the match.
+ auto& its = id_list_iterators[i];
+ switch (its.size()) {
+ case 0:
+ continue;
+
+ case 1:
+ if (its[0].valid()) {
+ seq_ids[i] = its[0].id();
+ one_is_valid = true;
+ } else {
+ its.clear();
+ }
+ continue;
+
+ case 2:
+ if (its[0].valid() && its[1].valid()) {
+ seq_ids[i] = its[0].id() < its[1].id() ? its[0].id() : its[1].id();
+ one_is_valid = true;
+ } else if (its[0].valid()) {
+ seq_ids[i] = its[0].id();
+ one_is_valid = true;
+ } else if (its[1].valid()) {
+ seq_ids[i] = its[1].id();
+ one_is_valid = true;
+ } else {
+ its.clear();
+ }
+ continue;
+
+ default:
+ auto is_valid = false;
+ auto lowest_id = UINT32_MAX;
+
+ for (auto it = its.begin(); it != its.end();) {
+ if (it->valid()) {
+ lowest_id = std::min(it->id(), lowest_id);
+ is_valid = true;
+ it++;
+ } else {
+ it = its.erase(it);
+ }
+ }
+
+ if (is_valid) {
+ seq_ids[i] = lowest_id;
+ one_is_valid = true;
+ }
+ }
+ }
+
+ if (!one_is_valid) {
+ validity = invalid;
+ return;
+ }
+ }
+
+ // Multiple filter values get ORed, so the lowest id is the match.
+ auto one_is_valid = false;
+ auto lowest_id = UINT32_MAX;
+ for (const auto& id: seq_ids) {
+ if (id < seq_id || (!init && id == seq_id)) {
+ continue;
+ }
+
+ lowest_id = std::min(id, lowest_id);
+ one_is_valid = true;
+ }
+
+ if (one_is_valid) {
+ equals_iterator_id = seq_id = lowest_id;
+ }
+
+ validity = one_is_valid ? valid : invalid;
+}
+
void filter_result_iterator_t::next() {
if (validity != valid) {
return;
@@ -514,14 +691,18 @@ void filter_result_iterator_t::next() {
return;
}
- if (f.is_bool()) {
+ if (f.is_integer() || f.is_float()) {
+ advance_numeric_filter_iterators();
+ get_numeric_filter_match();
+ return;
+ } else if (f.is_bool()) {
bool_iterator.next();
if (!bool_iterator.is_valid) {
validity = invalid;
return;
}
- seq_id = bool_iterator.seq_id;
+ equals_iterator_id = seq_id = bool_iterator.seq_id;
return;
} else if (f.is_string()) {
advance_string_filter_token_iterators();
@@ -742,44 +923,96 @@ void filter_result_iterator_t::init() {
filter_result.docs, filter_result.count);
}
}
+
+ if (a_filter.apply_not_equals) {
+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),
+ filter_result.docs, filter_result.count);
+ }
+
+ if (filter_result.count == 0) {
+ validity = invalid;
+ return;
+ }
+
+ seq_id = filter_result.docs[result_index];
+ is_filter_result_initialized = true;
+ approx_filter_ids_length = filter_result.count;
} else {
- auto num_tree = index->numerical_index.at(a_filter.field_name);
+ auto const& filter_values_count = a_filter.values.size();
- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
+ auto const& num_tree = index->numerical_index.at(a_filter.field_name);
+ size_t i = 0;
+ for (size_t fi = 0; fi < filter_values_count; fi++, i++) {
const std::string& filter_value = a_filter.values[fi];
- int64_t value = (int64_t)std::stol(filter_value);
+ auto const value = (int64_t)std::stol(filter_value);
+ auto const& comparator = a_filter.comparators[fi];
- size_t result_size = filter_result.count;
- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
+ if (comparator == NOT_EQUALS) {
+ numerical_not_iterator_index.emplace(i);
+ }
+
+ std::vector<void*> raw_id_lists;
+ if (comparator == RANGE_INCLUSIVE && fi+1 < filter_values_count) {
const std::string& next_filter_value = a_filter.values[fi + 1];
auto const range_end_value = (int64_t)std::stol(next_filter_value);
- num_tree->range_inclusive_search(value, range_end_value, &filter_result.docs, result_size);
+
+ raw_id_lists = num_tree->search(comparator, value, range_end_value);
fi++;
- } else if (a_filter.comparators[fi] == NOT_EQUALS) {
- numeric_not_equals_filter(num_tree, value,
- index->seq_ids->uncompress(), index->seq_ids->num_ids(),
- filter_result.docs, result_size);
} else {
- num_tree->search(a_filter.comparators[fi], value, &filter_result.docs, result_size);
+ raw_id_lists = num_tree->search(comparator, value);
}
- filter_result.count = result_size;
+ std::vector<id_list_t*> lists;
+ ids_t::to_expanded_id_lists(raw_id_lists, lists, expanded_id_lists);
+
+ std::vector<id_list_t::iterator_t> iters;
+ for (const auto& id_list: lists) {
+ iters.emplace_back(id_list->new_iterator());
+
+ if (comparator == NOT_EQUALS) {
+ auto const& filter_ids_length = id_list->num_ids();
+ auto const& num_ids = index->seq_ids->num_ids();
+
+ approx_filter_ids_length += (num_ids - filter_ids_length);
+ } else {
+ approx_filter_ids_length += id_list->num_ids();
+ }
+ }
+
+ id_lists.emplace_back(std::move(lists));
+ id_list_iterators.emplace_back(std::move(iters));
}
- }
- if (a_filter.apply_not_equals) {
- apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),
- filter_result.docs, filter_result.count);
- }
+ seq_ids = std::vector<uint32_t>(id_lists.size(), UINT32_MAX);
- if (filter_result.count == 0) {
- validity = invalid;
- return;
+ if (a_filter.values.size() > 10) {
+ compute_iterators();
+ return;
+ }
+
+ if (a_filter.apply_not_equals) {
+ auto const& num_ids = index->seq_ids->num_ids();
+ approx_filter_ids_length = approx_filter_ids_length >= num_ids ? num_ids : (num_ids - approx_filter_ids_length);
+
+ if (approx_filter_ids_length < numeric_filter_ids_threshold) {
+ // Since there are very few matches, and we have to apply not equals, iteration will be inefficient.
+ compute_iterators();
+ return;
+ } else {
+ is_not_equals_iterator = true;
+ }
+ } else if (approx_filter_ids_length < numeric_filter_ids_threshold) {
+ compute_iterators();
+ return;
+ }
+
+ get_numeric_filter_match(true);
+ if (is_not_equals_iterator) {
+ seq_id = 0;
+ }
+ last_valid_id = index->seq_ids->last_id();
}
- seq_id = filter_result.docs[result_index];
- is_filter_result_initialized = true;
- approx_filter_ids_length = filter_result.count;
return;
} else if (f.is_float()) {
if (f.range_index) {
@@ -816,45 +1049,97 @@ void filter_result_iterator_t::init() {
filter_result.docs, filter_result.count);
}
}
+
+ if (a_filter.apply_not_equals) {
+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),
+ filter_result.docs, filter_result.count);
+ }
+
+ if (filter_result.count == 0) {
+ validity = invalid;
+ return;
+ }
+
+ seq_id = filter_result.docs[result_index];
+ is_filter_result_initialized = true;
+ approx_filter_ids_length = filter_result.count;
} else {
- auto num_tree = index->numerical_index.at(a_filter.field_name);
+ auto const& filter_values_count = a_filter.values.size();
- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
+ auto num_tree = index->numerical_index.at(a_filter.field_name);
+ size_t i = 0;
+ for (size_t fi = 0; fi < filter_values_count; fi++, i++) {
const std::string& filter_value = a_filter.values[fi];
float value = (float)std::atof(filter_value.c_str());
int64_t float_int64 = Index::float_to_int64_t(value);
+ auto const& comparator = a_filter.comparators[fi];
- size_t result_size = filter_result.count;
- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
- const std::string& next_filter_value = a_filter.values[fi+1];
+ if (comparator == NOT_EQUALS) {
+ numerical_not_iterator_index.emplace(i);
+ }
+
+ std::vector<void*> raw_id_lists;
+ if (comparator == RANGE_INCLUSIVE && fi+1 < filter_values_count) {
+ const std::string& next_filter_value = a_filter.values[fi + 1];
int64_t range_end_value = Index::float_to_int64_t((float) std::atof(next_filter_value.c_str()));
- num_tree->range_inclusive_search(float_int64, range_end_value, &filter_result.docs, result_size);
+
+ raw_id_lists = num_tree->search(comparator, float_int64, range_end_value);
fi++;
- } else if (a_filter.comparators[fi] == NOT_EQUALS) {
- numeric_not_equals_filter(num_tree, float_int64,
- index->seq_ids->uncompress(), index->seq_ids->num_ids(),
- filter_result.docs, result_size);
} else {
- num_tree->search(a_filter.comparators[fi], float_int64, &filter_result.docs, result_size);
+ raw_id_lists = num_tree->search(comparator, float_int64);
}
- filter_result.count = result_size;
+ std::vector<id_list_t*> lists;
+ ids_t::to_expanded_id_lists(raw_id_lists, lists, expanded_id_lists);
+
+ std::vector<id_list_t::iterator_t> iters;
+ for (const auto& id_list: lists) {
+ iters.emplace_back(id_list->new_iterator());
+
+ if (comparator == NOT_EQUALS) {
+ auto const& filter_ids_length = id_list->num_ids();
+ auto const& num_ids = index->seq_ids->num_ids();
+
+ approx_filter_ids_length += (num_ids - filter_ids_length);
+ } else {
+ approx_filter_ids_length += id_list->num_ids();
+ }
+ }
+
+ id_lists.emplace_back(std::move(lists));
+ id_list_iterators.emplace_back(std::move(iters));
}
- }
- if (a_filter.apply_not_equals) {
- apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),
- filter_result.docs, filter_result.count);
- }
+ seq_ids = std::vector<uint32_t>(id_lists.size(), UINT32_MAX);
- if (filter_result.count == 0) {
- validity = invalid;
- return;
+ if (a_filter.values.size() > 10) {
+ compute_iterators();
+ return;
+ }
+
+ if (a_filter.apply_not_equals) {
+ auto const& num_ids = index->seq_ids->num_ids();
+ approx_filter_ids_length = approx_filter_ids_length >= num_ids ? num_ids : (num_ids - approx_filter_ids_length);
+
+ if (approx_filter_ids_length < numeric_filter_ids_threshold) {
+ // Since there are very few matches, and we have to apply not equals, iteration will be inefficient.
+ compute_iterators();
+ return;
+ } else {
+ is_not_equals_iterator = true;
+ }
+ } else if (approx_filter_ids_length < numeric_filter_ids_threshold) {
+ compute_iterators();
+ return;
+ }
+
+ get_numeric_filter_match(true);
+ if (is_not_equals_iterator) {
+ seq_id = 0;
+ }
+ last_valid_id = index->seq_ids->last_id();
}
- seq_id = filter_result.docs[result_index];
- is_filter_result_initialized = true;
- approx_filter_ids_length = filter_result.count;
return;
} else if (f.is_bool()) {
if (f.range_index) {
@@ -1285,14 +1570,158 @@ void filter_result_iterator_t::skip_to(uint32_t id) {
field f = index->search_schema.at(a_filter.field_name);
- if (f.is_bool()) {
+ if (f.is_integer() || f.is_float()) {
+ // Skip all the iterators and find a new match.
+ auto one_is_valid = false;
+ for (uint32_t i = 0; i < id_list_iterators.size(); i++) {
+ auto& its = id_list_iterators[i];
+
+ if (numerical_not_iterator_index.count(i) > 0) {
+ if (id > last_valid_id) {
+ continue;
+ }
+
+ one_is_valid = true;
+ if (!its[0].valid() || id < its[0].id()) {
+ seq_ids[i] = id;
+ continue;
+ } else if (id == its[0].id()) {
+ seq_ids[i] = id + 1;
+ continue;
+ }
+
+ its[0].skip_to(id);
+
+ if (!its[0].valid() || id < its[0].id()) {
+ seq_ids[i] = id;
+ continue;
+ } else if (id == its[0].id()) {
+ seq_ids[i] = id + 1;
+ continue;
+ }
+
+ continue;
+ }
+
+ switch (its.size()) {
+ case 0:
+ continue;
+
+ case 1:
+ if (!its[0].valid()) {
+ its.clear();
+ continue;
+ }
+ if (its[0].id() >= id) {
+ one_is_valid = true;
+ continue;
+ }
+
+ its[0].skip_to(id);
+ if (its[0].valid()) {
+ seq_ids[i] = its[0].id();
+ one_is_valid = true;
+ }
+ continue;
+
+ case 2:
+ if (!its[0].valid() && !its[1].valid()) {
+ its.clear();
+ continue;
+ }
+
+ if (its[0].valid() && its[0].id() < id) {
+ its[0].skip_to(id);
+ }
+ if (its[1].valid() && its[1].id() < id) {
+ its[1].skip_to(id);
+ }
+
+ if (its[0].valid() && its[1].valid()) {
+ seq_ids[i] = its[0].id() < its[1].id() ? its[0].id() : its[1].id();
+ one_is_valid = true;
+ } else if (its[0].valid()) {
+ seq_ids[i] = its[0].id();
+ one_is_valid = true;
+ } else if (its[1].valid()) {
+ seq_ids[i] = its[1].id();
+ one_is_valid = true;
+ }
+ continue;
+
+ default:
+ auto is_valid = false;
+ auto lowest_id = UINT32_MAX;
+
+ for (auto it = its.begin(); it != its.end();) {
+ if (!it->valid()) {
+ it = its.erase(it);
+ continue;
+ }
+
+ it->skip_to(id);
+ if (it->valid()) {
+ lowest_id = std::min(it->id(), lowest_id);
+ is_valid = true;
+ it++;
+ } else {
+ it = its.erase(it);
+ }
+ }
+
+ if (is_valid) {
+ seq_ids[i] = lowest_id;
+ one_is_valid = true;
+ }
+ }
+ }
+
+ if (!one_is_valid) {
+ is_equals_iterator_valid = false;
+ validity = invalid;
+ return;
+ }
+
+ auto lowest_id = UINT32_MAX;
+ for (uint32_t i = 0; i < seq_ids.size(); i++) {
+ if (numerical_not_iterator_index.count(i) == 0) {
+ if (id > seq_ids[i]) {
+ continue;
+ }
+
+ lowest_id = std::min(seq_ids[i], lowest_id);
+ one_is_valid = true;
+ continue;
+ }
+
+ // NOT_EQUALS comparator
+ if (id == seq_ids[i]) {
+ // In case id is match for not equals iterator, we need to set seq_id to id + 1.
+ seq_ids[i]++;
+ lowest_id = id;
+ one_is_valid = true;
+ } else if (id < seq_ids[i]) {
+ lowest_id = std::min(seq_ids[i], lowest_id);
+ one_is_valid = true;
+ continue;
+ }
+ }
+
+ if (one_is_valid) {
+ equals_iterator_id = seq_id = lowest_id;
+ }
+
+ is_equals_iterator_valid = one_is_valid;
+ validity = one_is_valid || is_not_equals_iterator ? valid : invalid;
+ return;
+ } else if (f.is_bool()) {
bool_iterator.skip_to(id);
if (!bool_iterator.is_valid) {
validity = invalid;
return;
}
- seq_id = bool_iterator.seq_id;
+ equals_iterator_id = seq_id = bool_iterator.seq_id;
return;
} else if (f.is_string()) {
// Skip all the token iterators and find a new match.
@@ -1412,7 +1841,7 @@ int filter_result_iterator_t::is_valid(uint32_t id, const bool& override_timeout
return 1;
}
- return validity ? (seq_id == id ? 1 : 0) : -1;
+ return validity ? (id == equals_iterator_id ? 1 : 0) : -1;
}
Option<bool> filter_result_iterator_t::init_status() {
@@ -1547,16 +1976,28 @@ void filter_result_iterator_t::reset(const bool& override_timeout) {
field f = index->search_schema.at(a_filter.field_name);
- if (f.is_bool()) {
- auto num_tree = index->numerical_index.at(a_filter.field_name);
- bool_iterator = num_tree_t::iterator_t(num_tree, EQUALS, (a_filter.values[0] == "1" ? 1 : 0));
- if (!bool_iterator.is_valid) {
- validity = invalid;
- return;
+ if (f.is_integer() || f.is_float()) {
+ for (uint32_t i = 0; i < id_lists.size(); i++) {
+ auto const& lists = id_lists[i];
+
+ id_list_iterators[i].clear();
+ for (auto const& list: lists) {
+ id_list_iterators[i].emplace_back(list->new_iterator());
+ }
+ }
+
+ seq_ids = std::vector<uint32_t>(id_lists.size(), UINT32_MAX);
+
+ get_numeric_filter_match(true);
+ if (is_not_equals_iterator) {
+ seq_id = 0;
}
+ return;
+ } else if (f.is_bool()) {
+ bool_iterator.reset();
seq_id = bool_iterator.seq_id;
- validity = valid;
+ validity = bool_iterator.is_valid ? valid : invalid;
return;
} else if (f.is_string()) {
for (uint32_t i = 0; i < posting_lists.size(); i++) {
@@ -1724,6 +2165,11 @@ filter_result_iterator_t::~filter_result_iterator_t() {
delete expanded_plist;
}
+ // In case the filter was on int/float field.
+ for (auto item: expanded_id_lists) {
+ delete item;
+ }
+
if (delete_filter_node) {
delete filter_node;
}
@@ -1970,7 +2416,65 @@ void filter_result_iterator_t::compute_iterators() {
const filter a_filter = filter_node->filter_exp;
field f = index->search_schema.at(a_filter.field_name);
- if (f.is_bool()) {
+ if (f.is_integer() || f.is_float()) {
+ uint32_t* filter_ids = nullptr;
+ size_t filter_ids_len = 0;
+
+ // aggregates IDs to reduce excessive ORing
+ std::vector<uint32_t> f_id_buff;
+
+ for (uint32_t i = 0; i < id_lists.size(); i++) {
+ auto& lists = id_lists[i];
+ for (const auto& list: lists) {
+ if (numerical_not_iterator_index.count(i) == 0) {
+ list->uncompress(f_id_buff);
+ continue;
+ }
+
+ // NOT_EQUALS comparator
+ std::vector<uint32_t> equals_ids;
+ list->uncompress(equals_ids);
+
+ uint32_t* not_equals_ids = nullptr;
+ auto const not_equals_ids_len = ArrayUtils::exclude_scalar(index->seq_ids->uncompress(), index->seq_ids->num_ids(),
+ &equals_ids[0], equals_ids.size(),
+ &not_equals_ids);
+
+ std::copy(not_equals_ids, not_equals_ids + not_equals_ids_len, std::back_inserter(f_id_buff));
+ }
+
+ if (f_id_buff.size() >= 100'000) {
+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());
+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
+
+ uint32_t* out = nullptr;
+ filter_ids_len = ArrayUtils::or_scalar(filter_ids, filter_ids_len, f_id_buff.data(), f_id_buff.size(),
+ &out);
+ delete[] filter_ids;
+ filter_ids = out;
+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory
+ }
+ }
+
+ if (!f_id_buff.empty()) {
+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());
+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
+
+ uint32_t* out = nullptr;
+ filter_ids_len = ArrayUtils::or_scalar(filter_ids, filter_ids_len, f_id_buff.data(), f_id_buff.size(),
+ &out);
+ delete[] filter_ids;
+ filter_ids = out;
+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory
+ }
+
+ filter_result.docs = filter_ids;
+ filter_result.count = filter_ids_len;
+
+ if (a_filter.apply_not_equals) {
+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(), filter_result.docs, filter_result.count);
+ }
+ } else if (f.is_bool()) {
auto num_tree = index->numerical_index.at(a_filter.field_name);
int64_t bool_int64 = (a_filter.values[0] == "1") ? 1 : 0;
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index 8e41511d..c04de4d9 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -187,6 +187,61 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids
}
}
+std::vector<void*> num_tree_t::search(const NUM_COMPARATOR& comparator, const int64_t& value,
+ const int64_t& range_end_value) const {
+ if (int64map.empty()) {
+ return {};
+ }
+
+ std::vector<void*> raw_id_lists;
+ auto const& range_start_value = value;
+ if (comparator == EQUALS || comparator == NOT_EQUALS) {
+ const auto& it = int64map.find(value);
+ if (it == int64map.end()) {
+ return {};
+ }
+
+ raw_id_lists.emplace_back(it->second);
+ } else if (comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {
+ // iter entries will be >= value, or end() if all entries are before value
+ auto iter_ge_value = int64map.lower_bound(value);
+ if (iter_ge_value == int64map.end()) {
+ return {};
+ }
+
+ if (comparator == GREATER_THAN && iter_ge_value->first == value) {
+ iter_ge_value++;
+ }
+
+ while (iter_ge_value != int64map.end()) {
+ raw_id_lists.emplace_back(iter_ge_value->second);
+ iter_ge_value++;
+ }
+ } else if (comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {
+ // iter entries will be >= value, or end() if all entries are before value
+ auto iter_ge_value = int64map.lower_bound(value);
+ auto it = int64map.begin();
+
+ while (it != iter_ge_value) {
+ raw_id_lists.emplace_back(it->second);
+ it++;
+ }
+
+ // for LESS_THAN_EQUALS, check if last iter entry is equal to value
+ if (it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {
+ raw_id_lists.emplace_back(it->second);
+ }
+ } else if (comparator == RANGE_INCLUSIVE) {
+ auto it_start = int64map.lower_bound(range_start_value); // iter values will be >= range_start_value
+ while (it_start != int64map.end() && it_start->first <= range_end_value) {
+ raw_id_lists.emplace_back(it_start->second);
+ it_start++;
+ }
+ }
+
+ return raw_id_lists;
+}
+
uint32_t num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value) {
if (int64map.empty()) {
return 0;
diff --git a/test/filter_test.cpp b/test/filter_test.cpp
index e5479101..14b6e7f0 100644
--- a/test/filter_test.cpp
+++ b/test/filter_test.cpp
@@ -1074,3 +1074,242 @@ TEST_F(FilterTest, NotEqualsStringFilter) {
delete filter_tree_root;
}
+
+TEST_F(FilterTest, NumericFilterIterator) {
+ nlohmann::json schema =
+ R"({
+ "name": "Collection",
+ "fields": [
+ {"name": "rating", "type": "float"},
+ {"name": "age", "type": "int32"},
+ {"name": "years", "type": "int32[]"},
+ {"name": "timestamps", "type": "int64[]"}
+ ]
+ })"_json;
+
+ Collection* coll = collectionManager.create_collection(schema).get();
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
+ std::string json_line;
+ while (std::getline(infile, json_line)) {
+ auto add_op = coll->add(json_line);
+ ASSERT_TRUE(add_op.ok());
+ }
+ infile.close();
+
+ const std::string doc_id_prefix = std::to_string(coll->get_collection_id()) + "_" + Collection::DOC_ID_PREFIX + "_";
+ filter_node_t* filter_tree_root = nullptr;
+
+ Option<bool> filter_op = filter::parse_filter_query("age: > 32", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto computed_greater_than_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(computed_greater_than_test.init_status().ok());
+ ASSERT_TRUE(computed_greater_than_test._get_is_filter_result_initialized());
+
+ std::vector<int> expected = {1, 3};
+ for (auto const& i : expected) {
+ ASSERT_EQ(filter_result_iterator_t::valid, computed_greater_than_test.validity);
+ ASSERT_EQ(i, computed_greater_than_test.seq_id);
+ computed_greater_than_test.next();
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, computed_greater_than_test.validity);
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("age: >= 32", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_greater_than_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_greater_than_test.init_status().ok());
+ ASSERT_FALSE(iter_greater_than_test._get_is_filter_result_initialized());
+
+ std::vector<uint32_t> validate_ids = {0, 1, 2, 3, 4, 5};
+ std::vector<uint32_t> seq_ids = {1, 3, 3, 4, 4, 4};
+ expected = {0, 1, 0, 1, 1, -1};
+ for (uint32_t i = 0; i < validate_ids.size(); i++) {
+ if (i < 5) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_greater_than_test.validity);
+ } else {
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);
+ }
+ ASSERT_EQ(expected[i], iter_greater_than_test.is_valid(validate_ids[i]));
+
+ if (expected[i] == 1) {
+ iter_greater_than_test.next();
+ }
+ ASSERT_EQ(seq_ids[i], iter_greater_than_test.seq_id);
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);
+
+ iter_greater_than_test.reset();
+ validate_ids = {0, 1, 3, 5};
+ seq_ids = {1, 3, 4, 4};
+ expected = {0, 1, 1, -1};
+ for (uint32_t i = 0; i < validate_ids.size(); i++) {
+ if (i < 5) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_greater_than_test.validity);
+ } else {
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);
+ }
+ ASSERT_EQ(expected[i], iter_greater_than_test.is_valid(validate_ids[i]));
+
+ if (expected[i] == 1) {
+ iter_greater_than_test.next();
+ }
+ ASSERT_EQ(seq_ids[i], iter_greater_than_test.seq_id);
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_greater_than_test.validity);
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("age: != 21", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_not_equals_test = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_not_equals_test.init_status().ok());
+ ASSERT_FALSE(iter_not_equals_test._get_is_filter_result_initialized());
+
+ validate_ids = {0, 1, 2, 3, 4, 5};
+ seq_ids = {1, 2, 3, 4, 5, 5};
+ expected = {1, 1, 0, 1, 1, -1};
+
+ for (uint32_t i = 0; i < validate_ids.size(); i++) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_not_equals_test.validity);
+ ASSERT_EQ(expected[i], iter_not_equals_test.is_valid(validate_ids[i]));
+
+ if (expected[i] == 1) {
+ iter_not_equals_test.next();
+ }
+ ASSERT_EQ(seq_ids[i], iter_not_equals_test.seq_id);
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_not_equals_test.validity);
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("age: != [21]", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_not_equals_test_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_not_equals_test_2.init_status().ok());
+ ASSERT_FALSE(iter_not_equals_test_2._get_is_filter_result_initialized());
+
+ validate_ids = {0, 1, 2, 3, 4, 5};
+ seq_ids = {1, 2, 3, 4, 5, 5};
+ expected = {1, 1, 0, 1, 1, -1};
+ std::vector<bool> equals_iterator_valid = {true, true, true, false, false, false};
+ std::vector<uint32_t> equals_match_seq_ids = {2, 2, 2, 2, 2, 2};
+
+ for (uint32_t i = 0; i < validate_ids.size(); i++) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_not_equals_test_2.validity);
+ ASSERT_EQ(expected[i], iter_not_equals_test_2.is_valid(validate_ids[i]));
+
+ ASSERT_EQ(equals_iterator_valid[i], iter_not_equals_test_2._get_is_equals_iterator_valid());
+ ASSERT_EQ(equals_match_seq_ids[i], iter_not_equals_test_2._get_equals_iterator_id());
+
+ if (expected[i] == 1) {
+ iter_not_equals_test_2.next();
+ }
+ ASSERT_EQ(seq_ids[i], iter_not_equals_test_2.seq_id);
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_not_equals_test_2.validity);
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("age: [<=21, >32]", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_multivalue_filter = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_multivalue_filter.init_status().ok());
+ ASSERT_FALSE(iter_multivalue_filter._get_is_filter_result_initialized());
+
+ validate_ids = {0, 1, 2, 3, 4, 5};
+ seq_ids = {1, 2, 3, 3, 3, 3};
+ expected = {0, 1, 1, 1, -1, -1};
+
+ for (uint32_t i = 0; i < validate_ids.size(); i++) {
+ if (i < 4) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_multivalue_filter.validity);
+ } else {
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter.validity);
+ }
+ ASSERT_EQ(expected[i], iter_multivalue_filter.is_valid(validate_ids[i]));
+
+ if (expected[i] == 1) {
+ iter_multivalue_filter.next();
+ }
+ ASSERT_EQ(seq_ids[i], iter_multivalue_filter.seq_id);
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter.validity);
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("age: != [<24, >44]", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_multivalue_filter_2 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_multivalue_filter_2.init_status().ok());
+ ASSERT_FALSE(iter_multivalue_filter_2._get_is_filter_result_initialized());
+
+ validate_ids = {0, 1, 2, 3, 4, 5};
+ seq_ids = {1, 2, 3, 4, 5, 5};
+ expected = {1, 1, 0, 0, 1, -1};
+ equals_iterator_valid = {true, true, true, true, false, false};
+ equals_match_seq_ids = {2, 2, 2, 3, 3, 3};
+
+ for (uint32_t i = 0; i < validate_ids.size(); i++) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_multivalue_filter_2.validity);
+ ASSERT_EQ(expected[i], iter_multivalue_filter_2.is_valid(validate_ids[i]));
+
+ ASSERT_EQ(equals_iterator_valid[i], iter_multivalue_filter_2._get_is_equals_iterator_valid());
+ ASSERT_EQ(equals_match_seq_ids[i], iter_multivalue_filter_2._get_equals_iterator_id());
+
+ if (expected[i] == 1) {
+ iter_multivalue_filter_2.next();
+ }
+ ASSERT_EQ(seq_ids[i], iter_multivalue_filter_2.seq_id);
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter_2.validity);
+
+ delete filter_tree_root;
+ filter_tree_root = nullptr;
+ filter_op = filter::parse_filter_query("age: [21..32, >44]", coll->get_schema(), store, doc_id_prefix,
+ filter_tree_root);
+ ASSERT_TRUE(filter_op.ok());
+
+ auto iter_multivalue_filter_3 = filter_result_iterator_t(coll->get_name(), coll->_get_index(), filter_tree_root);
+ ASSERT_TRUE(iter_multivalue_filter_3.init_status().ok());
+ ASSERT_FALSE(iter_multivalue_filter_3._get_is_filter_result_initialized());
+
+ validate_ids = {0, 1, 2, 3, 4, 5};
+ seq_ids = {2, 2, 3, 4, 4, 4};
+ expected = {1, 0, 1, 1, 1, -1};
+ equals_iterator_valid = {true, true, true, true, true, false};
+ equals_match_seq_ids = {0, 2, 2, 3, 4, 4};
+
+ for (uint32_t i = 0; i < validate_ids.size(); i++) {
+ if (i < 5) {
+ ASSERT_EQ(filter_result_iterator_t::valid, iter_multivalue_filter_3.validity);
+ } else {
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter_3.validity);
+ }
+ ASSERT_EQ(expected[i], iter_multivalue_filter_3.is_valid(validate_ids[i]));
+
+ ASSERT_EQ(equals_iterator_valid[i], iter_multivalue_filter_3._get_is_equals_iterator_valid());
+ ASSERT_EQ(equals_match_seq_ids[i], iter_multivalue_filter_3._get_equals_iterator_id());
+
+ if (expected[i] == 1) {
+ iter_multivalue_filter_3.next();
+ }
+ ASSERT_EQ(seq_ids[i], iter_multivalue_filter_3.seq_id);
+ }
+ ASSERT_EQ(filter_result_iterator_t::invalid, iter_multivalue_filter_3.validity);
+
+ delete filter_tree_root;
+}