Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Try merging id lists using priority queue

Harpreet Sangar10c63e5c3997
diff --git a/include/num_tree.h b/include/num_tree.h
index e554cdbb..09126fe1 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -81,4 +81,9 @@ public:
bool is_wildcard_no_filter_query);
size_t counter_list_size() const;
+
+ void merge_id_list_iterators(std::vector<id_list_t::iterator_t>& id_list_iterators,
+ const NUM_COMPARATOR &comparator,
+ uint32_t*& result_ids,
+ uint32_t& result_ids_len) const;
};
\ No newline at end of file
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 88485aa4..482c7475 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -606,22 +606,40 @@ void filter_result_iterator_t::init() {
for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
const std::string& filter_value = a_filter.values[fi];
int64_t value = (int64_t)std::stol(filter_value);
+ std::vector<id_list_t::iterator_t> id_list_iterators;
+ std::vector<id_list_t*> expanded_id_lists;
- size_t result_size = filter_result.count;
if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
const std::string& next_filter_value = a_filter.values[fi + 1];
auto const range_end_value = (int64_t)std::stol(next_filter_value);
- num_tree->range_inclusive_search(value, range_end_value, &filter_result.docs, result_size);
+ num_tree->range_inclusive_search_iterators(value, range_end_value, id_list_iterators, expanded_id_lists);
fi++;
- } else if (a_filter.comparators[fi] == NOT_EQUALS) {
- numeric_not_equals_filter(num_tree, value,
- index->seq_ids->uncompress(), index->seq_ids->num_ids(),
- filter_result.docs, result_size);
} else {
- num_tree->search(a_filter.comparators[fi], value, &filter_result.docs, result_size);
+ num_tree->search_iterators(a_filter.comparators[fi] == NOT_EQUALS ? EQUALS : a_filter.comparators[fi],
+ value, id_list_iterators, expanded_id_lists);
}
- filter_result.count = result_size;
+ uint32_t* filter_match_ids = nullptr;
+ uint32_t filter_ids_length;
+ num_tree->merge_id_list_iterators(id_list_iterators, a_filter.comparators[fi],
+ filter_match_ids, filter_ids_length);
+
+ if (a_filter.comparators[fi] == NOT_EQUALS) {
+ apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),
+ filter_match_ids, filter_ids_length);
+ }
+
+ uint32_t *out = nullptr;
+ filter_result.count = ArrayUtils::or_scalar(filter_match_ids, filter_ids_length,
+ filter_result.docs, filter_result.count, &out);
+
+ delete [] filter_match_ids;
+ delete [] filter_result.docs;
+ filter_result.docs = out;
+
+ for(id_list_t* expanded_id_list: expanded_id_lists) {
+ delete expanded_id_list;
+ }
}
if (a_filter.apply_not_equals) {
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index 8fdbdbb3..36b51c09 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -444,37 +444,74 @@ num_tree_t::~num_tree_t() {
size_t num_tree_t::intersect(const uint32_t* result_ids, int result_ids_len, int max_facet_count,
std::map<int64_t, uint32_t>& found, bool is_wildcard_no_filter_query) {
//LOG (INFO) << "intersecting field " << field;
-
+
// LOG (INFO) << "int64map size " << int64map.size()
// << " , counter_list size " << counter_list.size();
-
+
std::vector<uint32_t> id_list;
- for(const auto& counter_list_it : counter_list) {
+ for (const auto &counter_list_it: counter_list) {
// LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value
// << " having total count " << counter_list_it.count;
uint32_t count = 0;
- if(is_wildcard_no_filter_query) {
+ if (is_wildcard_no_filter_query) {
count = counter_list_it.count;
} else {
auto ids = int64map.at(counter_list_it.facet_value);
ids_t::uncompress(ids, id_list);
const auto ids_len = id_list.size();
- for(int i = 0; i < result_ids_len; ++i) {
- uint32_t* out = nullptr;
+ for (int i = 0; i < result_ids_len; ++i) {
+ uint32_t *out = nullptr;
count = ArrayUtils::and_scalar(id_list.data(), id_list.size(),
- result_ids, result_ids_len, &out);
+ result_ids, result_ids_len, &out);
}
id_list.clear();
}
- if(count) {
+ if (count) {
found[counter_list_it.facet_value] = count;
- if(found.size() == max_facet_count) {
+ if (found.size() == max_facet_count) {
break;
}
}
}
-
+
return found.size();
}
+
+void num_tree_t::merge_id_list_iterators(std::vector<id_list_t::iterator_t>& id_list_iterators,
+ const NUM_COMPARATOR &comparator,
+ uint32_t*& result_ids,
+ uint32_t& result_ids_len) const {
+ struct comp {
+ bool operator()(const id_list_t::iterator_t *lhs, const id_list_t::iterator_t *rhs) const {
+ return lhs->id() > rhs->id();
+ }
+ };
+
+ std::priority_queue<id_list_t::iterator_t*, std::vector<id_list_t::iterator_t*>, comp> iter_queue;
+ for (auto& id_list_iterator: id_list_iterators) {
+ if (id_list_iterator.valid()) {
+ iter_queue.push(&id_list_iterator);
+ }
+ }
+
+ std::vector<uint32_t> consolidated_ids;
+ while (!iter_queue.empty()) {
+ id_list_t::iterator_t* iter = iter_queue.top();
+ iter_queue.pop();
+
+ consolidated_ids.push_back(iter->id());
+ iter->next();
+
+ if (iter->valid()) {
+ iter_queue.push(iter);
+ }
+ }
+
+ consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());
+
+ result_ids_len = consolidated_ids.size();
+ result_ids = new uint32_t[consolidated_ids.size()];
+ std::copy(consolidated_ids.begin(), consolidated_ids.end(), result_ids);
+}