Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Optimize filtering + keyword searching.

Kishore Nallan6c54c1f6c765
diff --git a/include/or_iterator.h b/include/or_iterator.h
index 7307b523..566d310a 100644
--- a/include/or_iterator.h
+++ b/include/or_iterator.h
@@ -46,7 +46,7 @@ public:
[[nodiscard]] const std::vector<posting_list_t::iterator_t>& get_its() const;
- static bool take_id(result_iter_state_t& istate, uint32_t id);
+ static bool take_id(result_iter_state_t& istate, uint32_t id, bool& is_excluded);
template<class T>
static bool intersect(std::vector<or_iterator_t>& its, result_iter_state_t& istate, T func);
@@ -55,29 +55,55 @@ public:
template<class T>
bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_state_t& istate, T func) {
size_t it_size = its.size();
+ bool is_excluded;
switch (its.size()) {
case 0:
break;
case 1:
+ if(istate.filter_ids_length != 0) {
+ its[0].skip_to(istate.filter_ids[istate.filter_ids_index]);
+ }
+
while(its.size() == it_size && its[0].valid()) {
auto id = its[0].id();
- if(take_id(istate, id)) {
+ if(take_id(istate, id, is_excluded)) {
func(id, its);
+ its[0].next();
+ } else {
+ // skip iterator till next id available in filter
+ if(istate.filter_ids_length != 0 && !is_excluded) {
+ if(istate.filter_ids_index < istate.filter_ids_length) {
+ its[0].skip_to(istate.filter_ids[istate.filter_ids_index]);
+ } else {
+ break;
+ }
+ } else {
+ its[0].next();
+ }
}
-
- its[0].next();
}
break;
case 2:
while(its.size() == it_size && !at_end2(its)) {
if(equals2(its)) {
auto id = its[0].id();
- if(take_id(istate, id)) {
+ if(take_id(istate, id, is_excluded)) {
func(id, its);
+ advance_all2(its);
+ } else {
+ // skip iterator till next id available in filter
+ if(istate.filter_ids_length != 0 && !is_excluded) {
+ if(istate.filter_ids_index < istate.filter_ids_length) {
+ its[0].skip_to(istate.filter_ids[istate.filter_ids_index]);
+ its[1].skip_to(istate.filter_ids[istate.filter_ids_index]);
+ } else {
+ break;
+ }
+ } else {
+ advance_all2(its);
+ }
}
-
- advance_all2(its);
} else {
advance_non_largest2(its);
}
@@ -88,11 +114,23 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_state
if(equals(its)) {
//LOG(INFO) << its[0].id();
auto id = its[0].id();
- if(take_id(istate, its[0].id())) {
+ if(take_id(istate, id, is_excluded)) {
func(id, its);
+ advance_all(its);
+ } else {
+ // skip iterator till next id available in filter
+ if(istate.filter_ids_length != 0 && !is_excluded) {
+ if(istate.filter_ids_index < istate.filter_ids_length) {
+ for(auto& it: its) {
+ it.skip_to(istate.filter_ids[istate.filter_ids_index]);
+ }
+ } else {
+ break;
+ }
+ } else {
+ advance_all2(its);
+ }
}
-
- advance_all(its);
} else {
advance_non_largest(its);
}
diff --git a/src/or_iterator.cpp b/src/or_iterator.cpp
index c29be99b..e22ab63f 100644
--- a/src/or_iterator.cpp
+++ b/src/or_iterator.cpp
@@ -170,18 +170,42 @@ uint32_t or_iterator_t::id() const {
return its[curr_index].id();
}
-bool or_iterator_t::take_id(result_iter_state_t& istate, uint32_t id) {
+bool or_iterator_t::take_id(result_iter_state_t& istate, uint32_t id, bool& is_excluded) {
+ is_excluded = false;
+
// decide if this result id should be excluded
if(istate.excluded_result_ids_size != 0) {
if (std::binary_search(istate.excluded_result_ids,
istate.excluded_result_ids + istate.excluded_result_ids_size, id)) {
+ is_excluded = true;
return false;
}
}
// decide if this result be matched with filter results
if(istate.filter_ids_length != 0) {
- return std::binary_search(istate.filter_ids, istate.filter_ids + istate.filter_ids_length, id);
+ if(istate.filter_ids_index >= istate.filter_ids_length) {
+ return false;
+ }
+
+ // Returns iterator to the first element that is >= to value or last if no such element is found.
+ size_t found_index = std::lower_bound(istate.filter_ids + istate.filter_ids_index,
+ istate.filter_ids + istate.filter_ids_length, id) - istate.filter_ids;
+
+ if(found_index == istate.filter_ids_length) {
+ // all elements are lesser than lowest value (id), so we can stop looking
+ istate.filter_ids_index = found_index + 1;
+ return false;
+ } else {
+ if(istate.filter_ids[found_index] == id) {
+ istate.filter_ids_index = found_index + 1;
+ return true;
+ }
+
+ istate.filter_ids_index = found_index;
+ }
+
+ return false;
}
return true;