Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Optimize filtering + keyword searching.
Kishore Nallan6c54c1f6c765
diff --git a/include/or_iterator.h b/include/or_iterator.hindex 7307b523..566d310a 100644--- a/include/or_iterator.h+++ b/include/or_iterator.h@@ -46,7 +46,7 @@ public:[[nodiscard]] const std::vector<posting_list_t::iterator_t>& get_its() const;- static bool take_id(result_iter_state_t& istate, uint32_t id);+ static bool take_id(result_iter_state_t& istate, uint32_t id, bool& is_excluded);template<class T>static bool intersect(std::vector<or_iterator_t>& its, result_iter_state_t& istate, T func);@@ -55,29 +55,55 @@ public:template<class T>bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_state_t& istate, T func) {size_t it_size = its.size();+ bool is_excluded;switch (its.size()) {case 0:break;case 1:+ if(istate.filter_ids_length != 0) {+ its[0].skip_to(istate.filter_ids[istate.filter_ids_index]);+ }+while(its.size() == it_size && its[0].valid()) {auto id = its[0].id();- if(take_id(istate, id)) {+ if(take_id(istate, id, is_excluded)) {func(id, its);+ its[0].next();+ } else {+ // skip iterator till next id available in filter+ if(istate.filter_ids_length != 0 && !is_excluded) {+ if(istate.filter_ids_index < istate.filter_ids_length) {+ its[0].skip_to(istate.filter_ids[istate.filter_ids_index]);+ } else {+ break;+ }+ } else {+ its[0].next();+ }}-- its[0].next();}break;case 2:while(its.size() == it_size && !at_end2(its)) {if(equals2(its)) {auto id = its[0].id();- if(take_id(istate, id)) {+ if(take_id(istate, id, is_excluded)) {func(id, its);+ advance_all2(its);+ } else {+ // skip iterator till next id available in filter+ if(istate.filter_ids_length != 0 && !is_excluded) {+ if(istate.filter_ids_index < istate.filter_ids_length) {+ its[0].skip_to(istate.filter_ids[istate.filter_ids_index]);+ its[1].skip_to(istate.filter_ids[istate.filter_ids_index]);+ } else {+ break;+ }+ } else {+ advance_all2(its);+ }}-- advance_all2(its);} else {advance_non_largest2(its);}@@ -88,11 +114,23 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_stateif(equals(its)) {//LOG(INFO) << its[0].id();auto id = its[0].id();- if(take_id(istate, its[0].id())) {+ if(take_id(istate, id, is_excluded)) {func(id, its);+ advance_all(its);+ } else {+ // skip iterator till next id available in filter+ if(istate.filter_ids_length != 0 && !is_excluded) {+ if(istate.filter_ids_index < istate.filter_ids_length) {+ for(auto& it: its) {+ it.skip_to(istate.filter_ids[istate.filter_ids_index]);+ }+ } else {+ break;+ }+ } else {+ advance_all2(its);+ }}-- advance_all(its);} else {advance_non_largest(its);}diff --git a/src/or_iterator.cpp b/src/or_iterator.cppindex c29be99b..e22ab63f 100644--- a/src/or_iterator.cpp+++ b/src/or_iterator.cpp@@ -170,18 +170,42 @@ uint32_t or_iterator_t::id() const {return its[curr_index].id();}-bool or_iterator_t::take_id(result_iter_state_t& istate, uint32_t id) {+bool or_iterator_t::take_id(result_iter_state_t& istate, uint32_t id, bool& is_excluded) {+ is_excluded = false;+// decide if this result id should be excludedif(istate.excluded_result_ids_size != 0) {if (std::binary_search(istate.excluded_result_ids,istate.excluded_result_ids + istate.excluded_result_ids_size, id)) {+ is_excluded = true;return false;}}// decide if this result be matched with filter resultsif(istate.filter_ids_length != 0) {- return std::binary_search(istate.filter_ids, istate.filter_ids + istate.filter_ids_length, id);+ if(istate.filter_ids_index >= istate.filter_ids_length) {+ return false;+ }++ // Returns iterator to the first element that is >= to value or last if no such element is found.+ size_t found_index = std::lower_bound(istate.filter_ids + istate.filter_ids_index,+ istate.filter_ids + istate.filter_ids_length, id) - istate.filter_ids;++ if(found_index == istate.filter_ids_length) {+ // all elements are lesser than lowest value (id), so we can stop looking+ istate.filter_ids_index = found_index + 1;+ return false;+ } else {+ if(istate.filter_ids[found_index] == id) {+ istate.filter_ids_index = found_index + 1;+ return true;+ }++ istate.filter_ids_index = found_index;+ }++ return false;}return true;