Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Undo id list merge using priority queue.
Harpreet Sangarc075cb330792
diff --git a/include/num_tree.h b/include/num_tree.hindex 5359d7fd..5406a109 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -30,11 +30,6 @@ public:void range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len);- void range_inclusive_search_iterators(int64_t start,- int64_t end,- std::vector<id_list_t::iterator_t>& id_list_iterators,- std::vector<id_list_t*>& expanded_id_lists);-void approx_range_inclusive_search_count(int64_t start, int64_t end, uint32_t& ids_len);void range_inclusive_contains(const int64_t& start, const int64_t& end,@@ -47,11 +42,6 @@ public:void search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len);- void search_iterators(NUM_COMPARATOR comparator,- int64_t value,- std::vector<id_list_t::iterator_t>& id_list_iterators,- std::vector<id_list_t*>& expanded_id_lists);-void approx_search_count(NUM_COMPARATOR comparator, int64_t value, uint32_t& ids_len);void remove(uint64_t value, uint32_t id);@@ -63,9 +53,4 @@ public:uint32_t* const& context_ids,size_t& result_ids_len,uint32_t*& result_ids) const;-- void merge_id_list_iterators(std::vector<id_list_t::iterator_t>& id_list_iterators,- const NUM_COMPARATOR &comparator,- uint32_t*& result_ids,- uint32_t& result_ids_len) const;};\ No newline at end of filediff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex e58c9c17..99231679 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -651,40 +651,22 @@ void filter_result_iterator_t::init() {for (size_t fi = 0; fi < a_filter.values.size(); fi++) {const std::string& filter_value = a_filter.values[fi];int64_t value = (int64_t)std::stol(filter_value);- std::vector<id_list_t::iterator_t> id_list_iterators;- std::vector<id_list_t*> expanded_id_lists;+ size_t result_size = filter_result.count;if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {const std::string& next_filter_value = a_filter.values[fi + 1];auto const range_end_value = (int64_t)std::stol(next_filter_value);- num_tree->range_inclusive_search_iterators(value, range_end_value, id_list_iterators, expanded_id_lists);+ num_tree->range_inclusive_search(value, range_end_value, &filter_result.docs, result_size);fi++;+ } else if (a_filter.comparators[fi] == NOT_EQUALS) {+ numeric_not_equals_filter(num_tree, value,+ index->seq_ids->uncompress(), index->seq_ids->num_ids(),+ filter_result.docs, result_size);} else {- num_tree->search_iterators(a_filter.comparators[fi] == NOT_EQUALS ? EQUALS : a_filter.comparators[fi],- value, id_list_iterators, expanded_id_lists);- }-- uint32_t* filter_match_ids = nullptr;- uint32_t filter_ids_length;- num_tree->merge_id_list_iterators(id_list_iterators, a_filter.comparators[fi],- filter_match_ids, filter_ids_length);-- if (a_filter.comparators[fi] == NOT_EQUALS) {- apply_not_equals(index->seq_ids->uncompress(), index->seq_ids->num_ids(),- filter_match_ids, filter_ids_length);+ num_tree->search(a_filter.comparators[fi], value, &filter_result.docs, result_size);}- uint32_t *out = nullptr;- filter_result.count = ArrayUtils::or_scalar(filter_match_ids, filter_ids_length,- filter_result.docs, filter_result.count, &out);-- delete [] filter_match_ids;- delete [] filter_result.docs;- filter_result.docs = out;-- for(id_list_t* expanded_id_list: expanded_id_lists) {- delete expanded_id_list;- }+ filter_result.count = result_size;}if (a_filter.apply_not_equals) {@@ -1400,7 +1382,7 @@ void filter_result_iterator_t::get_n_ids(const uint32_t& n,}filter_result_iterator_t::filter_result_iterator_t(uint32_t approx_filter_ids_length) :- approx_filter_ids_length(approx_filter_ids_length) {+ approx_filter_ids_length(approx_filter_ids_length) {filter_node = new filter_node_t(AND, nullptr, nullptr);delete_filter_node = true;}diff --git a/src/num_tree.cpp b/src/num_tree.cppindex fc54f0fa..1bcdbc9f 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -43,30 +43,6 @@ void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** i*ids = out;}-void num_tree_t::range_inclusive_search_iterators(int64_t start,- int64_t end,- std::vector<id_list_t::iterator_t>& id_list_iterators,- std::vector<id_list_t*>& expanded_id_lists) {- if (int64map.empty()) {- return;- }-- auto it_start = int64map.lower_bound(start); // iter values will be >= start-- std::vector<void*> raw_id_lists;- while (it_start != int64map.end() && it_start->first <= end) {- raw_id_lists.push_back(it_start->second);- it_start++;- }-- std::vector<id_list_t*> id_lists;- ids_t::to_expanded_id_lists(raw_id_lists, id_lists, expanded_id_lists);-- for (const auto &id_list: id_lists) {- id_list_iterators.emplace_back(id_list->new_iterator());- }-}-void num_tree_t::approx_range_inclusive_search_count(int64_t start, int64_t end, uint32_t& ids_len) {if (int64map.empty()) {return;@@ -211,60 +187,6 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids}}-void num_tree_t::search_iterators(NUM_COMPARATOR comparator,- int64_t value,- std::vector<id_list_t::iterator_t>& id_list_iterators,- std::vector<id_list_t*>& expanded_id_lists) {- if (int64map.empty()) {- return ;- }-- std::vector<void*> raw_id_lists;- if (comparator == EQUALS) {- const auto& it = int64map.find(value);- if (it != int64map.end()) {- raw_id_lists.emplace_back(it->second);- }- } else if (comparator == GREATER_THAN || comparator == GREATER_THAN_EQUALS) {- // iter entries will be >= value, or end() if all entries are before value- auto iter_ge_value = int64map.lower_bound(value);-- if(iter_ge_value == int64map.end()) {- return ;- }-- if(comparator == GREATER_THAN && iter_ge_value->first == value) {- iter_ge_value++;- }-- while(iter_ge_value != int64map.end()) {- raw_id_lists.emplace_back(iter_ge_value->second);- iter_ge_value++;- }- } else if(comparator == LESS_THAN || comparator == LESS_THAN_EQUALS) {- // iter entries will be >= value, or end() if all entries are before value- auto iter_ge_value = int64map.lower_bound(value);-- auto it = int64map.begin();- while(it != iter_ge_value) {- raw_id_lists.emplace_back(it->second);- it++;- }-- // for LESS_THAN_EQUALS, check if last iter entry is equal to value- if(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {- raw_id_lists.emplace_back(it->second);- }- }-- std::vector<id_list_t*> id_lists;- ids_t::to_expanded_id_lists(raw_id_lists, id_lists, expanded_id_lists);-- for (const auto &id_list: id_lists) {- id_list_iterators.emplace_back(id_list->new_iterator());- }-}-void num_tree_t::approx_search_count(NUM_COMPARATOR comparator, int64_t value, uint32_t& ids_len) {if (int64map.empty()) {return;@@ -405,40 +327,3 @@ num_tree_t::~num_tree_t() {ids_t::destroy_list(kv.second);}}--void num_tree_t::merge_id_list_iterators(std::vector<id_list_t::iterator_t>& id_list_iterators,- const NUM_COMPARATOR &comparator,- uint32_t*& result_ids,- uint32_t& result_ids_len) const {- struct comp {- bool operator()(const id_list_t::iterator_t *lhs, const id_list_t::iterator_t *rhs) const {- return lhs->id() > rhs->id();- }- };-- std::priority_queue<id_list_t::iterator_t*, std::vector<id_list_t::iterator_t*>, comp> iter_queue;- for (auto& id_list_iterator: id_list_iterators) {- if (id_list_iterator.valid()) {- iter_queue.push(&id_list_iterator);- }- }-- std::vector<uint32_t> consolidated_ids;- while (!iter_queue.empty()) {- id_list_t::iterator_t* iter = iter_queue.top();- iter_queue.pop();-- consolidated_ids.push_back(iter->id());- iter->next();-- if (iter->valid()) {- iter_queue.push(iter);- }- }-- consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());-- result_ids_len = consolidated_ids.size();- result_ids = new uint32_t[consolidated_ids.size()];- std::copy(consolidated_ids.begin(), consolidated_ids.end(), result_ids);-}