Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Optimize numerical filter ID consolidation.
Kishore Nallan790fac008b3a
diff --git a/include/id_list.h b/include/id_list.hindex a1e07dfb..3b0ef7ae 100644--- a/include/id_list.h+++ b/include/id_list.h@@ -150,6 +150,8 @@ public:);uint32_t* uncompress();++ void uncompress(std::vector<uint32_t>& data);};template<class T>diff --git a/include/ids_t.h b/include/ids_t.hindex 7ebde632..15cf8c6e 100644--- a/include/ids_t.h+++ b/include/ids_t.h@@ -102,6 +102,8 @@ public:static void intersect(const std::vector<void*>& id_lists, std::vector<uint32_t>& result_ids);static uint32_t* uncompress(void*& obj);++ static void uncompress(void*& obj, std::vector<uint32_t>& ids);};template<class T>diff --git a/src/id_list.cpp b/src/id_list.cppindex 6e0770c8..4b308603 100644--- a/src/id_list.cpp+++ b/src/id_list.cpp@@ -626,6 +626,16 @@ bool id_list_t::take_id(result_iter_state_t& istate, uint32_t id) {return true;}+void id_list_t::uncompress(std::vector<uint32_t>& data) {+ auto it = new_iterator();+ data.reserve(data.size() + ids_length);++ while(it.valid()) {+ data.push_back(it.id());+ it.next();+ }+}+uint32_t* id_list_t::uncompress() {uint32_t* arr = new uint32_t[ids_length];auto it = new_iterator();diff --git a/src/ids_t.cpp b/src/ids_t.cppindex 812c4489..082d01f5 100644--- a/src/ids_t.cpp+++ b/src/ids_t.cpp@@ -370,6 +370,18 @@ uint32_t* ids_t::uncompress(void*& obj) {}}+void ids_t::uncompress(void*& obj, std::vector<uint32_t>& ids) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ for(size_t i = 0; i < list->length; i++) {+ ids.push_back(list->ids[i]);+ }+ } else {+ id_list_t* list = (id_list_t*)(obj);+ list->uncompress(ids);+ }+}+void ids_t::block_intersector_t::split_lists(size_t concurrency,std::vector<std::vector<id_list_t::iterator_t>>& partial_its_vec) {const size_t num_blocks = this->id_lists[0]->num_blocks();diff --git a/src/num_tree.cpp b/src/num_tree.cppindex a8e1f2d2..c8ce253c 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -89,18 +89,11 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** idsstd::vector<uint32_t> consolidated_ids;while(iter_ge_value != int64map.end()) {- uint32_t* values = ids_t::uncompress(iter_ge_value->second);-- for(size_t i = 0; i < ids_t::num_ids(iter_ge_value->second); i++) {- consolidated_ids.push_back(values[i]);- }-- delete [] values;+ ids_t::uncompress(iter_ge_value->second, consolidated_ids);iter_ge_value++;}gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());-consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());uint32_t *out = nullptr;@@ -118,23 +111,13 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** idsauto it = int64map.begin();while(it != iter_ge_value) {- uint32_t* values = ids_t::uncompress(it->second);-- for(size_t i = 0; i < ids_t::num_ids(it->second); i++) {- consolidated_ids.push_back(values[i]);- }-- delete [] values;+ ids_t::uncompress(it->second, consolidated_ids);it++;}// for LESS_THAN_EQUALS, check if last iter entry is equal to valueif(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {- uint32_t* values = ids_t::uncompress(it->second);- for(size_t i = 0; i < ids_t::num_ids(it->second); i++) {- consolidated_ids.push_back(values[i]);- }- delete [] values;+ ids_t::uncompress(it->second, consolidated_ids);}gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());