Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Optimize numerical filter ID consolidation.

Kishore Nallan790fac008b3a
diff --git a/include/id_list.h b/include/id_list.h
index a1e07dfb..3b0ef7ae 100644
--- a/include/id_list.h
+++ b/include/id_list.h
@@ -150,6 +150,8 @@ public:
);
uint32_t* uncompress();
+
+ void uncompress(std::vector<uint32_t>& data);
};
template<class T>
diff --git a/include/ids_t.h b/include/ids_t.h
index 7ebde632..15cf8c6e 100644
--- a/include/ids_t.h
+++ b/include/ids_t.h
@@ -102,6 +102,8 @@ public:
static void intersect(const std::vector<void*>& id_lists, std::vector<uint32_t>& result_ids);
static uint32_t* uncompress(void*& obj);
+
+ static void uncompress(void*& obj, std::vector<uint32_t>& ids);
};
template<class T>
diff --git a/src/id_list.cpp b/src/id_list.cpp
index 6e0770c8..4b308603 100644
--- a/src/id_list.cpp
+++ b/src/id_list.cpp
@@ -626,6 +626,16 @@ bool id_list_t::take_id(result_iter_state_t& istate, uint32_t id) {
return true;
}
+void id_list_t::uncompress(std::vector<uint32_t>& data) {
+ auto it = new_iterator();
+ data.reserve(data.size() + ids_length);
+
+ while(it.valid()) {
+ data.push_back(it.id());
+ it.next();
+ }
+}
+
uint32_t* id_list_t::uncompress() {
uint32_t* arr = new uint32_t[ids_length];
auto it = new_iterator();
diff --git a/src/ids_t.cpp b/src/ids_t.cpp
index 812c4489..082d01f5 100644
--- a/src/ids_t.cpp
+++ b/src/ids_t.cpp
@@ -370,6 +370,18 @@ uint32_t* ids_t::uncompress(void*& obj) {
}
}
+void ids_t::uncompress(void*& obj, std::vector<uint32_t>& ids) {
+ if(IS_COMPACT_IDS(obj)) {
+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);
+ for(size_t i = 0; i < list->length; i++) {
+ ids.push_back(list->ids[i]);
+ }
+ } else {
+ id_list_t* list = (id_list_t*)(obj);
+ list->uncompress(ids);
+ }
+}
+
void ids_t::block_intersector_t::split_lists(size_t concurrency,
std::vector<std::vector<id_list_t::iterator_t>>& partial_its_vec) {
const size_t num_blocks = this->id_lists[0]->num_blocks();
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index a8e1f2d2..c8ce253c 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -89,18 +89,11 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids
std::vector<uint32_t> consolidated_ids;
while(iter_ge_value != int64map.end()) {
- uint32_t* values = ids_t::uncompress(iter_ge_value->second);
-
- for(size_t i = 0; i < ids_t::num_ids(iter_ge_value->second); i++) {
- consolidated_ids.push_back(values[i]);
- }
-
- delete [] values;
+ ids_t::uncompress(iter_ge_value->second, consolidated_ids);
iter_ge_value++;
}
gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());
-
consolidated_ids.erase(unique(consolidated_ids.begin(), consolidated_ids.end()), consolidated_ids.end());
uint32_t *out = nullptr;
@@ -118,23 +111,13 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids
auto it = int64map.begin();
while(it != iter_ge_value) {
- uint32_t* values = ids_t::uncompress(it->second);
-
- for(size_t i = 0; i < ids_t::num_ids(it->second); i++) {
- consolidated_ids.push_back(values[i]);
- }
-
- delete [] values;
+ ids_t::uncompress(it->second, consolidated_ids);
it++;
}
// for LESS_THAN_EQUALS, check if last iter entry is equal to value
if(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {
- uint32_t* values = ids_t::uncompress(it->second);
- for(size_t i = 0; i < ids_t::num_ids(it->second); i++) {
- consolidated_ids.push_back(values[i]);
- }
- delete [] values;
+ ids_t::uncompress(it->second, consolidated_ids);
}
gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());