Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Improve posting list cached ids decompression.

Kishore Nallanfcbca97f5f9e
diff --git a/include/posting_list.h b/include/posting_list.h
index 54765eba..3580e7b6 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -43,7 +43,11 @@ public:
private:
block_t* block;
uint32_t index;
+
+ // uncompressed data structures for performance
+ block_t* uncompressed_block;
uint32_t* ids;
+
public:
explicit iterator_t(block_t* root);
iterator_t(iterator_t&& rhs) noexcept;
@@ -51,7 +55,7 @@ public:
[[nodiscard]] bool valid() const;
void next();
void skip_to(uint32_t id);
- [[nodiscard]] uint32_t id() const;
+ [[nodiscard]] inline uint32_t id();
void offsets(std::vector<uint32_t>& offsets);
};
@@ -75,8 +79,8 @@ private:
static bool at_end(const std::vector<posting_list_t::iterator_t>& its);
static bool at_end2(const std::vector<posting_list_t::iterator_t>& its);
- static bool equals(const std::vector<posting_list_t::iterator_t>& its);
- static bool equals2(const std::vector<posting_list_t::iterator_t>& its);
+ static bool equals(std::vector<posting_list_t::iterator_t>& its);
+ static bool equals2(std::vector<posting_list_t::iterator_t>& its);
static void advance_all(std::vector<posting_list_t::iterator_t>& its);
static void advance_all2(std::vector<posting_list_t::iterator_t>& its);
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 753f5ced..18986bb6 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -582,7 +582,7 @@ bool posting_list_t::at_end2(const std::vector<posting_list_t::iterator_t>& its)
return !its[0].valid() || !its[1].valid();
}
-bool posting_list_t::equals(const std::vector<posting_list_t::iterator_t>& its) {
+bool posting_list_t::equals(std::vector<posting_list_t::iterator_t>& its) {
for(size_t i = 0; i < its.size() - 1; i++) {
if(its[i].id() != its[i+1].id()) {
return false;
@@ -592,7 +592,7 @@ bool posting_list_t::equals(const std::vector<posting_list_t::iterator_t>& its)
return true;
}
-bool posting_list_t::equals2(const std::vector<posting_list_t::iterator_t>& its) {
+bool posting_list_t::equals2(std::vector<posting_list_t::iterator_t>& its) {
return its[0].id() == its[1].id();
}
@@ -642,8 +642,9 @@ size_t posting_list_t::num_ids() {
/* iterator_t operations */
-posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root): block(root), index(0) {
- ids = root->ids.uncompress();
+posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root):
+ block(root), index(0), uncompressed_block(nullptr), ids(nullptr) {
+
}
bool posting_list_t::iterator_t::valid() const {
@@ -655,16 +656,22 @@ void posting_list_t::iterator_t::next() {
if(index == block->size()) {
index = 0;
block = block->next;
+ }
+}
+
+uint32_t posting_list_t::iterator_t::id() {
+ //return block->ids.at(index);
+
+ if(uncompressed_block != block) {
delete [] ids;
ids = nullptr;
+ uncompressed_block = block;
+
if(block != nullptr) {
ids = block->ids.uncompress();
}
}
-}
-uint32_t posting_list_t::iterator_t::id() const {
- //return block->ids.at(index);
return ids[index];
}
@@ -681,15 +688,9 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {
if(skipped_block) {
index = 0;
- delete [] ids;
- ids = nullptr;
-
- if(block != nullptr) {
- ids = block->ids.uncompress();
- }
}
- while(block != nullptr && index < block->size() && ids[index] < id) {
+ while(block != nullptr && index < block->size() && this->id() < id) {
index++;
}
}
@@ -702,6 +703,7 @@ posting_list_t::iterator_t::~iterator_t() {
posting_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {
block = rhs.block;
index = rhs.index;
+ uncompressed_block = rhs.uncompressed_block;
ids = rhs.ids;
rhs.block = nullptr;