Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Simplify posting list offset storage and extraction.

Kishore Nalland54957c0c6aa
diff --git a/include/posting.h b/include/posting.h
index d9ae5e6d..767fab1f 100644
--- a/include/posting.h
+++ b/include/posting.h
@@ -65,6 +65,8 @@ public:
for(const auto& posting_list: plists) {
its.push_back(posting_list->new_iterator());
}
+
+ iter_state.num_lists = plists.size();
}
~block_intersector_t() {
diff --git a/include/posting_list.h b/include/posting_list.h
index 938a5135..657d3a39 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -64,8 +64,9 @@ public:
};
struct result_iter_state_t {
- std::vector<std::vector<block_t*>> blocks;
- std::vector<std::vector<uint32_t>> indices;
+ size_t num_lists;
+ std::vector<block_t*> blocks;
+ std::vector<uint32_t> indices;
std::vector<uint32_t> ids;
};
diff --git a/src/posting.cpp b/src/posting.cpp
index f5b9c81d..2a3b4745 100644
--- a/src/posting.cpp
+++ b/src/posting.cpp
@@ -156,7 +156,7 @@ compact_posting_list_t* compact_posting_list_t::create(uint32_t num_ids, const u
}
posting_list_t* compact_posting_list_t::to_full_posting_list() {
- posting_list_t* pl = new posting_list_t(512);
+ posting_list_t* pl = new posting_list_t(256);
size_t i = 0;
while(i < length) {
@@ -426,12 +426,10 @@ void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>&
posting_list_t::result_iter_state_t iter_state;
iter_state.ids.push_back(id);
+ iter_state.num_lists = plists.size();
- iter_state.indices.emplace_back();
- iter_state.blocks.emplace_back();
-
- std::vector<posting_list_t::block_t*>& block_vec = iter_state.blocks.back();
- std::vector<uint32_t>& index_vec = iter_state.indices.back();
+ std::vector<posting_list_t::block_t*>& block_vec = iter_state.blocks;
+ std::vector<uint32_t>& index_vec = iter_state.indices;
for(posting_list_t* pl: plists) {
posting_list_t::block_t* block = pl->block_of(id);
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 47ae7ade..32e74a18 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -675,6 +675,8 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting
for(const auto& posting_list: posting_lists) {
its.push_back(posting_list->new_iterator());
}
+
+ iter_state.num_lists = posting_lists.size();
} else {
// already in the middle of iteration: prepare for next batch
iter_state.ids.clear();
@@ -692,11 +694,8 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting
std::vector<block_t*> block_vec(1);
std::vector<uint32_t> index_vec(1);
- block_vec[0] = its[0].block();
- index_vec[0] = its[0].index();
-
- iter_state.blocks.emplace_back(block_vec);
- iter_state.indices.emplace_back(index_vec);
+ iter_state.blocks.push_back(its[0].block());
+ iter_state.indices.push_back(its[0].index());
its[0].next();
@@ -711,16 +710,11 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting
// still need to ensure that the ID exists in inclusion list but NOT in exclusion list
iter_state.ids.push_back(its[0].id());
- std::vector<block_t*> block_vec(2);
- std::vector<uint32_t> index_vec(2);
- block_vec[0] = its[0].block();
- block_vec[1] = its[1].block();
-
- index_vec[0] = its[0].index();
- index_vec[1] = its[1].index();
+ iter_state.blocks.push_back(its[0].block());
+ iter_state.blocks.push_back(its[1].block());
- iter_state.blocks.emplace_back(block_vec);
- iter_state.indices.emplace_back(index_vec);
+ iter_state.indices.push_back(its[0].index());
+ iter_state.indices.push_back(its[1].index());
advance_all2(its);
} else {
@@ -738,17 +732,11 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting
//LOG(INFO) << its[0].id();
iter_state.ids.push_back(its[0].id());
- std::vector<block_t*> block_vec(its.size());
- std::vector<uint32_t> index_vec(its.size());
-
for(size_t i = 0; i < its.size(); i++) {
- block_vec[i] = its[i].block();
- index_vec[i] = its[i].index();
+ iter_state.blocks.push_back(its[i].block());
+ iter_state.indices.push_back(its[i].index());
}
- iter_state.blocks.emplace_back(block_vec);
- iter_state.indices.emplace_back(index_vec);
-
advance_all(its);
} else {
advance_non_largest(its);
@@ -775,14 +763,16 @@ bool posting_list_t::get_offsets(posting_list_t::result_iter_state_t& iter_state
// For each result ID and for each block it is contained in, calculate offsets
+ size_t id_block_index = 0;
+
for(size_t i = 0; i < iter_state.ids.size(); i++) {
uint32_t id = iter_state.ids[i];
array_token_positions_vec.emplace_back();
std::unordered_map<size_t, std::vector<token_positions_t>>& array_tok_pos = array_token_positions_vec.back();
- for(size_t j = 0; j < iter_state.blocks[i].size(); j++) {
- block_t* curr_block = iter_state.blocks[i][j];
- uint32_t curr_index = iter_state.indices[i][j];
+ for(size_t j = 0; j < iter_state.num_lists; j++) {
+ block_t* curr_block = iter_state.blocks[id_block_index];
+ uint32_t curr_index = iter_state.indices[id_block_index++];
if(curr_block == nullptr || curr_index == UINT32_MAX) {
continue;
diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cpp
index 2444c2f5..6e8d67e1 100644
--- a/test/posting_list_test.cpp
+++ b/test/posting_list_test.cpp
@@ -614,11 +614,8 @@ TEST(PostingListTest, IntersectionBasics) {
ASSERT_EQ(3, iter_state.ids[0]);
ASSERT_EQ(20, iter_state.ids[1]);
- ASSERT_EQ(2, iter_state.blocks.size());
- ASSERT_EQ(3, iter_state.blocks[0].size());
- ASSERT_EQ(3, iter_state.blocks[1].size());
-
- ASSERT_EQ(2, iter_state.indices.size());
+ ASSERT_EQ(6, iter_state.blocks.size());
+ ASSERT_EQ(6, iter_state.indices.size());
// try with smaller batch size
@@ -653,22 +650,17 @@ TEST(PostingListTest, IntersectionBasics) {
ASSERT_EQ(0, iter_state3.ids[0]);
ASSERT_EQ(2, iter_state3.ids[1]);
ASSERT_EQ(2, iter_state3.blocks.size());
- ASSERT_EQ(1, iter_state3.blocks[0].size());
- ASSERT_EQ(1, iter_state3.blocks[1].size());
ASSERT_EQ(2, iter_state3.indices.size());
- ASSERT_EQ(1, iter_state3.indices[0].size());
- ASSERT_EQ(0, iter_state3.indices[0][0]);
- ASSERT_EQ(1, iter_state3.indices[1][0]);
+ ASSERT_EQ(0, iter_state3.indices[0]);
+ ASSERT_EQ(1, iter_state3.indices[1]);
has_more = posting_list_t::block_intersect(single_item_list, 2, its3, iter_state3);
ASSERT_FALSE(has_more);
ASSERT_EQ(2, iter_state3.ids.size());
ASSERT_EQ(2, iter_state3.blocks.size());
- ASSERT_EQ(1, iter_state3.blocks[0].size());
- ASSERT_EQ(1, iter_state3.blocks[1].size());
ASSERT_EQ(2, iter_state3.indices.size());
- ASSERT_EQ(1, iter_state3.indices[0].size());
- ASSERT_EQ(1, iter_state3.indices[1].size());
+ ASSERT_EQ(0, iter_state3.indices[0]);
+ ASSERT_EQ(1, iter_state3.indices[1]);
// empty intersection list
std::vector<posting_list_t*> empty_list;