Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Simplify posting list offset storage and extraction.
Kishore Nalland54957c0c6aa
diff --git a/include/posting.h b/include/posting.hindex d9ae5e6d..767fab1f 100644--- a/include/posting.h+++ b/include/posting.h@@ -65,6 +65,8 @@ public:for(const auto& posting_list: plists) {its.push_back(posting_list->new_iterator());}++ iter_state.num_lists = plists.size();}~block_intersector_t() {diff --git a/include/posting_list.h b/include/posting_list.hindex 938a5135..657d3a39 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -64,8 +64,9 @@ public:};struct result_iter_state_t {- std::vector<std::vector<block_t*>> blocks;- std::vector<std::vector<uint32_t>> indices;+ size_t num_lists;+ std::vector<block_t*> blocks;+ std::vector<uint32_t> indices;std::vector<uint32_t> ids;};diff --git a/src/posting.cpp b/src/posting.cppindex f5b9c81d..2a3b4745 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -156,7 +156,7 @@ compact_posting_list_t* compact_posting_list_t::create(uint32_t num_ids, const u}posting_list_t* compact_posting_list_t::to_full_posting_list() {- posting_list_t* pl = new posting_list_t(512);+ posting_list_t* pl = new posting_list_t(256);size_t i = 0;while(i < length) {@@ -426,12 +426,10 @@ void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>&posting_list_t::result_iter_state_t iter_state;iter_state.ids.push_back(id);+ iter_state.num_lists = plists.size();- iter_state.indices.emplace_back();- iter_state.blocks.emplace_back();-- std::vector<posting_list_t::block_t*>& block_vec = iter_state.blocks.back();- std::vector<uint32_t>& index_vec = iter_state.indices.back();+ std::vector<posting_list_t::block_t*>& block_vec = iter_state.blocks;+ std::vector<uint32_t>& index_vec = iter_state.indices;for(posting_list_t* pl: plists) {posting_list_t::block_t* block = pl->block_of(id);diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 47ae7ade..32e74a18 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -675,6 +675,8 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& postingfor(const auto& posting_list: posting_lists) {its.push_back(posting_list->new_iterator());}++ iter_state.num_lists = posting_lists.size();} else {// already in the middle of iteration: prepare for next batchiter_state.ids.clear();@@ -692,11 +694,8 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& postingstd::vector<block_t*> block_vec(1);std::vector<uint32_t> index_vec(1);- block_vec[0] = its[0].block();- index_vec[0] = its[0].index();-- iter_state.blocks.emplace_back(block_vec);- iter_state.indices.emplace_back(index_vec);+ iter_state.blocks.push_back(its[0].block());+ iter_state.indices.push_back(its[0].index());its[0].next();@@ -711,16 +710,11 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting// still need to ensure that the ID exists in inclusion list but NOT in exclusion listiter_state.ids.push_back(its[0].id());- std::vector<block_t*> block_vec(2);- std::vector<uint32_t> index_vec(2);- block_vec[0] = its[0].block();- block_vec[1] = its[1].block();-- index_vec[0] = its[0].index();- index_vec[1] = its[1].index();+ iter_state.blocks.push_back(its[0].block());+ iter_state.blocks.push_back(its[1].block());- iter_state.blocks.emplace_back(block_vec);- iter_state.indices.emplace_back(index_vec);+ iter_state.indices.push_back(its[0].index());+ iter_state.indices.push_back(its[1].index());advance_all2(its);} else {@@ -738,17 +732,11 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting//LOG(INFO) << its[0].id();iter_state.ids.push_back(its[0].id());- std::vector<block_t*> block_vec(its.size());- std::vector<uint32_t> index_vec(its.size());-for(size_t i = 0; i < its.size(); i++) {- block_vec[i] = its[i].block();- index_vec[i] = its[i].index();+ iter_state.blocks.push_back(its[i].block());+ iter_state.indices.push_back(its[i].index());}- iter_state.blocks.emplace_back(block_vec);- iter_state.indices.emplace_back(index_vec);-advance_all(its);} else {advance_non_largest(its);@@ -775,14 +763,16 @@ bool posting_list_t::get_offsets(posting_list_t::result_iter_state_t& iter_state// For each result ID and for each block it is contained in, calculate offsets+ size_t id_block_index = 0;+for(size_t i = 0; i < iter_state.ids.size(); i++) {uint32_t id = iter_state.ids[i];array_token_positions_vec.emplace_back();std::unordered_map<size_t, std::vector<token_positions_t>>& array_tok_pos = array_token_positions_vec.back();- for(size_t j = 0; j < iter_state.blocks[i].size(); j++) {- block_t* curr_block = iter_state.blocks[i][j];- uint32_t curr_index = iter_state.indices[i][j];+ for(size_t j = 0; j < iter_state.num_lists; j++) {+ block_t* curr_block = iter_state.blocks[id_block_index];+ uint32_t curr_index = iter_state.indices[id_block_index++];if(curr_block == nullptr || curr_index == UINT32_MAX) {continue;diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 2444c2f5..6e8d67e1 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -614,11 +614,8 @@ TEST(PostingListTest, IntersectionBasics) {ASSERT_EQ(3, iter_state.ids[0]);ASSERT_EQ(20, iter_state.ids[1]);- ASSERT_EQ(2, iter_state.blocks.size());- ASSERT_EQ(3, iter_state.blocks[0].size());- ASSERT_EQ(3, iter_state.blocks[1].size());-- ASSERT_EQ(2, iter_state.indices.size());+ ASSERT_EQ(6, iter_state.blocks.size());+ ASSERT_EQ(6, iter_state.indices.size());// try with smaller batch size@@ -653,22 +650,17 @@ TEST(PostingListTest, IntersectionBasics) {ASSERT_EQ(0, iter_state3.ids[0]);ASSERT_EQ(2, iter_state3.ids[1]);ASSERT_EQ(2, iter_state3.blocks.size());- ASSERT_EQ(1, iter_state3.blocks[0].size());- ASSERT_EQ(1, iter_state3.blocks[1].size());ASSERT_EQ(2, iter_state3.indices.size());- ASSERT_EQ(1, iter_state3.indices[0].size());- ASSERT_EQ(0, iter_state3.indices[0][0]);- ASSERT_EQ(1, iter_state3.indices[1][0]);+ ASSERT_EQ(0, iter_state3.indices[0]);+ ASSERT_EQ(1, iter_state3.indices[1]);has_more = posting_list_t::block_intersect(single_item_list, 2, its3, iter_state3);ASSERT_FALSE(has_more);ASSERT_EQ(2, iter_state3.ids.size());ASSERT_EQ(2, iter_state3.blocks.size());- ASSERT_EQ(1, iter_state3.blocks[0].size());- ASSERT_EQ(1, iter_state3.blocks[1].size());ASSERT_EQ(2, iter_state3.indices.size());- ASSERT_EQ(1, iter_state3.indices[0].size());- ASSERT_EQ(1, iter_state3.indices[1].size());+ ASSERT_EQ(0, iter_state3.indices[0]);+ ASSERT_EQ(1, iter_state3.indices[1]);// empty intersection liststd::vector<posting_list_t*> empty_list;