Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix posting list deletion edge case.
Kishore Nallan6ded5c7556f7
diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 823dd028..c888225e 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -186,7 +186,9 @@ bool posting_list_t::block_t::contains(uint32_t id) {/* posting_list_t operations */posting_list_t::posting_list_t(uint16_t max_block_elements): BLOCK_MAX_ELEMENTS(max_block_elements) {-+ if(max_block_elements <= 1) {+ throw std::invalid_argument("max_block_elements must be > 1");+ }}posting_list_t::~posting_list_t() {@@ -423,7 +425,7 @@ void posting_list_t::upsert(const uint32_t id, const std::vector<uint32_t>& offs} else {const auto it = id_block_map.lower_bound(id);upsert_block = (it == id_block_map.end()) ? id_block_map.rbegin()->second : it->second;- before_upsert_last_id = upsert_block->ids.at(upsert_block->size() - 1);+ before_upsert_last_id = upsert_block->ids.last();}// happy path: upsert_block is not full@@ -431,7 +433,7 @@ void posting_list_t::upsert(const uint32_t id, const std::vector<uint32_t>& offsuint32_t num_inserted = upsert_block->upsert(id, offsets);ids_length += num_inserted;- last_id_t after_upsert_last_id = upsert_block->ids.at(upsert_block->size() - 1);+ last_id_t after_upsert_last_id = upsert_block->ids.last();if(before_upsert_last_id != after_upsert_last_id) {id_block_map.erase(before_upsert_last_id);id_block_map.emplace(after_upsert_last_id, upsert_block);@@ -451,12 +453,12 @@ void posting_list_t::upsert(const uint32_t id, const std::vector<uint32_t>& offs// evenly divide elements between both blockssplit_block(upsert_block, new_block);- last_id_t after_upsert_last_id = upsert_block->ids.at(upsert_block->size() - 1);+ last_id_t after_upsert_last_id = upsert_block->ids.last();id_block_map.erase(before_upsert_last_id);id_block_map.emplace(after_upsert_last_id, upsert_block);}- last_id_t after_new_block_id = new_block->ids.at(new_block->size() - 1);+ last_id_t after_new_block_id = new_block->ids.last();id_block_map.emplace(after_new_block_id, new_block);new_block->next = upsert_block->next;@@ -485,6 +487,17 @@ void posting_list_t::erase(const uint32_t id) {// since we will be deleting the empty node, set the previous node's next pointer to nullstd::prev(it)->second->next = nullptr;delete erase_block;+ } else {+ // The root block cannot be empty if there are other blocks so we will pull some contents from next block+ // This is only an issue for blocks with max size of 2+ if(root_block.next != nullptr) {+ auto next_block_last_id = erase_block->next->ids.last();+ merge_adjacent_blocks(erase_block, erase_block->next, erase_block->next->size()/2);+ id_block_map.erase(next_block_last_id);++ id_block_map.emplace(erase_block->next->ids.last(), erase_block->next);+ id_block_map.emplace(erase_block->ids.last(), erase_block);+ }}id_block_map.erase(before_last_id);@@ -493,7 +506,7 @@ void posting_list_t::erase(const uint32_t id) {}if(new_ids_length >= BLOCK_MAX_ELEMENTS/2 || erase_block->next == nullptr) {- last_id_t after_last_id = erase_block->ids.at(new_ids_length-1);+ last_id_t after_last_id = erase_block->ids.last();if(before_last_id != after_last_id) {id_block_map.erase(before_last_id);id_block_map.emplace(after_last_id, erase_block);@@ -505,7 +518,7 @@ void posting_list_t::erase(const uint32_t id) {// block is less than 50% of max capacity and contains a next node which we can refill fromauto next_block = erase_block->next;- last_id_t next_block_last_id = next_block->ids.at(next_block->ids.getLength()-1);+ last_id_t next_block_last_id = next_block->ids.last();if(erase_block->size() + next_block->size() <= BLOCK_MAX_ELEMENTS) {// we can merge the contents of next block with `erase_block` and delete the next block@@ -515,13 +528,15 @@ void posting_list_t::erase(const uint32_t id) {id_block_map.erase(next_block_last_id);} else {- // only part of the next block can be moved over- size_t num_block2_ids = BLOCK_MAX_ELEMENTS - erase_block->size();+ // Only part of the next block can be moved over.+ // We will move only 50% of max elements to ensure that we don't end up "flipping" adjacent blocks:+ // 1, 5 -> 5, 1+ size_t num_block2_ids = BLOCK_MAX_ELEMENTS/2;merge_adjacent_blocks(erase_block, next_block, num_block2_ids);// NOTE: we don't have to update `id_block_map` for `next_block` as last element doesn't change}- last_id_t after_last_id = erase_block->ids.at(erase_block->ids.getLength()-1);+ last_id_t after_last_id = erase_block->ids.last();if(before_last_id != after_last_id) {id_block_map.erase(before_last_id);id_block_map.emplace(after_last_id, erase_block);diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 09c567ef..b9ecec3d 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -447,13 +447,13 @@ TEST_F(PostingListTest, RemovalsOnLaterBlocks) {// only part of the next node contents can be moved over when we delete 8 since (1 + 5) > 5pl.erase(8);- // [0..4], [9], [10..14] => [0..4], [9,10,11,12,13], [14]+ // [0..4], [9], [10..14] => [0..4], [9,10,11], [12,13,14]ASSERT_EQ(3, pl.num_blocks());ASSERT_EQ(11, pl.num_ids());- ASSERT_EQ(5, pl.get_root()->next->size());- ASSERT_EQ(1, pl.get_root()->next->next->size());- ASSERT_EQ(13, pl.get_root()->next->ids.last());+ ASSERT_EQ(3, pl.get_root()->next->size());+ ASSERT_EQ(3, pl.get_root()->next->next->size());+ ASSERT_EQ(11, pl.get_root()->next->ids.last());ASSERT_EQ(14, pl.get_root()->next->next->ids.last());for(size_t i = 0; i < pl.get_root()->next->offset_index.getLength(); i++) {@@ -616,27 +616,6 @@ TEST_F(PostingListTest, SplittingOfListsSimple) {std::vector<std::vector<posting_list_t::iterator_t>> partial_its_vec(4);intersector.split_lists(4, partial_its_vec);- /*for(size_t i = 0; i < partial_its_vec.size(); i++) {- auto& partial_its = partial_its_vec[i];-- if (partial_its.empty()) {- continue;- }-- LOG(INFO) << "Vec " << i;-- for (auto& it: partial_its) {- while (it.valid()) {- LOG(INFO) << it.id();- it.next();- }-- LOG(INFO) << "---";- }- }-- return ;*/-std::vector<std::vector<std::vector<uint32_t>>> split_ids = {{{0, 2}, {1, 3}, {2, 3}},{{3, 20}, {1, 3, 5, 10, 20}, {2, 3, 5, 7, 20}}@@ -1384,6 +1363,94 @@ TEST_F(PostingListTest, BlockIntersectionOnMixedLists) {free(list1);}+TEST_F(PostingListTest, InsertAndEraseSequence) {+ std::vector<uint32_t> offsets = {0, 1, 3};+ posting_list_t pl(5);++ pl.upsert(0, offsets);+ pl.upsert(2, offsets);+ pl.upsert(4, offsets);+ pl.upsert(6, offsets);+ pl.upsert(8, offsets);++ // this will cause a split of the root block+ pl.upsert(3, offsets); // 0,2,3 | 4,6,8+ pl.erase(0); // 2,3 | 4,6,8+ pl.upsert(5, offsets); // 2,3 | 4,5,6,8+ pl.upsert(7, offsets); // 2,3 | 4,5,6,7,8+ pl.upsert(10, offsets); // 2,3 | 4,5,6,7,8 | 10++ // this will cause adjacent block refill+ pl.erase(2); // 3,4,5,6,7 | 8 | 10++ // deletes second block+ pl.erase(8);++ // remove all elements+ pl.erase(3);+ pl.erase(4);+ pl.erase(5);+ pl.erase(6);+ pl.erase(7);+ pl.erase(10);++ ASSERT_EQ(0, pl.num_ids());+}++TEST_F(PostingListTest, InsertAndEraseSequenceWithBlockSizeTwo) {+ std::vector<uint32_t> offsets = {0, 1, 3};+ posting_list_t pl(2);++ pl.upsert(2, offsets);+ pl.upsert(3, offsets);+ pl.upsert(1, offsets); // inserting 2 again here? // inserting 4 here?++ // 1 | 2,3++ pl.erase(1);++ ASSERT_EQ(1, pl.get_root()->size());+ ASSERT_EQ(2, pl.num_blocks());++ pl.erase(3);+ pl.erase(2);++ ASSERT_EQ(0, pl.get_root()->size());+}++TEST_F(PostingListTest, PostingListMustHaveAtleast1Element) {+ try {+ std::vector<uint32_t> offsets = {0, 1, 3};+ posting_list_t pl(1);+ FAIL() << "Expected std::invalid_argument";+ }+ catch(std::invalid_argument const & err) {+ EXPECT_EQ(err.what(),std::string("max_block_elements must be > 1"));+ } catch(...) {+ FAIL() << "Expected std::invalid_argument";+ }+}++TEST_F(PostingListTest, DISABLED_RandInsertAndErase) {+ std::vector<uint32_t> offsets = {0, 1, 3};+ posting_list_t pl(5);++ time_t t;+ srand((unsigned) time(&t));++ for(size_t i = 0; i < 10000; i++) {+ LOG(INFO) << "i: " << i;+ uint32_t add_id = rand() % 15;+ pl.upsert(add_id, offsets);++ uint32_t del_id = rand() % 15;+ LOG(INFO) << "add: " << add_id << ", erase: " << del_id;+ pl.erase(del_id);+ }++ LOG(INFO) << "Num ids: " << pl.num_ids() << ", num bocks: " << pl.num_blocks();+}+TEST_F(PostingListTest, DISABLED_Benchmark) {std::vector<uint32_t> offsets = {0, 1, 3};posting_list_t pl(4096);