Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix compact positing list capacity expansion.
Kishore Nallan7641b5fc15d8
diff --git a/include/posting.h b/include/posting.hindex 094c3853..5f8cbb98 100644--- a/include/posting.h+++ b/include/posting.h@@ -18,7 +18,7 @@ struct compact_posting_list_t {// format: num_offsets, offset1,..,offsetn, id1 | num_offsets, offset1,..,offsetn, id2uint32_t id_offsets[];- static compact_posting_list_t* create(uint32_t num_ids, uint32_t* ids, const uint32_t* offset_index,+ static compact_posting_list_t* create(uint32_t num_ids, const uint32_t* ids, const uint32_t* offset_index,uint32_t num_offsets, uint32_t* offsets);posting_list_t* to_full_posting_list();diff --git a/src/posting.cpp b/src/posting.cppindex ff7412f8..645ba009 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -11,7 +11,7 @@ int64_t compact_posting_list_t::upsert(const uint32_t id, const uint32_t* offsetint64_t new_storage_needed = 0;if(length == 0 || id > last_id) {- new_storage_needed = num_offsets + 2;+ new_storage_needed = sizeof(uint32_t) * (num_offsets + 2);if(length + new_storage_needed > capacity) {// enough storage should have been provided upstreamreturn (length + new_storage_needed) - capacity;@@ -34,7 +34,7 @@ int64_t compact_posting_list_t::upsert(const uint32_t id, const uint32_t* offsetsize_t existing_id = id_offsets[i + num_existing_offsets + 1];if(existing_id == id) {- new_storage_needed = (num_offsets - num_existing_offsets);+ new_storage_needed = sizeof(uint32_t) * (num_offsets - num_existing_offsets);if(new_storage_needed > 0) {if(length + new_storage_needed > capacity) {// enough storage should have been provided upstream@@ -71,7 +71,7 @@ int64_t compact_posting_list_t::upsert(const uint32_t id, const uint32_t* offset}else if(existing_id > id) {- new_storage_needed = (num_offsets + 2);+ new_storage_needed = sizeof(uint32_t) * (num_offsets + 2);if(length + new_storage_needed > capacity) {// enough storage should have been provided upstreamreturn (length + new_storage_needed) - capacity;@@ -134,7 +134,7 @@ void compact_posting_list_t::erase(const uint32_t id) {ids_length--;}-compact_posting_list_t* compact_posting_list_t::create(uint32_t num_ids, uint32_t* ids, const uint32_t* offset_index,+compact_posting_list_t* compact_posting_list_t::create(uint32_t num_ids, const uint32_t* ids, const uint32_t* offset_index,uint32_t num_offsets, uint32_t* offsets) {// format: num_offsets, offset1,..,offsetn, id1 | num_offsets, offset1,..,offsetn, id2@@ -420,6 +420,10 @@ bool posting_t::block_intersect(const std::vector<void*>& raw_posting_lists, siz}void posting_t::destroy_list(void*& obj) {+ if(obj == nullptr) {+ return;+ }+if(IS_COMPACT_POSTING(obj)) {compact_posting_list_t* list = COMPACT_POSTING_PTR(obj);free(list); // assigned via malloc, so must be free()ddiff --git a/src/posting_list.cpp b/src/posting_list.cppindex e95fd283..29d6ef73 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -5,7 +5,7 @@/* block_t operations */uint32_t posting_list_t::block_t::upsert(const uint32_t id, const std::vector<uint32_t>& positions) {- if(id <= ids.last()) {+ if(id <= ids.last() && ids.getLength() != 0) {// we have to check if `id` already exists, for an opportunity to do in-place updatesuint32_t id_index = ids.indexOf(id);diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 5a646abb..04234bf2 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -1043,8 +1043,10 @@ TEST(PostingListTest, CompactPostingListContainsAtleastOne) {compact_posting_list_t* list1 = compact_posting_list_t::create(4, ids, offset_index, 12, offsets);ASSERT_TRUE(list1->contains_atleast_one(&target_ids1[0], target_ids1.size()));ASSERT_FALSE(list1->contains_atleast_one(&target_ids2[0], target_ids2.size()));+ free(list1);compact_posting_list_t* list2 = static_cast<compact_posting_list_t*>(malloc(sizeof(compact_posting_list_t)));+ list2->capacity = list2->ids_length = list2->length = 0;void* obj = SET_COMPACT_POSTING(list2);posting_t::upsert(obj, 3, {1, 5});@@ -1053,6 +1055,8 @@ TEST(PostingListTest, CompactPostingListContainsAtleastOne) {ASSERT_TRUE(COMPACT_POSTING_PTR(obj)->contains_atleast_one(&target_ids3[0], target_ids3.size()));ASSERT_FALSE(COMPACT_POSTING_PTR(obj)->contains_atleast_one(&target_ids4[0], target_ids4.size()));++ posting_t::destroy_list(obj);}TEST(PostingListTest, DISABLED_Benchmark) {