Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
In place updates for posting list.
Kishore Nallan6e4ecd409caf
diff --git a/include/array_base.h b/include/array_base.hindex 3ad75207..f04bbc9d 100644--- a/include/array_base.h+++ b/include/array_base.h@@ -42,4 +42,8 @@ public:uint32_t getSizeInBytes();uint32_t getLength() const;++ uint32_t getMin() const;++ uint32_t getMax() const;};\ No newline at end of filediff --git a/src/array_base.cpp b/src/array_base.cppindex 860a1a06..f61a123c 100644--- a/src/array_base.cpp+++ b/src/array_base.cpp@@ -14,3 +14,11 @@ uint32_t array_base::getSizeInBytes() {uint32_t array_base::getLength() const {return length;}++uint32_t array_base::getMin() const {+ return min;+}++uint32_t array_base::getMax() const {+ return max;+}diff --git a/src/posting_list.cpp b/src/posting_list.cppindex c6425b95..a5229150 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -25,19 +25,120 @@ void posting_list_t::block_t::insert_and_shift_offset_index(const uint32_t index}void posting_list_t::block_t::upsert(const uint32_t id, const std::vector<uint32_t>& positions) {- size_t inserted_index = ids.append(id);-- if(inserted_index == ids.getLength()-1) {- // treat as appends- uint32_t curr_index = offsets.getLength();- offset_index.append(curr_index);- for(uint32_t position : positions) {- offsets.append(position);+ if(id <= ids.last()) {+ // we have to check if `id` already exists, for an opportunity to do in-place updates+ uint32_t id_index = ids.indexOf(id);++ if(id_index != ids.getLength()) {+ // id is already present, so we will only update offset index and offsets+ uint32_t start_offset_index = offset_index.at(id_index);+ uint32_t end_offset_index = (id == ids.last()) ? offsets.getLength()-1 : offset_index.at(id_index + 1)-1;+ uint32_t num_offsets = (end_offset_index - start_offset_index) + 1;+ uint32_t* curr_offsets = offsets.uncompress();+ uint32_t m = offsets.getMin(), M = offsets.getMax();++ if(num_offsets == positions.size()) {+ // no need to touch the offset index and need to just do inplace updates of offsets+ bool find_new_min_max = false;+ for(size_t i = 0; i < positions.size(); i++) {+ if((curr_offsets[start_offset_index + i] == m || curr_offsets[start_offset_index + i] == M) &&+ curr_offsets[start_offset_index + i] != positions[i]) {+ // when an existing min/max is affected we will have to find the new min/max+ find_new_min_max = true;+ }++ if(positions[i] < m) {+ m = positions[i];+ }++ if(positions[i] > M) {+ M = positions[i];+ }++ curr_offsets[start_offset_index + i] = positions[i];+ }++ if(find_new_min_max) {+ for(size_t i = 0; i < offsets.getLength(); i++) {+ if(curr_offsets[i] < m) {+ m = curr_offsets[i];+ }++ if(curr_offsets[i] > M) {+ M = curr_offsets[i];+ }+ }+ }++ offsets.load(curr_offsets, offsets.getLength(), m, M);+ } else {+ // need to resize offsets array+ int64_t size_diff = int64_t(positions.size()) - num_offsets; // size_diff can be negative+ size_t new_offsets_length = offsets.getLength() + size_diff;+ uint32_t* new_offsets = new uint32_t[new_offsets_length];+ std::memmove(new_offsets, curr_offsets, sizeof(uint32_t) * start_offset_index);++ bool find_new_min_max = false;+ for(size_t i = 0; i < num_offsets; i++) {+ if(curr_offsets[start_offset_index + i] == m || curr_offsets[start_offset_index + i] == M) {+ // when an existing min/max is affected we will have to find the new min/max+ find_new_min_max = true;+ }+ }++ for(size_t i = 0; i < positions.size(); i++) {+ if(positions[i] < m) {+ m = positions[i];+ }++ if(positions[i] > M) {+ M = positions[i];+ }++ new_offsets[start_offset_index + i] = positions[i];+ }++ std::memmove(new_offsets + start_offset_index + positions.size(),+ curr_offsets + end_offset_index + 1,+ sizeof(uint32_t) * (offsets.getLength() - (end_offset_index + 1)));++ if(find_new_min_max) {+ for(size_t i = 0; i < offsets.getLength(); i++) {+ if(curr_offsets[i] < m) {+ m = curr_offsets[i];+ }++ if(curr_offsets[i] > M) {+ M = curr_offsets[i];+ }+ }+ }++ offsets.load(new_offsets, new_offsets_length, m, M);+ delete [] new_offsets;++ // shift offset index+ uint32_t* current_offset_index = offset_index.uncompress();+ for(size_t i = id_index+1; i < ids.getLength(); i++) {+ current_offset_index[i] += size_diff;+ }++ offset_index.load(current_offset_index, offset_index.getLength());+ delete [] current_offset_index;+ }++ delete [] curr_offsets;+ return;}- } else {- uint32_t existing_offset_index = offset_index.at(inserted_index);- insert_and_shift_offset_index(inserted_index, positions.size());- offsets.insert(existing_offset_index, &positions[0], positions.size());+ }++ // treat as regular append (either id not found or exceeds max id)++ ids.append(id);+ uint32_t curr_index = offsets.getLength();+ offset_index.append(curr_index);+ for(uint32_t position : positions) {+ offsets.append(position);}}diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 08f2657c..638e8007 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -110,6 +110,155 @@ TEST(PostingListTest, Insert) {}}+TEST(PostingListTest, InplaceUpserts) {+ std::vector<uint32_t> offsets = {1, 2, 3};+ posting_list_t pl(5);++ pl.upsert(2, offsets);+ pl.upsert(5, offsets);+ pl.upsert(7, offsets);++ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(9, pl.get_root()->offsets.getLength());++ // update starting ID with same length of offsets+ pl.upsert(2, {1, 2, 4});+ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(9, pl.get_root()->offsets.getLength());++ ASSERT_EQ(1, pl.get_root()->offsets.at(0));+ ASSERT_EQ(2, pl.get_root()->offsets.at(1));+ ASSERT_EQ(4, pl.get_root()->offsets.at(2));+ ASSERT_EQ(4, pl.get_root()->offsets.getMax());+ ASSERT_EQ(1, pl.get_root()->offsets.getMin());++ ASSERT_EQ(0, pl.get_root()->offset_index.at(0));+ ASSERT_EQ(3, pl.get_root()->offset_index.at(1));+ ASSERT_EQ(6, pl.get_root()->offset_index.at(2));++ // update starting ID with smaller number of offsets+ pl.upsert(2, {5, 7});+ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(8, pl.get_root()->offsets.getLength());++ ASSERT_EQ(5, pl.get_root()->offsets.at(0));+ ASSERT_EQ(7, pl.get_root()->offsets.at(1));+ ASSERT_EQ(1, pl.get_root()->offsets.at(2));+ ASSERT_EQ(7, pl.get_root()->offsets.getMax());+ ASSERT_EQ(1, pl.get_root()->offsets.getMin());++ ASSERT_EQ(0, pl.get_root()->offset_index.at(0));+ ASSERT_EQ(2, pl.get_root()->offset_index.at(1));+ ASSERT_EQ(5, pl.get_root()->offset_index.at(2));++ // update starting ID with larger number of offsets+ pl.upsert(2, {0, 2, 8});+ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(9, pl.get_root()->offsets.getLength());++ ASSERT_EQ(0, pl.get_root()->offsets.at(0));+ ASSERT_EQ(2, pl.get_root()->offsets.at(1));+ ASSERT_EQ(8, pl.get_root()->offsets.at(2));+ ASSERT_EQ(1, pl.get_root()->offsets.at(3));+ ASSERT_EQ(8, pl.get_root()->offsets.getMax());+ ASSERT_EQ(0, pl.get_root()->offsets.getMin());++ ASSERT_EQ(0, pl.get_root()->offset_index.at(0));+ ASSERT_EQ(3, pl.get_root()->offset_index.at(1));+ ASSERT_EQ(6, pl.get_root()->offset_index.at(2));++ // update middle ID with smaller number of offsets+ pl.upsert(5, {1, 10});+ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(8, pl.get_root()->offsets.getLength());++ ASSERT_EQ(0, pl.get_root()->offsets.at(0));+ ASSERT_EQ(2, pl.get_root()->offsets.at(1));+ ASSERT_EQ(8, pl.get_root()->offsets.at(2));+ ASSERT_EQ(1, pl.get_root()->offsets.at(3));+ ASSERT_EQ(10, pl.get_root()->offsets.at(4));++ ASSERT_EQ(10, pl.get_root()->offsets.getMax());+ ASSERT_EQ(0, pl.get_root()->offsets.getMin());++ ASSERT_EQ(0, pl.get_root()->offset_index.at(0));+ ASSERT_EQ(3, pl.get_root()->offset_index.at(1));+ ASSERT_EQ(5, pl.get_root()->offset_index.at(2));++ // update middle ID with larger number of offsets+ pl.upsert(5, {2, 4, 12});+ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(9, pl.get_root()->offsets.getLength());++ ASSERT_EQ(0, pl.get_root()->offsets.at(0));+ ASSERT_EQ(2, pl.get_root()->offsets.at(1));+ ASSERT_EQ(8, pl.get_root()->offsets.at(2));+ ASSERT_EQ(2, pl.get_root()->offsets.at(3));+ ASSERT_EQ(4, pl.get_root()->offsets.at(4));+ ASSERT_EQ(12, pl.get_root()->offsets.at(5));+ ASSERT_EQ(1, pl.get_root()->offsets.at(6));+ ASSERT_EQ(2, pl.get_root()->offsets.at(7));+ ASSERT_EQ(3, pl.get_root()->offsets.at(8));++ ASSERT_EQ(12, pl.get_root()->offsets.getMax());+ ASSERT_EQ(0, pl.get_root()->offsets.getMin());++ ASSERT_EQ(0, pl.get_root()->offset_index.at(0));+ ASSERT_EQ(3, pl.get_root()->offset_index.at(1));+ ASSERT_EQ(6, pl.get_root()->offset_index.at(2));++ // update last ID with smaller number of offsets++ pl.upsert(7, {3});+ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(7, pl.get_root()->offsets.getLength());++ ASSERT_EQ(0, pl.get_root()->offsets.at(0));+ ASSERT_EQ(2, pl.get_root()->offsets.at(1));+ ASSERT_EQ(8, pl.get_root()->offsets.at(2));+ ASSERT_EQ(2, pl.get_root()->offsets.at(3));+ ASSERT_EQ(4, pl.get_root()->offsets.at(4));+ ASSERT_EQ(12, pl.get_root()->offsets.at(5));+ ASSERT_EQ(3, pl.get_root()->offsets.at(6));++ ASSERT_EQ(12, pl.get_root()->offsets.getMax());+ ASSERT_EQ(0, pl.get_root()->offsets.getMin());++ ASSERT_EQ(0, pl.get_root()->offset_index.at(0));+ ASSERT_EQ(3, pl.get_root()->offset_index.at(1));+ ASSERT_EQ(6, pl.get_root()->offset_index.at(2));++ // update last ID with larger number of offsets++ pl.upsert(7, {5, 20});+ ASSERT_EQ(1, pl.size());+ ASSERT_EQ(3, pl.get_root()->ids.getLength());+ ASSERT_EQ(8, pl.get_root()->offsets.getLength());++ ASSERT_EQ(0, pl.get_root()->offsets.at(0));+ ASSERT_EQ(2, pl.get_root()->offsets.at(1));+ ASSERT_EQ(8, pl.get_root()->offsets.at(2));+ ASSERT_EQ(2, pl.get_root()->offsets.at(3));+ ASSERT_EQ(4, pl.get_root()->offsets.at(4));+ ASSERT_EQ(12, pl.get_root()->offsets.at(5));+ ASSERT_EQ(5, pl.get_root()->offsets.at(6));+ ASSERT_EQ(20, pl.get_root()->offsets.at(7));++ ASSERT_EQ(20, pl.get_root()->offsets.getMax());+ ASSERT_EQ(0, pl.get_root()->offsets.getMin());++ ASSERT_EQ(0, pl.get_root()->offset_index.at(0));+ ASSERT_EQ(3, pl.get_root()->offset_index.at(1));+ ASSERT_EQ(6, pl.get_root()->offset_index.at(2));+}+TEST(PostingListTest, RemovalsOnFirstBlock) {std::vector<uint32_t> offsets = {0, 1, 3};posting_list_t pl(5);@@ -318,11 +467,10 @@ TEST(PostingListTest, RandomInsertAndDeletes) {std::vector<uint32_t> offsets1 = {0, 1, 3};std::vector<uint32_t> offsets2 = {10, 12};- // generate unique random IDs- std::set<uint32_t> ids;+ std::vector<uint32_t> ids;for(size_t i = 0; i < 100000; i++) {- ids.insert(rand() % 100000);+ ids.push_back(rand() % 100000);}size_t index = 0;@@ -337,8 +485,8 @@ TEST(PostingListTest, RandomInsertAndDeletes) {pl.erase(rand() % 100000);}- ASSERT_LT(pl.size(), 750);- ASSERT_GT(pl.size(), 500);+ ASSERT_GT(pl.size(), 750);+ ASSERT_LT(pl.size(), 1000);}TEST(PostingListTest, IntersectionBasics) {