Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Posting list merge.
Kishore Nallan267aabb70150
diff --git a/include/posting.h b/include/posting.hindex 61a703b8..3596c963 100644--- a/include/posting.h+++ b/include/posting.h@@ -38,6 +38,9 @@ class posting_t {private:static constexpr size_t COMPACT_LIST_THRESHOLD_LENGTH = 64;+ static void to_expanded_plists(const std::vector<void*>& raw_posting_lists, std::vector<posting_list_t*>& plists,+ std::vector<uint32_t>& expanded_plist_indices);+public:static void upsert(void*& obj, uint32_t id, const std::vector<uint32_t>& offsets);@@ -48,5 +51,7 @@ public:static uint32_t first_id(const void* obj);+ static void merge(const std::vector<void*>& posting_lists, std::vector<uint32_t>& result_ids);+static void intersect(const std::vector<void*>& posting_lists, std::vector<uint32_t>& result_ids);};\ No newline at end of filediff --git a/include/posting_list.h b/include/posting_list.hindex 681a382f..9ba6ff7d 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -95,8 +95,11 @@ private:static void advance_all(std::vector<posting_list_t::iterator_t>& its);static void advance_all2(std::vector<posting_list_t::iterator_t>& its);- static void advance_least(std::vector<posting_list_t::iterator_t>& its);- static void advance_least2(std::vector<posting_list_t::iterator_t>& its);+ static void advance_non_largest(std::vector<posting_list_t::iterator_t>& its);+ static void advance_non_largest2(std::vector<posting_list_t::iterator_t>& its);++ static uint32_t advance_smallest(std::vector<posting_list_t::iterator_t>& its);+ static uint32_t advance_smallest2(std::vector<posting_list_t::iterator_t>& its);public:@@ -122,6 +125,8 @@ public:iterator_t new_iterator();+ static void merge(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids);+static void intersect(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids);static bool block_intersect(diff --git a/src/posting.cpp b/src/posting.cppindex 06b8058e..7b04441c 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -293,11 +293,34 @@ uint32_t posting_t::first_id(const void* obj) {}}+void posting_t::merge(const std::vector<void*>& raw_posting_lists, std::vector<uint32_t>& result_ids) {+ // we will have to convert the compact posting list (if any) to full form+ std::vector<posting_list_t*> plists;+ std::vector<uint32_t> expanded_plist_indices;+ to_expanded_plists(raw_posting_lists, plists, expanded_plist_indices);++ posting_list_t::merge(plists, result_ids);++ for(uint32_t expanded_plist_index: expanded_plist_indices) {+ delete plists[expanded_plist_index];+ }+}+void posting_t::intersect(const std::vector<void*>& raw_posting_lists, std::vector<uint32_t>& result_ids) {// we will have to convert the compact posting list (if any) to full formstd::vector<posting_list_t*> plists;std::vector<uint32_t> expanded_plist_indices;+ to_expanded_plists(raw_posting_lists, plists, expanded_plist_indices);++ posting_list_t::intersect(plists, result_ids);+ for(uint32_t expanded_plist_index: expanded_plist_indices) {+ delete plists[expanded_plist_index];+ }+}++void posting_t::to_expanded_plists(const std::vector<void*>& raw_posting_lists, std::vector<posting_list_t*>& plists,+ std::vector<uint32_t>& expanded_plist_indices) {for(size_t i = 0; i < raw_posting_lists.size(); i++) {auto raw_posting_list = raw_posting_lists[i];@@ -310,10 +333,4 @@ void posting_t::intersect(const std::vector<void*>& raw_posting_lists, std::vectplists.emplace_back(full_posting_list);}}-- posting_list_t::intersect(plists, result_ids);-- for(uint32_t expanded_plist_index: expanded_plist_indices) {- delete plists[expanded_plist_index];- }}diff --git a/src/posting_list.cpp b/src/posting_list.cppindex b245845e..3755867b 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -530,6 +530,65 @@ posting_list_t::block_t* posting_list_t::block_of(last_id_t id) {return nullptr;}++void posting_list_t::merge(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids) {+ auto its = std::vector<posting_list_t::iterator_t>();+ its.reserve(posting_lists.size());++ size_t sum_sizes = 0;++ for(const auto& posting_list: posting_lists) {+ its.push_back(posting_list->new_iterator());+ sum_sizes += posting_list->num_ids();+ }++ result_ids.reserve(sum_sizes);+ size_t num_lists = its.size();++ switch (num_lists) {+ case 2:+ while(!at_end2(its)) {+ if(equals2(its)) {+ //LOG(INFO) << its[0].id();+ result_ids.push_back(its[0].id());+ advance_all2(its);+ } else {+ uint32_t smallest_value = advance_smallest2(its);+ result_ids.push_back(smallest_value);+ }+ }++ while(its[0].valid()) {+ result_ids.push_back(its[0].id());+ its[0].next();+ }++ while(its[1].valid()) {+ result_ids.push_back(its[1].id());+ its[1].next();+ }++ break;+ default:+ while(!at_end(its)) {+ if(equals(its)) {+ result_ids.push_back(its[0].id());+ advance_all(its);+ } else {+ uint32_t smallest_value = advance_smallest(its);+ result_ids.push_back(smallest_value);+ }+ }++ for(auto& it: its) {+ while(it.valid()) {+ result_ids.push_back(it.id());+ it.next();+ }+ }+ }+}+// Inspired by: https://stackoverflow.com/a/25509185/131050void posting_list_t::intersect(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids) {auto its = std::vector<posting_list_t::iterator_t>();@@ -549,7 +608,7 @@ void posting_list_t::intersect(const std::vector<posting_list_t*>& posting_listsresult_ids.push_back(its[0].id());advance_all2(its);} else {- advance_least2(its);+ advance_non_largest2(its);}}break;@@ -560,7 +619,7 @@ void posting_list_t::intersect(const std::vector<posting_list_t*>& posting_listsresult_ids.push_back(its[0].id());advance_all(its);} else {- advance_least(its);+ advance_non_largest(its);}}}@@ -604,7 +663,7 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& postingadvance_all2(its);} else {- advance_least2(its);+ advance_non_largest2(its);}if(iter_state.ids.size() == batch_size) {@@ -631,7 +690,7 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& postingadvance_all(its);} else {- advance_least(its);+ advance_non_largest(its);}if(iter_state.ids.size() == batch_size) {@@ -770,7 +829,7 @@ void posting_list_t::advance_all2(std::vector<posting_list_t::iterator_t>& its)its[1].next();}-void posting_list_t::advance_least(std::vector<posting_list_t::iterator_t>& its) {+void posting_list_t::advance_non_largest(std::vector<posting_list_t::iterator_t>& its) {// we will find the iter with greatest value and then advance the rest until their value catches upuint32_t greatest_value = 0;@@ -787,7 +846,7 @@ void posting_list_t::advance_least(std::vector<posting_list_t::iterator_t>& its)}}-void posting_list_t::advance_least2(std::vector<posting_list_t::iterator_t>& its) {+void posting_list_t::advance_non_largest2(std::vector<posting_list_t::iterator_t>& its) {if(its[0].id() > its[1].id()) {its[1].skip_to(its[0].id());} else {@@ -795,6 +854,39 @@ void posting_list_t::advance_least2(std::vector<posting_list_t::iterator_t>& its}}+uint32_t posting_list_t::advance_smallest(std::vector<posting_list_t::iterator_t>& its) {+ // we will advance the iterator(s) with the smallest value and then return that value+ uint32_t smallest_value = UINT32_MAX;++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() < smallest_value) {+ smallest_value = its[i].id();+ }+ }++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() == smallest_value) {+ its[i].next();+ }+ }++ return smallest_value;+}++uint32_t posting_list_t::advance_smallest2(std::vector<posting_list_t::iterator_t>& its) {+ uint32_t smallest_value = 0;++ if(its[0].id() < its[1].id()) {+ smallest_value = its[0].id();+ its[0].next();+ } else {+ smallest_value = its[1].id();+ its[1].next();+ }++ return smallest_value;+}+size_t posting_list_t::num_ids() {return ids_length;}diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 85be7e4c..4f0fca8f 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -516,6 +516,50 @@ TEST(PostingListTest, RandomInsertAndDeletes) {ASSERT_LT(pl.num_blocks(), 1000);}+TEST(PostingListTest, MergeBasics) {+ std::vector<uint32_t> offsets = {0, 1, 3};+ std::vector<posting_list_t*> lists;++ // [0, 2] [3, 20]+ // [1, 3], [5, 10], [20]+ // [2, 3], [5, 7], [20]++ posting_list_t p1(2);+ p1.upsert(0, offsets);+ p1.upsert(2, offsets);+ p1.upsert(3, offsets);+ p1.upsert(20, offsets);++ posting_list_t p2(2);+ p2.upsert(1, offsets);+ p2.upsert(3, offsets);+ p2.upsert(5, offsets);+ p2.upsert(10, offsets);+ p2.upsert(20, offsets);++ posting_list_t p3(2);+ p3.upsert(2, offsets);+ p3.upsert(3, offsets);+ p3.upsert(5, offsets);+ p3.upsert(7, offsets);+ p3.upsert(20, offsets);++ lists.push_back(&p1);+ lists.push_back(&p2);+ lists.push_back(&p3);++ std::vector<uint32_t> result_ids;++ posting_list_t::merge(lists, result_ids);++ std::vector<uint32_t> expected_ids = {0, 1, 2, 3, 5, 7, 10, 20};+ ASSERT_EQ(expected_ids.size(), result_ids.size());++ for(size_t i = 0; i < expected_ids.size(); i++) {+ ASSERT_EQ(expected_ids[i], result_ids[i]);+ }+}+TEST(PostingListTest, IntersectionBasics) {std::vector<uint32_t> offsets = {0, 1, 3};std::vector<posting_list_t*> lists;@@ -956,7 +1000,7 @@ TEST(PostingListTest, DISABLED_Benchmark) {LOG(INFO) << "Time taken for 5 sorted array updates: " << timeMicros;}-TEST(PostingListTest, BenchmarkIntersection) {+TEST(PostingListTest, DISABLED_BenchmarkIntersection) {std::vector<uint32_t> offsets = {0, 1, 3};time_t t;