Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Block based posting list intersection.
Kishore Nallan0e9c021a1e2e
diff --git a/include/posting_list.h b/include/posting_list.hindex a05acdec..b7c321d1 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -39,6 +39,22 @@ public:}};+ class iterator_t {+ private:+ block_t* block;+ uint32_t index;+ uint32_t* ids;+ public:+ explicit iterator_t(block_t* root);+ iterator_t(iterator_t&& rhs) noexcept;+ ~iterator_t();+ [[nodiscard]] bool valid() const;+ void next();+ void skip_to(uint32_t id);+ [[nodiscard]] uint32_t id() const;+ void offsets(std::vector<uint32_t>& offsets);+ };+private:// when a block reaches pre-allocated storage, it is expanded by this factor@@ -58,6 +74,18 @@ private:static void merge_adjacent_blocks(block_t* block1, block_t* block2, size_t num_block2_ids);+ static bool at_end(const std::vector<posting_list_t::iterator_t>& its);+ static bool at_end2(const std::vector<posting_list_t::iterator_t>& its);++ static bool equals(const std::vector<posting_list_t::iterator_t>& its);+ static bool equals2(const std::vector<posting_list_t::iterator_t>& its);++ static void advance_all(std::vector<posting_list_t::iterator_t>& its);+ static void advance_all2(std::vector<posting_list_t::iterator_t>& its);++ static void advance_least(std::vector<posting_list_t::iterator_t>& its);+ static void advance_least2(std::vector<posting_list_t::iterator_t>& its);+public:posting_list_t() = delete;@@ -75,4 +103,9 @@ public:size_t size();block_t* block_of(last_id_t id);-};\ No newline at end of file++ iterator_t new_iterator();++ static posting_list_t* intersect(const std::vector<posting_list_t*>& posting_lists,+ std::vector<uint32_t>& result_ids);+};diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 740386ac..c6425b95 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -412,3 +412,180 @@ posting_list_t::block_t* posting_list_t::block_of(last_id_t id) {}return nullptr;}++// Inspired by: https://stackoverflow.com/a/25509185/131050+posting_list_t* posting_list_t::intersect(const std::vector<posting_list_t*>& posting_lists,+ std::vector<uint32_t>& result_ids) {+ auto its = std::vector<posting_list_t::iterator_t>();+ its.reserve(posting_lists.size());++ for(const auto& posting_list: posting_lists) {+ its.push_back(posting_list->new_iterator());+ }++ size_t num_lists = its.size();++ switch (num_lists) {+ case 2:+ while(!at_end2(its)) {+ if(equals2(its)) {+ //LOG(INFO) << its[0].id();+ result_ids.push_back(its[0].id());+ advance_all2(its);+ } else {+ advance_least2(its);+ }+ }+ break;+ default:+ while(!at_end(its)) {+ if(equals(its)) {+ //LOG(INFO) << its[0].id();+ result_ids.push_back(its[0].id());+ advance_all(its);+ } else {+ advance_least(its);+ }+ }+ }++ return nullptr;+}++bool posting_list_t::at_end(const std::vector<posting_list_t::iterator_t>& its) {+ // if any one iterator is at end, we can stop+ for(const auto& it : its) {+ if(!it.valid()) {+ return true;+ }+ }++ return false;+}++bool posting_list_t::at_end2(const std::vector<posting_list_t::iterator_t>& its) {+ // if any one iterator is at end, we can stop+ return !its[0].valid() || !its[1].valid();+}++bool posting_list_t::equals(const std::vector<posting_list_t::iterator_t>& its) {+ for(size_t i = 0; i < its.size() - 1; i++) {+ if(its[i].id() != its[i+1].id()) {+ return false;+ }+ }++ return true;+}++bool posting_list_t::equals2(const std::vector<posting_list_t::iterator_t>& its) {+ return its[0].id() == its[1].id();+}++posting_list_t::iterator_t posting_list_t::new_iterator() {+ return posting_list_t::iterator_t(&root_block);+}++void posting_list_t::advance_all(std::vector<posting_list_t::iterator_t>& its) {+ for(auto& it: its) {+ it.next();+ }+}++void posting_list_t::advance_all2(std::vector<posting_list_t::iterator_t>& its) {+ its[0].next();+ its[1].next();+}++void posting_list_t::advance_least(std::vector<posting_list_t::iterator_t>& its) {+ // we will find the iter with greatest value and then advance the rest until their value catches up+ uint32_t greatest_value = 0;++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() > greatest_value) {+ greatest_value = its[i].id();+ }+ }++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() != greatest_value) {+ its[i].skip_to(greatest_value);+ }+ }+}++void posting_list_t::advance_least2(std::vector<posting_list_t::iterator_t>& its) {+ if(its[0].id() > its[1].id()) {+ its[1].skip_to(its[0].id());+ } else {+ its[0].skip_to(its[1].id());+ }+}++/* iterator_t operations */++posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root): block(root), index(0) {+ ids = root->ids.uncompress();+}++bool posting_list_t::iterator_t::valid() const {+ return (block != nullptr) && (index < block->size());+}++void posting_list_t::iterator_t::next() {+ index++;+ if(index == block->size()) {+ index = 0;+ block = block->next;+ delete [] ids;+ ids = nullptr;+ if(block != nullptr) {+ ids = block->ids.uncompress();+ }+ }+}++uint32_t posting_list_t::iterator_t::id() const {+ //return block->ids.at(index);+ return ids[index];+}++void posting_list_t::iterator_t::offsets(std::vector<uint32_t>& offsets) {+ // TODO+}++void posting_list_t::iterator_t::skip_to(uint32_t id) {+ bool skipped_block = false;+ while(block != nullptr && block->ids.last() < id) {+ block = block->next;+ skipped_block = true;+ }++ if(skipped_block) {+ index = 0;+ delete [] ids;+ ids = nullptr;++ if(block != nullptr) {+ ids = block->ids.uncompress();+ }+ }++ while(block != nullptr && index < block->size() && ids[index] < id) {+ index++;+ }+}++posting_list_t::iterator_t::~iterator_t() {+ delete [] ids;+ ids = nullptr;+}++posting_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {+ block = rhs.block;+ index = rhs.index;+ ids = rhs.ids;++ rhs.block = nullptr;+ rhs.ids = nullptr;+}diff --git a/src/sorted_array.cpp b/src/sorted_array.cppindex 80b757a6..ccab32c9 100644--- a/src/sorted_array.cpp+++ b/src/sorted_array.cpp@@ -399,9 +399,5 @@ void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex,}uint32_t sorted_array::last() {- if(getLength() == 0) {- return UINT32_MAX;- }-- return at(getLength()-1);+ return (length == 0) ? UINT32_MAX : max;}diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 0eea3a8f..00e20b42 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -1,5 +1,7 @@#include <gtest/gtest.h>#include "posting_list.h"+#include "array_utils.h"+#include <chrono>#include <vector>TEST(PostingListTest, Insert) {@@ -316,16 +318,250 @@ TEST(PostingListTest, RandomInsertAndDeletes) {std::vector<uint32_t> offsets1 = {0, 1, 3};std::vector<uint32_t> offsets2 = {10, 12};+ // generate unique random IDs+ std::set<uint32_t> ids;+for(size_t i = 0; i < 100000; i++) {- const std::vector<uint32_t>& offsets = (i % 2 == 0) ? offsets1 : offsets2;- pl.upsert(rand() % 100000, offsets);+ ids.insert(rand() % 100000);+ }++ size_t index = 0;++ for(auto id: ids) {+ const std::vector<uint32_t>& offsets = (index % 2 == 0) ? offsets1 : offsets2;+ pl.upsert(id, offsets);+ index++;}for(size_t i = 0; i < 10000; i++) {- const std::vector<uint32_t>& offsets = (i % 2 == 0) ? offsets1 : offsets2;pl.erase(rand() % 100000);}- bool size_within_range = (pl.size() < 1500) && (pl.size() > 1000);- ASSERT_TRUE(size_within_range);+ ASSERT_LT(pl.size(), 750);+ ASSERT_GT(pl.size(), 500);+}++TEST(PostingListTest, IntersectionBasics) {+ std::vector<uint32_t> offsets = {0, 1, 3};+ std::vector<posting_list_t*> lists;++ // [0, 2] [3, 20]+ // [1, 3], [5, 10], [20]+ // [2, 3], [5, 7], [20]++ posting_list_t p1(2);+ p1.upsert(0, offsets);+ p1.upsert(2, offsets);+ p1.upsert(3, offsets);+ p1.upsert(20, offsets);++ posting_list_t p2(2);+ p2.upsert(1, offsets);+ p2.upsert(3, offsets);+ p2.upsert(5, offsets);+ p2.upsert(10, offsets);+ p2.upsert(20, offsets);++ posting_list_t p3(2);+ p3.upsert(2, offsets);+ p3.upsert(3, offsets);+ p3.upsert(5, offsets);+ p3.upsert(7, offsets);+ p3.upsert(20, offsets);++ lists.push_back(&p1);+ lists.push_back(&p2);+ lists.push_back(&p3);++ std::vector<uint32_t> result_ids;++ posting_list_t::intersect(lists, result_ids);++ ASSERT_EQ(2, result_ids.size());+ ASSERT_EQ(3, result_ids[0]);+ ASSERT_EQ(20, result_ids[1]);+}++TEST(PostingListTest, IntersectionSkipBlocks) {+ std::vector<uint32_t> offsets = {0, 1, 3};+ std::vector<posting_list_t*> lists;++ std::vector<uint32_t> p1_ids = {9, 11};+ std::vector<uint32_t> p2_ids = {1, 2, 3, 4, 5, 6, 7, 8, 9, 11};+ std::vector<uint32_t> p3_ids = {2, 3, 8, 9, 11, 20};++ // [9, 11]+ // [1, 2], [3, 4], [5, 6], [7, 8], [9, 11]+ // [2, 3], [8, 9], [11, 20]++ posting_list_t p1(2);+ posting_list_t p2(2);+ posting_list_t p3(2);++ for(auto id: p1_ids) {+ p1.upsert(id, offsets);+ }++ for(auto id: p2_ids) {+ p2.upsert(id, offsets);+ }++ for(auto id: p3_ids) {+ p3.upsert(id, offsets);+ }++ lists.push_back(&p1);+ lists.push_back(&p2);+ lists.push_back(&p3);++ std::vector<uint32_t> result_ids;+ posting_list_t::intersect(lists, result_ids);++ uint32_t* p1_p2_intersected;+ uint32_t* final_results;++ size_t temp_len = ArrayUtils::and_scalar(&p1_ids[0], p1_ids.size(), &p2_ids[0], p2_ids.size(), &p1_p2_intersected);+ size_t final_len = ArrayUtils::and_scalar(&p3_ids[0], p3_ids.size(), p1_p2_intersected, temp_len, &final_results);++ ASSERT_EQ(final_len, result_ids.size());++ for(size_t i = 0; i < result_ids.size(); i++) {+ ASSERT_EQ(final_results[i], result_ids[i]);+ }++ delete [] p1_p2_intersected;+ delete [] final_results;+}++TEST(PostingListTest, DISABLED_Benchmark) {+ std::vector<uint32_t> offsets = {0, 1, 3};+ posting_list_t pl(4096);+ sorted_array arr;++ for(size_t i = 0; i < 500000; i++) {+ pl.upsert(i, offsets);+ arr.append(i);+ }++ auto begin = std::chrono::high_resolution_clock::now();++ for(size_t i = 250000; i < 250005; i++) {+ pl.upsert(i, offsets);+ }++ long long int timeMicros =+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();++ LOG(INFO) << "Time taken for 5 posting list updates: " << timeMicros;++ begin = std::chrono::high_resolution_clock::now();++ for(size_t i = 250000; i < 250005; i++) {+ arr.remove_value(i);+ arr.append(i);+ }++ timeMicros =+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();++ LOG(INFO) << "Time taken for 5 sorted array updates: " << timeMicros;+}++TEST(PostingListTest, DISABLED_BenchmarkIntersection) {+ std::vector<uint32_t> offsets = {0, 1, 3};++ time_t t;+ srand((unsigned) time(&t));++ std::set<uint32_t> rand_ids1;+ std::set<uint32_t> rand_ids2;+ std::set<uint32_t> rand_ids3;++ const size_t list1_size = 100000;+ const size_t list2_size = 50000;+ const size_t list3_size = 25000;+ const size_t num_range = 1000000;++ /*const size_t list1_size = 10;+ const size_t list2_size = 10;+ const size_t num_range = 50;*/++ for(size_t i = 0; i < list1_size; i++) {+ rand_ids1.insert(rand() % num_range);+ }++ for(size_t i = 0; i < list2_size; i++) {+ rand_ids2.insert(rand() % num_range);+ }++ for(size_t i = 0; i < list3_size; i++) {+ rand_ids3.insert(rand() % num_range);+ }++ posting_list_t pl1(1024);+ posting_list_t pl2(1024);+ posting_list_t pl3(1024);+ sorted_array arr1;+ sorted_array arr2;+ sorted_array arr3;++ std::string id1_str = "";+ std::string id2_str = "";+ std::string id3_str = "";++ for(auto id: rand_ids1) {+ //id1_str += std::to_string(id) + " ";+ pl1.upsert(id, offsets);+ arr1.append(id);+ }++ for(auto id: rand_ids2) {+ //id2_str += std::to_string(id) + " ";+ pl2.upsert(id, offsets);+ arr2.append(id);+ }++ for(auto id: rand_ids3) {+ //id2_str += std::to_string(id) + " ";+ pl3.upsert(id, offsets);+ arr3.append(id);+ }++ //LOG(INFO) << "id1_str: " << id1_str;+ //LOG(INFO) << "id2_str: " << id2_str;++ std::vector<uint32_t> result_ids;+ auto begin = std::chrono::high_resolution_clock::now();++ posting_list_t::intersect({&pl1, &pl2, &pl3}, result_ids);++ long long int timeMicros =+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();++ LOG(INFO) << "Posting list result len: " << result_ids.size();+ LOG(INFO) << "Time taken for posting list intersection: " << timeMicros;++ begin = std::chrono::high_resolution_clock::now();++ auto a = arr1.uncompress();+ auto b = arr2.uncompress();+ auto c = arr3.uncompress();++ uint32_t* ab;+ size_t ab_len = ArrayUtils::and_scalar(a, arr1.getLength(), b, arr2.getLength(), &ab);++ uint32_t* abc;+ size_t abc_len = ArrayUtils::and_scalar(ab, ab_len, c, arr3.getLength(), &abc);++ delete [] a;+ delete [] b;+ delete [] c;+ delete [] ab;+ delete [] abc;++ timeMicros =+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();++ LOG(INFO) << "Sorted array result len: " << abc_len;+ LOG(INFO) << "Time taken for sorted array intersection: " << timeMicros;}