Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Block based posting list intersection.

Kishore Nallan0e9c021a1e2e
diff --git a/include/posting_list.h b/include/posting_list.h
index a05acdec..b7c321d1 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -39,6 +39,22 @@ public:
}
};
+ class iterator_t {
+ private:
+ block_t* block;
+ uint32_t index;
+ uint32_t* ids;
+ public:
+ explicit iterator_t(block_t* root);
+ iterator_t(iterator_t&& rhs) noexcept;
+ ~iterator_t();
+ [[nodiscard]] bool valid() const;
+ void next();
+ void skip_to(uint32_t id);
+ [[nodiscard]] uint32_t id() const;
+ void offsets(std::vector<uint32_t>& offsets);
+ };
+
private:
// when a block reaches pre-allocated storage, it is expanded by this factor
@@ -58,6 +74,18 @@ private:
static void merge_adjacent_blocks(block_t* block1, block_t* block2, size_t num_block2_ids);
+ static bool at_end(const std::vector<posting_list_t::iterator_t>& its);
+ static bool at_end2(const std::vector<posting_list_t::iterator_t>& its);
+
+ static bool equals(const std::vector<posting_list_t::iterator_t>& its);
+ static bool equals2(const std::vector<posting_list_t::iterator_t>& its);
+
+ static void advance_all(std::vector<posting_list_t::iterator_t>& its);
+ static void advance_all2(std::vector<posting_list_t::iterator_t>& its);
+
+ static void advance_least(std::vector<posting_list_t::iterator_t>& its);
+ static void advance_least2(std::vector<posting_list_t::iterator_t>& its);
+
public:
posting_list_t() = delete;
@@ -75,4 +103,9 @@ public:
size_t size();
block_t* block_of(last_id_t id);
-};
\ No newline at end of file
+
+ iterator_t new_iterator();
+
+ static posting_list_t* intersect(const std::vector<posting_list_t*>& posting_lists,
+ std::vector<uint32_t>& result_ids);
+};
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 740386ac..c6425b95 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -412,3 +412,180 @@ posting_list_t::block_t* posting_list_t::block_of(last_id_t id) {
}
return nullptr;
}
+
+// Inspired by: https://stackoverflow.com/a/25509185/131050
+posting_list_t* posting_list_t::intersect(const std::vector<posting_list_t*>& posting_lists,
+ std::vector<uint32_t>& result_ids) {
+ auto its = std::vector<posting_list_t::iterator_t>();
+ its.reserve(posting_lists.size());
+
+ for(const auto& posting_list: posting_lists) {
+ its.push_back(posting_list->new_iterator());
+ }
+
+ size_t num_lists = its.size();
+
+ switch (num_lists) {
+ case 2:
+ while(!at_end2(its)) {
+ if(equals2(its)) {
+ //LOG(INFO) << its[0].id();
+ result_ids.push_back(its[0].id());
+ advance_all2(its);
+ } else {
+ advance_least2(its);
+ }
+ }
+ break;
+ default:
+ while(!at_end(its)) {
+ if(equals(its)) {
+ //LOG(INFO) << its[0].id();
+ result_ids.push_back(its[0].id());
+ advance_all(its);
+ } else {
+ advance_least(its);
+ }
+ }
+ }
+
+ return nullptr;
+}
+
+bool posting_list_t::at_end(const std::vector<posting_list_t::iterator_t>& its) {
+ // if any one iterator is at end, we can stop
+ for(const auto& it : its) {
+ if(!it.valid()) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool posting_list_t::at_end2(const std::vector<posting_list_t::iterator_t>& its) {
+ // if any one iterator is at end, we can stop
+ return !its[0].valid() || !its[1].valid();
+}
+
+bool posting_list_t::equals(const std::vector<posting_list_t::iterator_t>& its) {
+ for(size_t i = 0; i < its.size() - 1; i++) {
+ if(its[i].id() != its[i+1].id()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool posting_list_t::equals2(const std::vector<posting_list_t::iterator_t>& its) {
+ return its[0].id() == its[1].id();
+}
+
+posting_list_t::iterator_t posting_list_t::new_iterator() {
+ return posting_list_t::iterator_t(&root_block);
+}
+
+void posting_list_t::advance_all(std::vector<posting_list_t::iterator_t>& its) {
+ for(auto& it: its) {
+ it.next();
+ }
+}
+
+void posting_list_t::advance_all2(std::vector<posting_list_t::iterator_t>& its) {
+ its[0].next();
+ its[1].next();
+}
+
+void posting_list_t::advance_least(std::vector<posting_list_t::iterator_t>& its) {
+ // we will find the iter with greatest value and then advance the rest until their value catches up
+ uint32_t greatest_value = 0;
+
+ for(size_t i = 0; i < its.size(); i++) {
+ if(its[i].id() > greatest_value) {
+ greatest_value = its[i].id();
+ }
+ }
+
+ for(size_t i = 0; i < its.size(); i++) {
+ if(its[i].id() != greatest_value) {
+ its[i].skip_to(greatest_value);
+ }
+ }
+}
+
+void posting_list_t::advance_least2(std::vector<posting_list_t::iterator_t>& its) {
+ if(its[0].id() > its[1].id()) {
+ its[1].skip_to(its[0].id());
+ } else {
+ its[0].skip_to(its[1].id());
+ }
+}
+
+/* iterator_t operations */
+
+posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root): block(root), index(0) {
+ ids = root->ids.uncompress();
+}
+
+bool posting_list_t::iterator_t::valid() const {
+ return (block != nullptr) && (index < block->size());
+}
+
+void posting_list_t::iterator_t::next() {
+ index++;
+ if(index == block->size()) {
+ index = 0;
+ block = block->next;
+ delete [] ids;
+ ids = nullptr;
+ if(block != nullptr) {
+ ids = block->ids.uncompress();
+ }
+ }
+}
+
+uint32_t posting_list_t::iterator_t::id() const {
+ //return block->ids.at(index);
+ return ids[index];
+}
+
+void posting_list_t::iterator_t::offsets(std::vector<uint32_t>& offsets) {
+ // TODO
+}
+
+void posting_list_t::iterator_t::skip_to(uint32_t id) {
+ bool skipped_block = false;
+ while(block != nullptr && block->ids.last() < id) {
+ block = block->next;
+ skipped_block = true;
+ }
+
+ if(skipped_block) {
+ index = 0;
+ delete [] ids;
+ ids = nullptr;
+
+ if(block != nullptr) {
+ ids = block->ids.uncompress();
+ }
+ }
+
+ while(block != nullptr && index < block->size() && ids[index] < id) {
+ index++;
+ }
+}
+
+posting_list_t::iterator_t::~iterator_t() {
+ delete [] ids;
+ ids = nullptr;
+}
+
+posting_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {
+ block = rhs.block;
+ index = rhs.index;
+ ids = rhs.ids;
+
+ rhs.block = nullptr;
+ rhs.ids = nullptr;
+}
diff --git a/src/sorted_array.cpp b/src/sorted_array.cpp
index 80b757a6..ccab32c9 100644
--- a/src/sorted_array.cpp
+++ b/src/sorted_array.cpp
@@ -399,9 +399,5 @@ void sorted_array::binary_count_indices(const uint32_t *values, int low_vindex,
}
uint32_t sorted_array::last() {
- if(getLength() == 0) {
- return UINT32_MAX;
- }
-
- return at(getLength()-1);
+ return (length == 0) ? UINT32_MAX : max;
}
diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cpp
index 0eea3a8f..00e20b42 100644
--- a/test/posting_list_test.cpp
+++ b/test/posting_list_test.cpp
@@ -1,5 +1,7 @@
#include <gtest/gtest.h>
#include "posting_list.h"
+#include "array_utils.h"
+#include <chrono>
#include <vector>
TEST(PostingListTest, Insert) {
@@ -316,16 +318,250 @@ TEST(PostingListTest, RandomInsertAndDeletes) {
std::vector<uint32_t> offsets1 = {0, 1, 3};
std::vector<uint32_t> offsets2 = {10, 12};
+ // generate unique random IDs
+ std::set<uint32_t> ids;
+
for(size_t i = 0; i < 100000; i++) {
- const std::vector<uint32_t>& offsets = (i % 2 == 0) ? offsets1 : offsets2;
- pl.upsert(rand() % 100000, offsets);
+ ids.insert(rand() % 100000);
+ }
+
+ size_t index = 0;
+
+ for(auto id: ids) {
+ const std::vector<uint32_t>& offsets = (index % 2 == 0) ? offsets1 : offsets2;
+ pl.upsert(id, offsets);
+ index++;
}
for(size_t i = 0; i < 10000; i++) {
- const std::vector<uint32_t>& offsets = (i % 2 == 0) ? offsets1 : offsets2;
pl.erase(rand() % 100000);
}
- bool size_within_range = (pl.size() < 1500) && (pl.size() > 1000);
- ASSERT_TRUE(size_within_range);
+ ASSERT_LT(pl.size(), 750);
+ ASSERT_GT(pl.size(), 500);
+}
+
+TEST(PostingListTest, IntersectionBasics) {
+ std::vector<uint32_t> offsets = {0, 1, 3};
+ std::vector<posting_list_t*> lists;
+
+ // [0, 2] [3, 20]
+ // [1, 3], [5, 10], [20]
+ // [2, 3], [5, 7], [20]
+
+ posting_list_t p1(2);
+ p1.upsert(0, offsets);
+ p1.upsert(2, offsets);
+ p1.upsert(3, offsets);
+ p1.upsert(20, offsets);
+
+ posting_list_t p2(2);
+ p2.upsert(1, offsets);
+ p2.upsert(3, offsets);
+ p2.upsert(5, offsets);
+ p2.upsert(10, offsets);
+ p2.upsert(20, offsets);
+
+ posting_list_t p3(2);
+ p3.upsert(2, offsets);
+ p3.upsert(3, offsets);
+ p3.upsert(5, offsets);
+ p3.upsert(7, offsets);
+ p3.upsert(20, offsets);
+
+ lists.push_back(&p1);
+ lists.push_back(&p2);
+ lists.push_back(&p3);
+
+ std::vector<uint32_t> result_ids;
+
+ posting_list_t::intersect(lists, result_ids);
+
+ ASSERT_EQ(2, result_ids.size());
+ ASSERT_EQ(3, result_ids[0]);
+ ASSERT_EQ(20, result_ids[1]);
+}
+
+TEST(PostingListTest, IntersectionSkipBlocks) {
+ std::vector<uint32_t> offsets = {0, 1, 3};
+ std::vector<posting_list_t*> lists;
+
+ std::vector<uint32_t> p1_ids = {9, 11};
+ std::vector<uint32_t> p2_ids = {1, 2, 3, 4, 5, 6, 7, 8, 9, 11};
+ std::vector<uint32_t> p3_ids = {2, 3, 8, 9, 11, 20};
+
+ // [9, 11]
+ // [1, 2], [3, 4], [5, 6], [7, 8], [9, 11]
+ // [2, 3], [8, 9], [11, 20]
+
+ posting_list_t p1(2);
+ posting_list_t p2(2);
+ posting_list_t p3(2);
+
+ for(auto id: p1_ids) {
+ p1.upsert(id, offsets);
+ }
+
+ for(auto id: p2_ids) {
+ p2.upsert(id, offsets);
+ }
+
+ for(auto id: p3_ids) {
+ p3.upsert(id, offsets);
+ }
+
+ lists.push_back(&p1);
+ lists.push_back(&p2);
+ lists.push_back(&p3);
+
+ std::vector<uint32_t> result_ids;
+ posting_list_t::intersect(lists, result_ids);
+
+ uint32_t* p1_p2_intersected;
+ uint32_t* final_results;
+
+ size_t temp_len = ArrayUtils::and_scalar(&p1_ids[0], p1_ids.size(), &p2_ids[0], p2_ids.size(), &p1_p2_intersected);
+ size_t final_len = ArrayUtils::and_scalar(&p3_ids[0], p3_ids.size(), p1_p2_intersected, temp_len, &final_results);
+
+ ASSERT_EQ(final_len, result_ids.size());
+
+ for(size_t i = 0; i < result_ids.size(); i++) {
+ ASSERT_EQ(final_results[i], result_ids[i]);
+ }
+
+ delete [] p1_p2_intersected;
+ delete [] final_results;
+}
+
+TEST(PostingListTest, DISABLED_Benchmark) {
+ std::vector<uint32_t> offsets = {0, 1, 3};
+ posting_list_t pl(4096);
+ sorted_array arr;
+
+ for(size_t i = 0; i < 500000; i++) {
+ pl.upsert(i, offsets);
+ arr.append(i);
+ }
+
+ auto begin = std::chrono::high_resolution_clock::now();
+
+ for(size_t i = 250000; i < 250005; i++) {
+ pl.upsert(i, offsets);
+ }
+
+ long long int timeMicros =
+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+
+ LOG(INFO) << "Time taken for 5 posting list updates: " << timeMicros;
+
+ begin = std::chrono::high_resolution_clock::now();
+
+ for(size_t i = 250000; i < 250005; i++) {
+ arr.remove_value(i);
+ arr.append(i);
+ }
+
+ timeMicros =
+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+
+ LOG(INFO) << "Time taken for 5 sorted array updates: " << timeMicros;
+}
+
+TEST(PostingListTest, DISABLED_BenchmarkIntersection) {
+ std::vector<uint32_t> offsets = {0, 1, 3};
+
+ time_t t;
+ srand((unsigned) time(&t));
+
+ std::set<uint32_t> rand_ids1;
+ std::set<uint32_t> rand_ids2;
+ std::set<uint32_t> rand_ids3;
+
+ const size_t list1_size = 100000;
+ const size_t list2_size = 50000;
+ const size_t list3_size = 25000;
+ const size_t num_range = 1000000;
+
+ /*const size_t list1_size = 10;
+ const size_t list2_size = 10;
+ const size_t num_range = 50;*/
+
+ for(size_t i = 0; i < list1_size; i++) {
+ rand_ids1.insert(rand() % num_range);
+ }
+
+ for(size_t i = 0; i < list2_size; i++) {
+ rand_ids2.insert(rand() % num_range);
+ }
+
+ for(size_t i = 0; i < list3_size; i++) {
+ rand_ids3.insert(rand() % num_range);
+ }
+
+ posting_list_t pl1(1024);
+ posting_list_t pl2(1024);
+ posting_list_t pl3(1024);
+ sorted_array arr1;
+ sorted_array arr2;
+ sorted_array arr3;
+
+ std::string id1_str = "";
+ std::string id2_str = "";
+ std::string id3_str = "";
+
+ for(auto id: rand_ids1) {
+ //id1_str += std::to_string(id) + " ";
+ pl1.upsert(id, offsets);
+ arr1.append(id);
+ }
+
+ for(auto id: rand_ids2) {
+ //id2_str += std::to_string(id) + " ";
+ pl2.upsert(id, offsets);
+ arr2.append(id);
+ }
+
+ for(auto id: rand_ids3) {
+ //id2_str += std::to_string(id) + " ";
+ pl3.upsert(id, offsets);
+ arr3.append(id);
+ }
+
+ //LOG(INFO) << "id1_str: " << id1_str;
+ //LOG(INFO) << "id2_str: " << id2_str;
+
+ std::vector<uint32_t> result_ids;
+ auto begin = std::chrono::high_resolution_clock::now();
+
+ posting_list_t::intersect({&pl1, &pl2, &pl3}, result_ids);
+
+ long long int timeMicros =
+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+
+ LOG(INFO) << "Posting list result len: " << result_ids.size();
+ LOG(INFO) << "Time taken for posting list intersection: " << timeMicros;
+
+ begin = std::chrono::high_resolution_clock::now();
+
+ auto a = arr1.uncompress();
+ auto b = arr2.uncompress();
+ auto c = arr3.uncompress();
+
+ uint32_t* ab;
+ size_t ab_len = ArrayUtils::and_scalar(a, arr1.getLength(), b, arr2.getLength(), &ab);
+
+ uint32_t* abc;
+ size_t abc_len = ArrayUtils::and_scalar(ab, ab_len, c, arr3.getLength(), &abc);
+
+ delete [] a;
+ delete [] b;
+ delete [] c;
+ delete [] ab;
+ delete [] abc;
+
+ timeMicros =
+ std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+
+ LOG(INFO) << "Sorted array result len: " << abc_len;
+ LOG(INFO) << "Time taken for sorted array intersection: " << timeMicros;
}