Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Posting list: split up intersection and offset compute.

Kishore Nallan36580dfb6299
diff --git a/include/match_score.h b/include/match_score.h
index 575a8004..a42da5c0 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -1,9 +1,10 @@
#pragma once
-#include <stdint.h>
+#include <cstdint>
#include <vector>
#include <queue>
-#include <stdlib.h>
+#include <algorithm>
+#include <cstdlib>
#include <limits>
#include "logger.h"
diff --git a/include/posting_list.h b/include/posting_list.h
index 3580e7b6..681a382f 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -1,7 +1,10 @@
#pragma once
+
#include <map>
+#include <unordered_map>
#include "sorted_array.h"
#include "array.h"
+#include "match_score.h"
typedef uint32_t last_id_t;
@@ -41,8 +44,8 @@ public:
class iterator_t {
private:
- block_t* block;
- uint32_t index;
+ block_t* curr_block;
+ uint32_t curr_index;
// uncompressed data structures for performance
block_t* uncompressed_block;
@@ -56,7 +59,14 @@ public:
void next();
void skip_to(uint32_t id);
[[nodiscard]] inline uint32_t id();
- void offsets(std::vector<uint32_t>& offsets);
+ [[nodiscard]] inline uint32_t index() const;
+ [[nodiscard]] inline block_t* block() const;
+ };
+
+ struct result_iter_state_t {
+ std::vector<std::vector<block_t*>> blocks;
+ std::vector<std::vector<uint32_t>> indices;
+ std::vector<uint32_t> ids;
};
private:
@@ -113,4 +123,16 @@ public:
iterator_t new_iterator();
static void intersect(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids);
+
+ static bool block_intersect(
+ const std::vector<posting_list_t*>& posting_lists,
+ size_t batch_size,
+ std::vector<posting_list_t::iterator_t>& its,
+ result_iter_state_t& iter_state
+ );
+
+ static bool get_offsets(
+ result_iter_state_t& iter_state,
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions
+ );
};
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 18986bb6..b245845e 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -566,6 +566,165 @@ void posting_list_t::intersect(const std::vector<posting_list_t*>& posting_lists
}
}
+bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting_lists, const size_t batch_size,
+ std::vector<posting_list_t::iterator_t>& its,
+ result_iter_state_t& iter_state) {
+ if(its.empty()) {
+ its.reserve(posting_lists.size());
+
+ for(const auto& posting_list: posting_lists) {
+ its.push_back(posting_list->new_iterator());
+ }
+ } else {
+ // already in the middle of iteration: prepare for next batch
+ iter_state.ids.clear();
+ iter_state.indices.clear();
+ iter_state.blocks.clear();
+ }
+
+ size_t num_lists = its.size();
+
+ switch (num_lists) {
+ case 2:
+ while(!at_end2(its)) {
+ if(equals2(its)) {
+ // still need to ensure that the ID exists in inclusion list but NOT in exclusion list
+ iter_state.ids.push_back(its[0].id());
+
+ std::vector<block_t*> block_vec(2);
+ std::vector<uint32_t> index_vec(its.size());
+ block_vec[0] = its[0].block();
+ block_vec[1] = its[1].block();
+
+ index_vec[0] = its[0].index();
+ index_vec[1] = its[1].index();
+
+ iter_state.blocks.emplace_back(block_vec);
+ iter_state.indices.emplace_back(index_vec);
+
+ advance_all2(its);
+ } else {
+ advance_least2(its);
+ }
+
+ if(iter_state.ids.size() == batch_size) {
+ return true;
+ }
+ }
+ break;
+ default:
+ while(!at_end(its)) {
+ if(equals(its)) {
+ //LOG(INFO) << its[0].id();
+ iter_state.ids.push_back(its[0].id());
+
+ std::vector<block_t*> block_vec(its.size());
+ std::vector<uint32_t> index_vec(its.size());
+
+ for(size_t i = 0; i < its.size(); i++) {
+ block_vec[i] = its[i].block();
+ index_vec[i] = its[i].index();
+ }
+
+ iter_state.blocks.emplace_back(block_vec);
+ iter_state.indices.emplace_back(index_vec);
+
+ advance_all(its);
+ } else {
+ advance_least(its);
+ }
+
+ if(iter_state.ids.size() == batch_size) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+bool posting_list_t::get_offsets(posting_list_t::result_iter_state_t& iter_state,
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec) {
+
+ // Plain string format:
+ // offset1, offset2, ... , 0 (if token is the last offset for the document)
+
+ // Array string format:
+ // offset1, ... , offsetn, offsetn, array_index, 0 (if token is the last offset for the document)
+ // (last offset is repeated to indicate end of offsets for a given array index)
+
+ // For each result ID and for each block it is contained in, calculate offsets
+
+ for(size_t i = 0; i < iter_state.ids.size(); i++) {
+ uint32_t id = iter_state.ids[i];
+ array_token_positions_vec.emplace_back();
+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_tok_pos = array_token_positions_vec.back();
+
+ for(size_t j = 0; j < iter_state.blocks[i].size(); j++) {
+ block_t* curr_block = iter_state.blocks[i][j];
+ uint32_t curr_index = iter_state.indices[i][j];
+
+ uint32_t* offsets = curr_block->offsets.uncompress();
+
+ uint32_t start_offset = curr_block->offset_index.at(curr_index);
+ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?
+ curr_block->offsets.getLength() :
+ curr_block->offset_index.at(curr_index + 1);
+
+ std::vector<uint16_t> positions;
+ int prev_pos = -1;
+ bool is_last_token = false;
+
+ while(start_offset < end_offset) {
+ int pos = offsets[start_offset];
+ start_offset++;
+
+ if(pos == 0) {
+ // indicates that token is the last token on the doc
+ is_last_token = true;
+ start_offset++;
+ continue;
+ }
+
+ if(pos == prev_pos) { // indicates end of array index
+ if(!positions.empty()) {
+ size_t array_index = (size_t) offsets[start_offset];
+ is_last_token = false;
+
+ if(start_offset+1 < end_offset) {
+ size_t next_offset = (size_t) offsets[start_offset + 1];
+ if(next_offset == 0) {
+ // indicates that token is the last token on the doc
+ is_last_token = true;
+ start_offset++;
+ }
+ }
+
+ array_tok_pos[array_index].push_back(token_positions_t{is_last_token, positions});
+ positions.clear();
+ }
+
+ start_offset++; // skip current value which is the array index or flag for last index
+ prev_pos = -1;
+ continue;
+ }
+
+ prev_pos = pos;
+ positions.push_back((uint16_t)pos - 1);
+ }
+
+ if(!positions.empty()) {
+ // for plain string fields
+ array_tok_pos[0].push_back(token_positions_t{is_last_token, positions});
+ }
+
+ delete [] offsets;
+ }
+ }
+
+ return false;
+}
+
bool posting_list_t::at_end(const std::vector<posting_list_t::iterator_t>& its) {
// if any one iterator is at end, we can stop
for(const auto& it : its) {
@@ -643,55 +802,58 @@ size_t posting_list_t::num_ids() {
/* iterator_t operations */
posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root):
- block(root), index(0), uncompressed_block(nullptr), ids(nullptr) {
+ curr_block(root), curr_index(0), uncompressed_block(nullptr), ids(nullptr) {
}
bool posting_list_t::iterator_t::valid() const {
- return (block != nullptr) && (index < block->size());
+ return (curr_block != nullptr) && (curr_index < curr_block->size());
}
void posting_list_t::iterator_t::next() {
- index++;
- if(index == block->size()) {
- index = 0;
- block = block->next;
+ curr_index++;
+ if(curr_index == curr_block->size()) {
+ curr_index = 0;
+ curr_block = curr_block->next;
}
}
uint32_t posting_list_t::iterator_t::id() {
- //return block->ids.at(index);
+ if(uncompressed_block != curr_block) {
+ uncompressed_block = curr_block;
- if(uncompressed_block != block) {
delete [] ids;
ids = nullptr;
- uncompressed_block = block;
- if(block != nullptr) {
- ids = block->ids.uncompress();
+ if(curr_block != nullptr) {
+ ids = curr_block->ids.uncompress();
}
}
- return ids[index];
+ return ids[curr_index];
+}
+
+uint32_t posting_list_t::iterator_t::index() const {
+ return curr_index;
}
-void posting_list_t::iterator_t::offsets(std::vector<uint32_t>& offsets) {
- // TODO
+posting_list_t::block_t* posting_list_t::iterator_t::block() const {
+ return curr_block;
}
void posting_list_t::iterator_t::skip_to(uint32_t id) {
bool skipped_block = false;
- while(block != nullptr && block->ids.last() < id) {
- block = block->next;
+ while(curr_block != nullptr && curr_block->ids.last() < id) {
+ curr_block = curr_block->next;
skipped_block = true;
}
if(skipped_block) {
- index = 0;
+ curr_index = 0;
}
- while(block != nullptr && index < block->size() && this->id() < id) {
- index++;
+ while(curr_block != nullptr && curr_index < curr_block->size() && this->id() < id) {
+ curr_index++;
}
}
@@ -701,11 +863,11 @@ posting_list_t::iterator_t::~iterator_t() {
}
posting_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {
- block = rhs.block;
- index = rhs.index;
+ curr_block = rhs.curr_block;
+ curr_index = rhs.curr_index;
uncompressed_block = rhs.uncompressed_block;
ids = rhs.ids;
- rhs.block = nullptr;
+ rhs.curr_block = nullptr;
rhs.ids = nullptr;
}
diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cpp
index 4e7e8aa9..85be7e4c 100644
--- a/test/posting_list_test.cpp
+++ b/test/posting_list_test.cpp
@@ -557,6 +557,86 @@ TEST(PostingListTest, IntersectionBasics) {
ASSERT_EQ(20, result_ids[1]);
}
+TEST(PostingListTest, ResultsAndOffsetsBasics) {
+ // NOTE: due to the way offsets1 are parsed, the actual positions are 1 less than the offset values stored
+ // (to account for the special offset `0` which indicates last offset
+ std::vector<uint32_t> offsets1 = {1, 2, 4};
+ std::vector<uint32_t> offsets2 = {5, 6};
+ std::vector<uint32_t> offsets3 = {7};
+
+ std::vector<posting_list_t*> lists;
+
+ // T1: [0, 2] [3, 20]
+ // T2: [1, 3], [5, 10], [20]
+ // T3: [2, 3], [5, 7], [20]
+
+ // 3: (0, 1, 3} {4, 5} {6}
+ // 2: {6} {4, 5} {0, 1, 3}
+
+ std::vector<token_positions_t> actual_offsets_3 = {
+ token_positions_t{false, {0, 1, 3}},
+ token_positions_t{false, {4, 5}},
+ token_positions_t{false, {6}},
+ };
+
+ std::vector<token_positions_t> actual_offsets_20 = {
+ token_positions_t{false, {6}},
+ token_positions_t{false, {4, 5}},
+ token_positions_t{false, {0, 1, 3}},
+ };
+
+ posting_list_t p1(2);
+ p1.upsert(0, offsets1);
+ p1.upsert(2, offsets1);
+ p1.upsert(3, offsets1);
+ p1.upsert(20, offsets3);
+
+ posting_list_t p2(2);
+ p2.upsert(1, offsets1);
+ p2.upsert(3, offsets2);
+ p2.upsert(5, offsets1);
+ p2.upsert(10, offsets1);
+ p2.upsert(20, offsets2);
+
+ posting_list_t p3(2);
+ p3.upsert(2, offsets1);
+ p3.upsert(3, offsets3);
+ p3.upsert(5, offsets1);
+ p3.upsert(7, offsets1);
+ p3.upsert(20, offsets1);
+
+ lists.push_back(&p1);
+ lists.push_back(&p2);
+ lists.push_back(&p3);
+
+ std::vector<posting_list_t::iterator_t> its;
+
+ posting_list_t::result_iter_state_t iter_state;
+ posting_list_t::block_intersect(lists, 2, its, iter_state);
+
+ ASSERT_EQ(2, iter_state.ids.size());
+ ASSERT_EQ(3, iter_state.ids[0]);
+ ASSERT_EQ(20, iter_state.ids[1]);
+
+ ASSERT_EQ(2, iter_state.blocks.size());
+ ASSERT_EQ(3, iter_state.blocks[0].size());
+ ASSERT_EQ(3, iter_state.blocks[1].size());
+
+ ASSERT_EQ(2, iter_state.indices.size());
+
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;
+ posting_list_t::get_offsets(iter_state, array_token_positions_vec);
+ ASSERT_EQ(2, array_token_positions_vec.size());
+
+ ASSERT_EQ(actual_offsets_3[0].positions, array_token_positions_vec[0].at(0)[0].positions);
+ ASSERT_EQ(actual_offsets_3[1].positions, array_token_positions_vec[0].at(0)[1].positions);
+ ASSERT_EQ(actual_offsets_3[2].positions, array_token_positions_vec[0].at(0)[2].positions);
+
+ ASSERT_EQ(actual_offsets_20[0].positions, array_token_positions_vec[1].at(0)[0].positions);
+ ASSERT_EQ(actual_offsets_20[1].positions, array_token_positions_vec[1].at(0)[1].positions);
+ ASSERT_EQ(actual_offsets_20[2].positions, array_token_positions_vec[1].at(0)[2].positions);
+}
+
TEST(PostingListTest, IntersectionSkipBlocks) {
std::vector<uint32_t> offsets = {0, 1, 3};
std::vector<posting_list_t*> lists;
@@ -876,7 +956,7 @@ TEST(PostingListTest, DISABLED_Benchmark) {
LOG(INFO) << "Time taken for 5 sorted array updates: " << timeMicros;
}
-TEST(PostingListTest, DISABLED_BenchmarkIntersection) {
+TEST(PostingListTest, BenchmarkIntersection) {
std::vector<uint32_t> offsets = {0, 1, 3};
time_t t;