Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Helper for fetching array token positions.

Kishore Nallan9f581c0ef93e
diff --git a/include/posting.h b/include/posting.h
index 63db6c1a..d9ae5e6d 100644
--- a/include/posting.h
+++ b/include/posting.h
@@ -95,4 +95,10 @@ public:
static void merge(const std::vector<void*>& posting_lists, std::vector<uint32_t>& result_ids);
static void intersect(const std::vector<void*>& posting_lists, std::vector<uint32_t>& result_ids);
+
+ static void get_array_token_positions(
+ uint32_t id,
+ const std::vector<void*>& posting_lists,
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec
+ );
};
\ No newline at end of file
diff --git a/include/posting_list.h b/include/posting_list.h
index 0f389472..64345302 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -121,7 +121,7 @@ public:
uint32_t first_id();
- block_t* block_of(last_id_t id);
+ block_t* block_of(uint32_t id);
bool contains(uint32_t id);
diff --git a/src/posting.cpp b/src/posting.cpp
index 8b25dc2e..7acef8bf 100644
--- a/src/posting.cpp
+++ b/src/posting.cpp
@@ -416,3 +416,45 @@ void posting_t::destroy_list(void*& obj) {
obj = nullptr;
}
+
+void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>& raw_posting_lists,
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec) {
+
+ std::vector<posting_list_t*> plists;
+ std::vector<uint32_t> expanded_plist_indices;
+ to_expanded_plists(raw_posting_lists, plists, expanded_plist_indices);
+
+ posting_list_t::result_iter_state_t iter_state;
+ iter_state.ids.push_back(id);
+
+ iter_state.indices.emplace_back();
+ iter_state.blocks.emplace_back();
+
+ std::vector<posting_list_t::block_t*>& block_vec = iter_state.blocks.back();
+ std::vector<uint32_t>& index_vec = iter_state.indices.back();
+
+ for(posting_list_t* pl: plists) {
+ posting_list_t::block_t* block = pl->block_of(id);
+ block_vec.push_back(block);
+
+ bool found_index = false;
+
+ if(block != nullptr) {
+ uint32_t index = block->ids.indexOf(id);
+ if(index != block->ids.getLength()) {
+ index_vec.push_back(index);
+ found_index = true;
+ }
+ }
+
+ if(!found_index) {
+ index_vec.push_back(UINT32_MAX);
+ }
+ }
+
+ posting_list_t::get_offsets(iter_state, array_token_positions_vec);
+
+ for(uint32_t expanded_plist_index: expanded_plist_indices) {
+ delete plists[expanded_plist_index];
+ }
+}
\ No newline at end of file
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 29d6ef73..2f1dbe71 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -506,12 +506,13 @@ uint32_t posting_list_t::first_id() {
return root_block.ids.at(0);
}
-posting_list_t::block_t* posting_list_t::block_of(last_id_t id) {
- auto it = id_block_map.find(id);
- if(it != id_block_map.end()) {
- return it->second;
+posting_list_t::block_t* posting_list_t::block_of(uint32_t id) {
+ const auto it = id_block_map.lower_bound(id);
+ if(it == id_block_map.end()) {
+ return nullptr;
}
- return nullptr;
+
+ return it->second;
}
@@ -707,6 +708,10 @@ bool posting_list_t::get_offsets(posting_list_t::result_iter_state_t& iter_state
block_t* curr_block = iter_state.blocks[i][j];
uint32_t curr_index = iter_state.indices[i][j];
+ if(curr_block == nullptr || curr_index == UINT32_MAX) {
+ continue;
+ }
+
uint32_t* offsets = curr_block->offsets.uncompress();
uint32_t start_offset = curr_block->offset_index.at(curr_index);