Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add `posting_t::get_or_iterator`.

Harpreet Sangarc7b07499ae98
diff --git a/include/posting.h b/include/posting.h
index 6b9e6882..772ddba1 100644
--- a/include/posting.h
+++ b/include/posting.h
@@ -2,6 +2,7 @@
#include <cstdint>
#include <vector>
+#include <or_iterator.h>
#include "posting_list.h"
#include "threadpool.h"
@@ -111,6 +112,9 @@ public:
static void get_matching_array_indices(const std::vector<void*>& raw_posting_lists,
uint32_t id, std::vector<size_t>& indices);
+
+ static void get_or_iterator(void*& raw_posting_lists, std::vector<or_iterator_t>& or_iterators,
+ std::vector<posting_list_t*>& expanded_plists);
};
template<class T>
diff --git a/src/art.cpp b/src/art.cpp
index e88f8afc..befdd194 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1001,24 +1001,6 @@ bool validate_and_add_leaf(art_leaf* leaf, const bool last_token, const std::str
return true;
}
-void leaf_values_iterator(art_leaf*& leaf, std::vector<or_iterator_t>& or_iterators,
- std::vector<posting_list_t*>& expanded_plists) {
- if(IS_COMPACT_POSTING(leaf->values)) {
- auto compact_posting_list = COMPACT_POSTING_PTR(leaf->values);
- posting_list_t* full_posting_list = compact_posting_list->to_full_posting_list();
- expanded_plists.emplace_back(full_posting_list);
-
- std::vector<posting_list_t::iterator_t> its;
- its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));
- or_iterators.emplace_back(or_iterator_t(its));
- } else {
- posting_list_t* full_posting_list = (posting_list_t*)(leaf->values);
- std::vector<posting_list_t::iterator_t> its;
- its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));
- or_iterators.emplace_back(or_iterator_t(its));
- }
-}
-
bool validate_and_add_leaf(art_leaf* leaf,
const std::string& prev_token, const art_leaf* prev_leaf,
const art_leaf* exact_leaf,
@@ -1042,8 +1024,8 @@ bool validate_and_add_leaf(art_leaf* leaf,
std::vector<or_iterator_t> or_iterators;
std::vector<posting_list_t*> expanded_plists;
- leaf_values_iterator(const_cast<art_leaf*&>(prev_leaf), or_iterators, expanded_plists);
- leaf_values_iterator(leaf, or_iterators, expanded_plists);
+ posting_t::get_or_iterator(const_cast<art_leaf*&>(prev_leaf)->values, or_iterators, expanded_plists);
+ posting_t::get_or_iterator(leaf->values, or_iterators, expanded_plists);
auto found = or_iterator_t::contains_atleast_one(or_iterators,
result_iter_state_t(nullptr, 0, filter_result_iterator));
diff --git a/src/posting.cpp b/src/posting.cpp
index 05b5b061..efb47a13 100644
--- a/src/posting.cpp
+++ b/src/posting.cpp
@@ -523,6 +523,24 @@ void posting_t::get_matching_array_indices(const std::vector<void*>& raw_posting
}
}
+void posting_t::get_or_iterator(void*& raw_posting_list, std::vector<or_iterator_t>& or_iterators,
+ std::vector<posting_list_t*>& expanded_plists) {
+ if(IS_COMPACT_POSTING(raw_posting_list)) {
+ auto compact_posting_list = COMPACT_POSTING_PTR(raw_posting_list);
+ posting_list_t* full_posting_list = compact_posting_list->to_full_posting_list();
+ expanded_plists.emplace_back(full_posting_list);
+
+ std::vector<posting_list_t::iterator_t> its;
+ its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));
+ or_iterators.emplace_back(or_iterator_t(its));
+ } else {
+ posting_list_t* full_posting_list = (posting_list_t*)(raw_posting_list);
+ std::vector<posting_list_t::iterator_t> its;
+ its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));
+ or_iterators.emplace_back(or_iterator_t(its));
+ }
+}
+
void posting_t::get_phrase_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,
const uint32_t* ids, uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {
diff --git a/test/or_iterator_test.cpp b/test/or_iterator_test.cpp
index be3dac1e..7b4f396b 100644
--- a/test/or_iterator_test.cpp
+++ b/test/or_iterator_test.cpp
@@ -1,6 +1,8 @@
#include <gtest/gtest.h>
#include <or_iterator.h>
#include <posting_list.h>
+#include <posting.h>
+#include <filter_result_iterator.h>
#include "logger.h"
TEST(OrIteratorTest, IntersectTwoListsWith3SubLists) {
@@ -256,3 +258,69 @@ TEST(OrIteratorTest, IntersectAndFilterTwoIts) {
delete p1;
delete p2;
}
+
+TEST(OrIteratorTest, ContainsAtLeastOne) {
+ std::vector<uint32_t> ids = {1, 3, 5};
+
+ std::vector<or_iterator_t> or_iterators;
+ std::vector<posting_list_t*> expanded_plists;
+
+ posting_list_t p_list1(2);
+ for (const auto &id: ids) {
+ p_list1.upsert(id, {1, 2, 3});
+ }
+ void* raw_pointer = &p_list1;
+
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ ASSERT_EQ(1, or_iterators.size());
+
+ posting_list_t p_list2(2);
+ ids = {2, 4};
+ for (const auto &id: ids) {
+ p_list2.upsert(id, {1, 2, 3});
+ }
+ raw_pointer = &p_list2;
+
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ ASSERT_EQ(2, or_iterators.size());
+
+ auto found = or_iterator_t::contains_atleast_one(or_iterators,
+ result_iter_state_t(nullptr, 0, nullptr));
+ ASSERT_FALSE(found);
+
+ or_iterators.clear();
+
+ posting_list_t p_list3(2);
+ ids = {1, 2, 4, 5};
+ for (const auto &id: ids) {
+ p_list3.upsert(id, {1, 2, 3});
+ }
+
+ raw_pointer = &p_list1;
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ raw_pointer = &p_list3;
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ ASSERT_EQ(2, or_iterators.size());
+
+ found = or_iterator_t::contains_atleast_one(or_iterators,
+ result_iter_state_t(nullptr, 0, nullptr));
+ ASSERT_TRUE(found);
+ ASSERT_EQ(1, or_iterators.front().id()); // Match found on id 1
+
+ or_iterators.clear();
+
+ raw_pointer = &p_list1;
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ raw_pointer = &p_list3;
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ ASSERT_EQ(2, or_iterators.size());
+
+ auto filter_ids = new uint32_t[1]{5};
+ auto filter_iterator = new filter_result_iterator_t(filter_ids, 1);
+ found = or_iterator_t::contains_atleast_one(or_iterators,
+ result_iter_state_t(nullptr, 0, filter_iterator));
+ ASSERT_TRUE(found);
+ ASSERT_EQ(5, or_iterators.front().id()); // Match found on id 5
+
+ delete filter_iterator;
+}
diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cpp
index 16e510aa..fb95e281 100644
--- a/test/posting_list_test.cpp
+++ b/test/posting_list_test.cpp
@@ -1540,3 +1540,51 @@ TEST_F(PostingListTest, DISABLED_BenchmarkIntersection) {
LOG(INFO) << "Sorted array result len: " << abc_len;
LOG(INFO) << "Time taken for sorted array intersection: " << timeMicros;
}
+
+TEST_F(PostingListTest, GetOrIterator) {
+ std::vector<uint32_t> ids = {1, 3, 5};
+ std::vector<uint32_t> offset_index = {0, 3, 6};
+ std::vector<uint32_t> offsets = {0, 3, 4, 0, 3, 4, 0, 3, 4};
+ compact_posting_list_t* c_list = compact_posting_list_t::create(3, &ids[0], &offset_index[0], 9, &offsets[0]);
+ void* raw_pointer = SET_COMPACT_POSTING(c_list);
+
+ std::vector<or_iterator_t> or_iterators;
+ std::vector<posting_list_t*> expanded_plists;
+
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ ASSERT_EQ(1, or_iterators.size());
+ ASSERT_EQ(1, expanded_plists.size());
+
+ for (const auto &id: ids) {
+ ASSERT_TRUE(or_iterators.front().valid());
+ ASSERT_EQ(id, or_iterators.front().id());
+ or_iterators.front().next();
+ }
+ ASSERT_FALSE(or_iterators.front().valid());
+
+ free(c_list);
+ or_iterators.clear();
+ for (auto& item: expanded_plists) {
+ delete item;
+ }
+ expanded_plists.clear();
+
+ posting_list_t p_list(2);
+ for (const auto &id: ids) {
+ p_list.upsert(id, {1, 2, 3});
+ }
+ raw_pointer = &p_list;
+
+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);
+ ASSERT_EQ(1, or_iterators.size());
+ ASSERT_TRUE(expanded_plists.empty());
+
+ for (const auto &id: ids) {
+ ASSERT_TRUE(or_iterators.front().valid());
+ ASSERT_EQ(id, or_iterators.front().id());
+ or_iterators.front().next();
+ }
+ ASSERT_FALSE(or_iterators.front().valid());
+
+ or_iterators.clear();
+}