Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `posting_t::get_or_iterator`.
Harpreet Sangarc7b07499ae98
diff --git a/include/posting.h b/include/posting.hindex 6b9e6882..772ddba1 100644--- a/include/posting.h+++ b/include/posting.h@@ -2,6 +2,7 @@#include <cstdint>#include <vector>+#include <or_iterator.h>#include "posting_list.h"#include "threadpool.h"@@ -111,6 +112,9 @@ public:static void get_matching_array_indices(const std::vector<void*>& raw_posting_lists,uint32_t id, std::vector<size_t>& indices);++ static void get_or_iterator(void*& raw_posting_lists, std::vector<or_iterator_t>& or_iterators,+ std::vector<posting_list_t*>& expanded_plists);};template<class T>diff --git a/src/art.cpp b/src/art.cppindex e88f8afc..befdd194 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1001,24 +1001,6 @@ bool validate_and_add_leaf(art_leaf* leaf, const bool last_token, const std::strreturn true;}-void leaf_values_iterator(art_leaf*& leaf, std::vector<or_iterator_t>& or_iterators,- std::vector<posting_list_t*>& expanded_plists) {- if(IS_COMPACT_POSTING(leaf->values)) {- auto compact_posting_list = COMPACT_POSTING_PTR(leaf->values);- posting_list_t* full_posting_list = compact_posting_list->to_full_posting_list();- expanded_plists.emplace_back(full_posting_list);-- std::vector<posting_list_t::iterator_t> its;- its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));- or_iterators.emplace_back(or_iterator_t(its));- } else {- posting_list_t* full_posting_list = (posting_list_t*)(leaf->values);- std::vector<posting_list_t::iterator_t> its;- its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));- or_iterators.emplace_back(or_iterator_t(its));- }-}-bool validate_and_add_leaf(art_leaf* leaf,const std::string& prev_token, const art_leaf* prev_leaf,const art_leaf* exact_leaf,@@ -1042,8 +1024,8 @@ bool validate_and_add_leaf(art_leaf* leaf,std::vector<or_iterator_t> or_iterators;std::vector<posting_list_t*> expanded_plists;- leaf_values_iterator(const_cast<art_leaf*&>(prev_leaf), or_iterators, expanded_plists);- leaf_values_iterator(leaf, or_iterators, expanded_plists);+ posting_t::get_or_iterator(const_cast<art_leaf*&>(prev_leaf)->values, or_iterators, expanded_plists);+ posting_t::get_or_iterator(leaf->values, or_iterators, expanded_plists);auto found = or_iterator_t::contains_atleast_one(or_iterators,result_iter_state_t(nullptr, 0, filter_result_iterator));diff --git a/src/posting.cpp b/src/posting.cppindex 05b5b061..efb47a13 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -523,6 +523,24 @@ void posting_t::get_matching_array_indices(const std::vector<void*>& raw_posting}}+void posting_t::get_or_iterator(void*& raw_posting_list, std::vector<or_iterator_t>& or_iterators,+ std::vector<posting_list_t*>& expanded_plists) {+ if(IS_COMPACT_POSTING(raw_posting_list)) {+ auto compact_posting_list = COMPACT_POSTING_PTR(raw_posting_list);+ posting_list_t* full_posting_list = compact_posting_list->to_full_posting_list();+ expanded_plists.emplace_back(full_posting_list);++ std::vector<posting_list_t::iterator_t> its;+ its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));+ or_iterators.emplace_back(or_iterator_t(its));+ } else {+ posting_list_t* full_posting_list = (posting_list_t*)(raw_posting_list);+ std::vector<posting_list_t::iterator_t> its;+ its.push_back(full_posting_list->new_iterator(nullptr, nullptr, 0));+ or_iterators.emplace_back(or_iterator_t(its));+ }+}+void posting_t::get_phrase_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,const uint32_t* ids, uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {diff --git a/test/or_iterator_test.cpp b/test/or_iterator_test.cppindex be3dac1e..7b4f396b 100644--- a/test/or_iterator_test.cpp+++ b/test/or_iterator_test.cpp@@ -1,6 +1,8 @@#include <gtest/gtest.h>#include <or_iterator.h>#include <posting_list.h>+#include <posting.h>+#include <filter_result_iterator.h>#include "logger.h"TEST(OrIteratorTest, IntersectTwoListsWith3SubLists) {@@ -256,3 +258,69 @@ TEST(OrIteratorTest, IntersectAndFilterTwoIts) {delete p1;delete p2;}++TEST(OrIteratorTest, ContainsAtLeastOne) {+ std::vector<uint32_t> ids = {1, 3, 5};++ std::vector<or_iterator_t> or_iterators;+ std::vector<posting_list_t*> expanded_plists;++ posting_list_t p_list1(2);+ for (const auto &id: ids) {+ p_list1.upsert(id, {1, 2, 3});+ }+ void* raw_pointer = &p_list1;++ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ ASSERT_EQ(1, or_iterators.size());++ posting_list_t p_list2(2);+ ids = {2, 4};+ for (const auto &id: ids) {+ p_list2.upsert(id, {1, 2, 3});+ }+ raw_pointer = &p_list2;++ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ ASSERT_EQ(2, or_iterators.size());++ auto found = or_iterator_t::contains_atleast_one(or_iterators,+ result_iter_state_t(nullptr, 0, nullptr));+ ASSERT_FALSE(found);++ or_iterators.clear();++ posting_list_t p_list3(2);+ ids = {1, 2, 4, 5};+ for (const auto &id: ids) {+ p_list3.upsert(id, {1, 2, 3});+ }++ raw_pointer = &p_list1;+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ raw_pointer = &p_list3;+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ ASSERT_EQ(2, or_iterators.size());++ found = or_iterator_t::contains_atleast_one(or_iterators,+ result_iter_state_t(nullptr, 0, nullptr));+ ASSERT_TRUE(found);+ ASSERT_EQ(1, or_iterators.front().id()); // Match found on id 1++ or_iterators.clear();++ raw_pointer = &p_list1;+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ raw_pointer = &p_list3;+ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ ASSERT_EQ(2, or_iterators.size());++ auto filter_ids = new uint32_t[1]{5};+ auto filter_iterator = new filter_result_iterator_t(filter_ids, 1);+ found = or_iterator_t::contains_atleast_one(or_iterators,+ result_iter_state_t(nullptr, 0, filter_iterator));+ ASSERT_TRUE(found);+ ASSERT_EQ(5, or_iterators.front().id()); // Match found on id 5++ delete filter_iterator;+}diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 16e510aa..fb95e281 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -1540,3 +1540,51 @@ TEST_F(PostingListTest, DISABLED_BenchmarkIntersection) {LOG(INFO) << "Sorted array result len: " << abc_len;LOG(INFO) << "Time taken for sorted array intersection: " << timeMicros;}++TEST_F(PostingListTest, GetOrIterator) {+ std::vector<uint32_t> ids = {1, 3, 5};+ std::vector<uint32_t> offset_index = {0, 3, 6};+ std::vector<uint32_t> offsets = {0, 3, 4, 0, 3, 4, 0, 3, 4};+ compact_posting_list_t* c_list = compact_posting_list_t::create(3, &ids[0], &offset_index[0], 9, &offsets[0]);+ void* raw_pointer = SET_COMPACT_POSTING(c_list);++ std::vector<or_iterator_t> or_iterators;+ std::vector<posting_list_t*> expanded_plists;++ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ ASSERT_EQ(1, or_iterators.size());+ ASSERT_EQ(1, expanded_plists.size());++ for (const auto &id: ids) {+ ASSERT_TRUE(or_iterators.front().valid());+ ASSERT_EQ(id, or_iterators.front().id());+ or_iterators.front().next();+ }+ ASSERT_FALSE(or_iterators.front().valid());++ free(c_list);+ or_iterators.clear();+ for (auto& item: expanded_plists) {+ delete item;+ }+ expanded_plists.clear();++ posting_list_t p_list(2);+ for (const auto &id: ids) {+ p_list.upsert(id, {1, 2, 3});+ }+ raw_pointer = &p_list;++ posting_t::get_or_iterator(raw_pointer, or_iterators, expanded_plists);+ ASSERT_EQ(1, or_iterators.size());+ ASSERT_TRUE(expanded_plists.empty());++ for (const auto &id: ids) {+ ASSERT_TRUE(or_iterators.front().valid());+ ASSERT_EQ(id, or_iterators.front().id());+ or_iterators.front().next();+ }+ ASSERT_FALSE(or_iterators.front().valid());++ or_iterators.clear();+}