Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `or_filter_result`.
Harpreet Sangar08d41114cc87
diff --git a/include/field.h b/include/field.hindex b7865a29..0af1db89 100644--- a/include/field.h+++ b/include/field.h@@ -625,6 +625,17 @@ struct reference_filter_result_t {uint32_t count = 0;uint32_t* docs = nullptr;+ reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {+ if (&obj == this)+ return *this;++ count = obj.count;+ docs = new uint32_t[count];+ memcpy(docs, obj.docs, count * sizeof(uint32_t));++ return *this;+ }+~reference_filter_result_t() {delete[] docs;}@@ -636,10 +647,29 @@ struct filter_result_t {// Collection name -> Reference filter resultstd::map<std::string, reference_filter_result_t*> reference_filter_results;- filter_result_t() {}+ filter_result_t() = default;filter_result_t(uint32_t count, uint32_t* docs) : count(count), docs(docs) {}+ filter_result_t& operator=(const filter_result_t& obj) noexcept {+ if (&obj == this)+ return *this;++ count = obj.count;+ docs = new uint32_t[count];+ memcpy(docs, obj.docs, count * sizeof(uint32_t));++ for (const auto &item: obj.reference_filter_results) {+ reference_filter_results[item.first] = new reference_filter_result_t[count];++ for (uint32_t i = 0; i < count; i++) {+ reference_filter_results[item.first][i] = item.second[i];+ }+ }++ return *this;+ }+filter_result_t& operator=(filter_result_t&& obj) noexcept {if (&obj == this)return *this;diff --git a/src/field.cpp b/src/field.cppindex 9b20aeef..0a0380b5 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -1025,16 +1025,10 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_*out = *A;for (auto const& item: a.reference_filter_results) {- auto& reference = result.reference_filter_results[item.first][out - result.docs];- reference.count = item.second[A - a.docs].count;- reference.docs = new uint32_t[reference.count];- memcpy(reference.docs, item.second[A - a.docs].docs, reference.count * sizeof(uint32_t));+ result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];}for (auto const& item: b.reference_filter_results) {- auto& reference = result.reference_filter_results[item.first][out - result.docs];- reference.count = item.second[B - b.docs].count;- reference.docs = new uint32_t[reference.count];- memcpy(reference.docs, item.second[B - b.docs].docs, reference.count * sizeof(uint32_t));+ result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];}out++;@@ -1048,3 +1042,104 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_}}}++void filter_result_t::or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {+ if (a.count == 0 && b.count == 0) {+ return;+ }++ if (a.count == 0) {+ result = b;+ return;+ }++ if (b.count == 0) {+ result = a;+ return;+ }++ size_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;+ result.docs = new uint32_t[lenA + lenB];++ for (auto const& item: a.reference_filter_results) {+ if (result.reference_filter_results.count(item.first) == 0) {+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];+ }+ }+ for (auto const& item: b.reference_filter_results) {+ if (result.reference_filter_results.count(item.first) == 0) {+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];+ }+ }++ while (indexA < lenA && indexB < lenB) {+ if (a.docs[indexA] < b.docs[indexB]) {+ // check for duplicate+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {+ result.docs[res_index] = a.docs[indexA];+ res_index++;+ }++ for (auto const& item: a.reference_filter_results) {+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];+ }++ indexA++;+ } else {+ if (res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {+ result.docs[res_index] = b.docs[indexB];+ res_index++;+ }++ for (auto const& item: b.reference_filter_results) {+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];+ }++ indexB++;+ }+ }++ while (indexA < lenA) {+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {+ result.docs[res_index] = a.docs[indexA];+ res_index++;+ }++ for (auto const& item: a.reference_filter_results) {+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];+ }++ indexA++;+ }++ while (indexB < lenB) {+ if(res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {+ result.docs[res_index] = b.docs[indexB];+ res_index++;+ }++ for (auto const& item: b.reference_filter_results) {+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];+ }++ indexB++;+ }++ result.count = res_index;++ // shrink fit+ auto out = new uint32_t[res_index];+ memcpy(out, result.docs, res_index * sizeof(uint32_t));+ delete[] result.docs;+ result.docs = out;++ for (auto &item: result.reference_filter_results) {+ auto out_references = new reference_filter_result_t[res_index];++ for (uint32_t i = 0; i < result.count; i++) {+ out_references[i] = item.second[i];+ }+ delete[] item.second;+ item.second = out_references;+ }+}diff --git a/src/index.cpp b/src/index.cppindex c81e4881..f20efe47 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2095,29 +2095,6 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,return Option(true);}-void copy_reference_ids(filter_result_t& from, filter_result_t& to) {- if (to.count > 0 && !from.reference_filter_results.empty()) {- for (const auto &item: from.reference_filter_results) {- auto& from_reference_result = from.reference_filter_results[item.first];- auto& to_reference_result = to.reference_filter_results[item.first];- to_reference_result = new reference_filter_result_t[to.count];-- size_t to_index = 0, from_index = 0;- while (to_index < to.count && from_index < from.count) {- if (to.docs[to_index] == from.docs[from_index]) {- to_reference_result[to_index] = from_reference_result[from_index];- to_index++;- from_index++;- } else if (to.docs[to_index] < from.docs[from_index]) {- to_index++;- } else {- from_index++;- }- }- }- }-}-Option<bool> Index::recursive_filter(filter_node_t* const root,filter_result_t& result,const std::string& collection_name,@@ -2147,15 +2124,7 @@ Option<bool> Index::recursive_filter(filter_node_t* const root,if (root->filter_operator == AND) {filter_result_t::and_filter_results(l_result, r_result, result);} else {- uint32_t* filtered_results = nullptr;- result.count = ArrayUtils::or_scalar(- l_result.docs, l_result.count, r_result.docs,- r_result.count, &filtered_results);-- result.docs = filtered_results;- if (!l_result.reference_filter_results.empty() || !r_result.reference_filter_results.empty()) {- copy_reference_ids(!l_result.reference_filter_results.empty() ? l_result : r_result, result);- }+ filter_result_t::or_filter_results(l_result, r_result, result);}return Option(true);diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex 23c4b022..d3d2ce23 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -634,6 +634,178 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {}}+TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {+ filter_result_t a, b;+ a.count = 0;+ uint32_t limit = 10;+ a.docs = new uint32_t[limit];+ for (size_t i = 2; i < limit; i++) {+ if (i % 3 == 0) {+ a.docs[a.count++] = i;+ }+ }++ // a.docs: [3, 6, 9], b.docs: []+ filter_result_t result1;+ filter_result_t::or_filter_results(a, b, result1);+ ASSERT_EQ(3, result1.count);+ ASSERT_EQ(0, result1.reference_filter_results.size());++ std::vector<uint32_t> expected = {3, 6, 9};+ for (size_t i = 0; i < result1.count; i++) {+ ASSERT_EQ(expected[i], result1.docs[i]);+ }++ b.count = 9;+ b.docs = new uint32_t[b.count];+ for (size_t i = 0; i < b.count; i++) {+ b.docs[i] = i;+ }++ // a.docs: [3, 6, 9], b.docs: [0..8]+ filter_result_t result2;+ filter_result_t::or_filter_results(a, b, result2);+ ASSERT_EQ(10, result2.count);+ ASSERT_EQ(0, result2.reference_filter_results.size());++ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};+ for (size_t i = 0; i < result2.count; i++) {+ ASSERT_EQ(expected[i], result2.docs[i]);+ }+++ filter_result_t c, result3;++ std::vector<uint32_t> vec = {0, 4, 5};+ c.docs = new uint32_t[vec.size()];+ auto j = 0;+ for(auto i: vec) {+ a.docs[j++] = i;+ }++ // b.docs: [0..8], c.docs: [0, 4, 5]+ filter_result_t::or_filter_results(b, c, result3);+ ASSERT_EQ(9, result3.count);+ ASSERT_EQ(0, result3.reference_filter_results.size());++ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};+ for(size_t i = 0; i < result3.count; i++) {+ ASSERT_EQ(expected[i], result3.docs[i]);+ }+}++TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {+ filter_result_t a, b;+ uint32_t limit = 10;++ a.count = 0;+ a.docs = new uint32_t[limit];+ a.reference_filter_results["foo"] = new reference_filter_result_t[limit];+ for (size_t i = 2; i < limit; i++) {+ if (i % 3 == 0) {+ a.docs[a.count] = i;++ auto& reference = a.reference_filter_results["foo"][a.count++];+ reference.count = 1;+ reference.docs = new uint32_t[1];+ reference.docs[0] = 2 * i;+ }+ }++ // a.docs: [3, 6, 9], b.docs: []+ filter_result_t result1;+ filter_result_t::or_filter_results(a, b, result1);++ ASSERT_EQ(3, result1.count);+ ASSERT_EQ(1, result1.reference_filter_results.size());+ ASSERT_EQ(1, result1.reference_filter_results.count("foo"));++ std::vector<uint32_t> expected = {3, 6, 9}, foo_reference = {6, 12, 18};+ for (size_t i = 0; i < result1.count; i++) {+ ASSERT_EQ(expected[i], result1.docs[i]);++ ASSERT_EQ(1, result1.reference_filter_results["foo"][i].count);+ ASSERT_EQ(foo_reference[i], result1.reference_filter_results["foo"][i].docs[0]);+ }++ b.count = 9;+ b.docs = new uint32_t[b.count];+ b.reference_filter_results["bar"] = new reference_filter_result_t[b.count];+ for (size_t i = 0; i < b.count; i++) {+ b.docs[i] = i;++ auto& reference = b.reference_filter_results["bar"][i];+ reference.count = 1;+ reference.docs = new uint32_t[1];+ reference.docs[0] = 10 - i;+ }++ // a.docs: [3, 6, 9], b.docs: [0..8]+ filter_result_t result2;+ filter_result_t::or_filter_results(a, b, result2);+ ASSERT_EQ(10, result2.count);++ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};+ std::map<uint32_t, uint32_t> foo_map = {{3, 6}, {6, 12}, {9, 18}}, bar_map = {{0, 10}, {1, 9}, {2, 8}, {3, 7},+ {4, 6}, {5, 5}, {6, 4}, {7, 3}, {8, 2}};+ for (size_t i = 0; i < result2.count; i++) {+ ASSERT_EQ(expected[i], result2.docs[i]);++ if (foo_map.count(i) != 0) {+ ASSERT_EQ(1, result2.reference_filter_results["foo"][i].count);+ ASSERT_EQ(foo_map[i], result2.reference_filter_results["foo"][i].docs[0]);+ } else {+ ASSERT_EQ(0, result2.reference_filter_results["foo"][i].count);+ }++ if (bar_map.count(i) != 0) {+ ASSERT_EQ(1, result2.reference_filter_results["bar"][i].count);+ ASSERT_EQ(bar_map[i], result2.reference_filter_results["bar"][i].docs[0]);+ } else {+ ASSERT_EQ(0, result2.reference_filter_results["bar"][i].count);+ }+ }++ filter_result_t c, result3;++ std::map<uint32_t, uint32_t> baz_map = {{0, 2}, {4, 0}, {5, 8}};+ c.count = baz_map.size();+ c.docs = new uint32_t[baz_map.size()];+ c.reference_filter_results["baz"] = new reference_filter_result_t[baz_map.size()];+ auto j = 0;+ for(auto i: baz_map) {+ c.docs[j] = i.first;++ auto& reference = c.reference_filter_results["baz"][j++];+ reference.count = 1;+ reference.docs = new uint32_t[1];+ reference.docs[0] = i.second;+ }++ // b.docs: [0..8], c.docs: [0, 4, 5]+ filter_result_t::or_filter_results(b, c, result3);+ ASSERT_EQ(9, result3.count);++ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};+ for (size_t i = 0; i < result3.count; i++) {+ ASSERT_EQ(expected[i], result3.docs[i]);++ if (bar_map.count(i) != 0) {+ ASSERT_EQ(1, result3.reference_filter_results["bar"][i].count);+ ASSERT_EQ(bar_map[i], result3.reference_filter_results["bar"][i].docs[0]);+ } else {+ ASSERT_EQ(0, result3.reference_filter_results["bar"][i].count);+ }++ if (baz_map.count(i) != 0) {+ ASSERT_EQ(1, result3.reference_filter_results["baz"][i].count);+ ASSERT_EQ(baz_map[i], result3.reference_filter_results["baz"][i].docs[0]);+ } else {+ ASSERT_EQ(0, result3.reference_filter_results["baz"][i].count);+ }+ }+}+TEST_F(CollectionJoinTest, FilterByNReferences) {auto schema_json =R"({