Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add `or_filter_result`.

Harpreet Sangar08d41114cc87
diff --git a/include/field.h b/include/field.h
index b7865a29..0af1db89 100644
--- a/include/field.h
+++ b/include/field.h
@@ -625,6 +625,17 @@ struct reference_filter_result_t {
uint32_t count = 0;
uint32_t* docs = nullptr;
+ reference_filter_result_t& operator=(const reference_filter_result_t& obj) noexcept {
+ if (&obj == this)
+ return *this;
+
+ count = obj.count;
+ docs = new uint32_t[count];
+ memcpy(docs, obj.docs, count * sizeof(uint32_t));
+
+ return *this;
+ }
+
~reference_filter_result_t() {
delete[] docs;
}
@@ -636,10 +647,29 @@ struct filter_result_t {
// Collection name -> Reference filter result
std::map<std::string, reference_filter_result_t*> reference_filter_results;
- filter_result_t() {}
+ filter_result_t() = default;
filter_result_t(uint32_t count, uint32_t* docs) : count(count), docs(docs) {}
+ filter_result_t& operator=(const filter_result_t& obj) noexcept {
+ if (&obj == this)
+ return *this;
+
+ count = obj.count;
+ docs = new uint32_t[count];
+ memcpy(docs, obj.docs, count * sizeof(uint32_t));
+
+ for (const auto &item: obj.reference_filter_results) {
+ reference_filter_results[item.first] = new reference_filter_result_t[count];
+
+ for (uint32_t i = 0; i < count; i++) {
+ reference_filter_results[item.first][i] = item.second[i];
+ }
+ }
+
+ return *this;
+ }
+
filter_result_t& operator=(filter_result_t&& obj) noexcept {
if (&obj == this)
return *this;
diff --git a/src/field.cpp b/src/field.cpp
index 9b20aeef..0a0380b5 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -1025,16 +1025,10 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_
*out = *A;
for (auto const& item: a.reference_filter_results) {
- auto& reference = result.reference_filter_results[item.first][out - result.docs];
- reference.count = item.second[A - a.docs].count;
- reference.docs = new uint32_t[reference.count];
- memcpy(reference.docs, item.second[A - a.docs].docs, reference.count * sizeof(uint32_t));
+ result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];
}
for (auto const& item: b.reference_filter_results) {
- auto& reference = result.reference_filter_results[item.first][out - result.docs];
- reference.count = item.second[B - b.docs].count;
- reference.docs = new uint32_t[reference.count];
- memcpy(reference.docs, item.second[B - b.docs].docs, reference.count * sizeof(uint32_t));
+ result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];
}
out++;
@@ -1048,3 +1042,104 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_
}
}
}
+
+void filter_result_t::or_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
+ if (a.count == 0 && b.count == 0) {
+ return;
+ }
+
+ if (a.count == 0) {
+ result = b;
+ return;
+ }
+
+ if (b.count == 0) {
+ result = a;
+ return;
+ }
+
+ size_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;
+ result.docs = new uint32_t[lenA + lenB];
+
+ for (auto const& item: a.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
+ }
+ }
+ for (auto const& item: b.reference_filter_results) {
+ if (result.reference_filter_results.count(item.first) == 0) {
+ result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
+ }
+ }
+
+ while (indexA < lenA && indexB < lenB) {
+ if (a.docs[indexA] < b.docs[indexB]) {
+ // check for duplicate
+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
+ result.docs[res_index] = a.docs[indexA];
+ res_index++;
+ }
+
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ }
+
+ indexA++;
+ } else {
+ if (res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
+ result.docs[res_index] = b.docs[indexB];
+ res_index++;
+ }
+
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ }
+
+ indexB++;
+ }
+ }
+
+ while (indexA < lenA) {
+ if (res_index == 0 || result.docs[res_index - 1] != a.docs[indexA]) {
+ result.docs[res_index] = a.docs[indexA];
+ res_index++;
+ }
+
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
+ }
+
+ indexA++;
+ }
+
+ while (indexB < lenB) {
+ if(res_index == 0 || result.docs[res_index - 1] != b.docs[indexB]) {
+ result.docs[res_index] = b.docs[indexB];
+ res_index++;
+ }
+
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][res_index - 1] = item.second[indexB];
+ }
+
+ indexB++;
+ }
+
+ result.count = res_index;
+
+ // shrink fit
+ auto out = new uint32_t[res_index];
+ memcpy(out, result.docs, res_index * sizeof(uint32_t));
+ delete[] result.docs;
+ result.docs = out;
+
+ for (auto &item: result.reference_filter_results) {
+ auto out_references = new reference_filter_result_t[res_index];
+
+ for (uint32_t i = 0; i < result.count; i++) {
+ out_references[i] = item.second[i];
+ }
+ delete[] item.second;
+ item.second = out_references;
+ }
+}
diff --git a/src/index.cpp b/src/index.cpp
index c81e4881..f20efe47 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2095,29 +2095,6 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,
return Option(true);
}
-void copy_reference_ids(filter_result_t& from, filter_result_t& to) {
- if (to.count > 0 && !from.reference_filter_results.empty()) {
- for (const auto &item: from.reference_filter_results) {
- auto& from_reference_result = from.reference_filter_results[item.first];
- auto& to_reference_result = to.reference_filter_results[item.first];
- to_reference_result = new reference_filter_result_t[to.count];
-
- size_t to_index = 0, from_index = 0;
- while (to_index < to.count && from_index < from.count) {
- if (to.docs[to_index] == from.docs[from_index]) {
- to_reference_result[to_index] = from_reference_result[from_index];
- to_index++;
- from_index++;
- } else if (to.docs[to_index] < from.docs[from_index]) {
- to_index++;
- } else {
- from_index++;
- }
- }
- }
- }
-}
-
Option<bool> Index::recursive_filter(filter_node_t* const root,
filter_result_t& result,
const std::string& collection_name,
@@ -2147,15 +2124,7 @@ Option<bool> Index::recursive_filter(filter_node_t* const root,
if (root->filter_operator == AND) {
filter_result_t::and_filter_results(l_result, r_result, result);
} else {
- uint32_t* filtered_results = nullptr;
- result.count = ArrayUtils::or_scalar(
- l_result.docs, l_result.count, r_result.docs,
- r_result.count, &filtered_results);
-
- result.docs = filtered_results;
- if (!l_result.reference_filter_results.empty() || !r_result.reference_filter_results.empty()) {
- copy_reference_ids(!l_result.reference_filter_results.empty() ? l_result : r_result, result);
- }
+ filter_result_t::or_filter_results(l_result, r_result, result);
}
return Option(true);
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 23c4b022..d3d2ce23 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -634,6 +634,178 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {
}
}
+TEST_F(CollectionJoinTest, OrFilterResults_NoReference) {
+ filter_result_t a, b;
+ a.count = 0;
+ uint32_t limit = 10;
+ a.docs = new uint32_t[limit];
+ for (size_t i = 2; i < limit; i++) {
+ if (i % 3 == 0) {
+ a.docs[a.count++] = i;
+ }
+ }
+
+ // a.docs: [3, 6, 9], b.docs: []
+ filter_result_t result1;
+ filter_result_t::or_filter_results(a, b, result1);
+ ASSERT_EQ(3, result1.count);
+ ASSERT_EQ(0, result1.reference_filter_results.size());
+
+ std::vector<uint32_t> expected = {3, 6, 9};
+ for (size_t i = 0; i < result1.count; i++) {
+ ASSERT_EQ(expected[i], result1.docs[i]);
+ }
+
+ b.count = 9;
+ b.docs = new uint32_t[b.count];
+ for (size_t i = 0; i < b.count; i++) {
+ b.docs[i] = i;
+ }
+
+ // a.docs: [3, 6, 9], b.docs: [0..8]
+ filter_result_t result2;
+ filter_result_t::or_filter_results(a, b, result2);
+ ASSERT_EQ(10, result2.count);
+ ASSERT_EQ(0, result2.reference_filter_results.size());
+
+ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+ for (size_t i = 0; i < result2.count; i++) {
+ ASSERT_EQ(expected[i], result2.docs[i]);
+ }
+
+
+ filter_result_t c, result3;
+
+ std::vector<uint32_t> vec = {0, 4, 5};
+ c.docs = new uint32_t[vec.size()];
+ auto j = 0;
+ for(auto i: vec) {
+ a.docs[j++] = i;
+ }
+
+ // b.docs: [0..8], c.docs: [0, 4, 5]
+ filter_result_t::or_filter_results(b, c, result3);
+ ASSERT_EQ(9, result3.count);
+ ASSERT_EQ(0, result3.reference_filter_results.size());
+
+ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};
+ for(size_t i = 0; i < result3.count; i++) {
+ ASSERT_EQ(expected[i], result3.docs[i]);
+ }
+}
+
+TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
+ filter_result_t a, b;
+ uint32_t limit = 10;
+
+ a.count = 0;
+ a.docs = new uint32_t[limit];
+ a.reference_filter_results["foo"] = new reference_filter_result_t[limit];
+ for (size_t i = 2; i < limit; i++) {
+ if (i % 3 == 0) {
+ a.docs[a.count] = i;
+
+ auto& reference = a.reference_filter_results["foo"][a.count++];
+ reference.count = 1;
+ reference.docs = new uint32_t[1];
+ reference.docs[0] = 2 * i;
+ }
+ }
+
+ // a.docs: [3, 6, 9], b.docs: []
+ filter_result_t result1;
+ filter_result_t::or_filter_results(a, b, result1);
+
+ ASSERT_EQ(3, result1.count);
+ ASSERT_EQ(1, result1.reference_filter_results.size());
+ ASSERT_EQ(1, result1.reference_filter_results.count("foo"));
+
+ std::vector<uint32_t> expected = {3, 6, 9}, foo_reference = {6, 12, 18};
+ for (size_t i = 0; i < result1.count; i++) {
+ ASSERT_EQ(expected[i], result1.docs[i]);
+
+ ASSERT_EQ(1, result1.reference_filter_results["foo"][i].count);
+ ASSERT_EQ(foo_reference[i], result1.reference_filter_results["foo"][i].docs[0]);
+ }
+
+ b.count = 9;
+ b.docs = new uint32_t[b.count];
+ b.reference_filter_results["bar"] = new reference_filter_result_t[b.count];
+ for (size_t i = 0; i < b.count; i++) {
+ b.docs[i] = i;
+
+ auto& reference = b.reference_filter_results["bar"][i];
+ reference.count = 1;
+ reference.docs = new uint32_t[1];
+ reference.docs[0] = 10 - i;
+ }
+
+ // a.docs: [3, 6, 9], b.docs: [0..8]
+ filter_result_t result2;
+ filter_result_t::or_filter_results(a, b, result2);
+ ASSERT_EQ(10, result2.count);
+
+ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+ std::map<uint32_t, uint32_t> foo_map = {{3, 6}, {6, 12}, {9, 18}}, bar_map = {{0, 10}, {1, 9}, {2, 8}, {3, 7},
+ {4, 6}, {5, 5}, {6, 4}, {7, 3}, {8, 2}};
+ for (size_t i = 0; i < result2.count; i++) {
+ ASSERT_EQ(expected[i], result2.docs[i]);
+
+ if (foo_map.count(i) != 0) {
+ ASSERT_EQ(1, result2.reference_filter_results["foo"][i].count);
+ ASSERT_EQ(foo_map[i], result2.reference_filter_results["foo"][i].docs[0]);
+ } else {
+ ASSERT_EQ(0, result2.reference_filter_results["foo"][i].count);
+ }
+
+ if (bar_map.count(i) != 0) {
+ ASSERT_EQ(1, result2.reference_filter_results["bar"][i].count);
+ ASSERT_EQ(bar_map[i], result2.reference_filter_results["bar"][i].docs[0]);
+ } else {
+ ASSERT_EQ(0, result2.reference_filter_results["bar"][i].count);
+ }
+ }
+
+ filter_result_t c, result3;
+
+ std::map<uint32_t, uint32_t> baz_map = {{0, 2}, {4, 0}, {5, 8}};
+ c.count = baz_map.size();
+ c.docs = new uint32_t[baz_map.size()];
+ c.reference_filter_results["baz"] = new reference_filter_result_t[baz_map.size()];
+ auto j = 0;
+ for(auto i: baz_map) {
+ c.docs[j] = i.first;
+
+ auto& reference = c.reference_filter_results["baz"][j++];
+ reference.count = 1;
+ reference.docs = new uint32_t[1];
+ reference.docs[0] = i.second;
+ }
+
+ // b.docs: [0..8], c.docs: [0, 4, 5]
+ filter_result_t::or_filter_results(b, c, result3);
+ ASSERT_EQ(9, result3.count);
+
+ expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};
+ for (size_t i = 0; i < result3.count; i++) {
+ ASSERT_EQ(expected[i], result3.docs[i]);
+
+ if (bar_map.count(i) != 0) {
+ ASSERT_EQ(1, result3.reference_filter_results["bar"][i].count);
+ ASSERT_EQ(bar_map[i], result3.reference_filter_results["bar"][i].docs[0]);
+ } else {
+ ASSERT_EQ(0, result3.reference_filter_results["bar"][i].count);
+ }
+
+ if (baz_map.count(i) != 0) {
+ ASSERT_EQ(1, result3.reference_filter_results["baz"][i].count);
+ ASSERT_EQ(baz_map[i], result3.reference_filter_results["baz"][i].docs[0]);
+ } else {
+ ASSERT_EQ(0, result3.reference_filter_results["baz"][i].count);
+ }
+ }
+}
+
TEST_F(CollectionJoinTest, FilterByNReferences) {
auto schema_json =
R"({