Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add `and_filter_result` function.

Harpreet Sangar391d693ffa45
diff --git a/src/index.cpp b/src/index.cpp
index bb5ca2c4..03bc00ab 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2095,6 +2095,66 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,
return Option(true);
}
+void and_filter_result(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
+ auto lenA = a.count, lenB = b.count;
+ if (lenA == 0 || lenB == 0) {
+ return;
+ }
+
+ result.docs = new uint32_t[std::min(lenA, lenB)];
+
+ auto A = a.docs, B = b.docs, out = result.docs;
+ const uint32_t *endA = A + lenA;
+ const uint32_t *endB = B + lenB;
+
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first];
+ }
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first];
+ }
+ for (auto& item: result.reference_filter_results) {
+ item.second = new reference_filter_result_t[std::min(lenA, lenB)];
+ }
+
+ while (true) {
+ while (*A < *B) {
+ SKIP_FIRST_COMPARE:
+ if (++A == endA) {
+ result.count = out - result.docs;
+ return;
+ }
+ }
+ while (*A > *B) {
+ if (++B == endB) {
+ result.count = out - result.docs;
+ return;
+ }
+ }
+ if (*A == *B) {
+ *out = *A;
+
+ for (auto const& item: a.reference_filter_results) {
+ result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];
+ item.second[A - a.docs].docs = nullptr;
+ }
+ for (auto const& item: b.reference_filter_results) {
+ result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];
+ item.second[B - b.docs].docs = nullptr;
+ }
+
+ out++;
+
+ if (++A == endA || ++B == endB) {
+ result.count = out - result.docs;
+ return;
+ }
+ } else {
+ goto SKIP_FIRST_COMPARE;
+ }
+ }
+}
+
void copy_reference_ids(filter_result_t& from, filter_result_t& to) {
if (to.count > 0 && !from.reference_filter_results.empty()) {
for (const auto &item: from.reference_filter_results) {
@@ -2144,22 +2204,21 @@ Option<bool> Index::recursive_filter(filter_node_t* const root,
}
}
- uint32_t* filtered_results = nullptr;
if (root->filter_operator == AND) {
- result.count = ArrayUtils::and_scalar(
- l_result.docs, l_result.count, r_result.docs,
- r_result.count, &filtered_results);
+ and_filter_result(l_result, r_result, result);
} else {
+ uint32_t* filtered_results = nullptr;
result.count = ArrayUtils::or_scalar(
l_result.docs, l_result.count, r_result.docs,
r_result.count, &filtered_results);
- }
- result.docs = filtered_results;
- if (!l_result.reference_filter_results.empty() || !r_result.reference_filter_results.empty()) {
- copy_reference_ids(!l_result.reference_filter_results.empty() ? l_result : r_result, result);
+ result.docs = filtered_results;
+ if (!l_result.reference_filter_results.empty() || !r_result.reference_filter_results.empty()) {
+ copy_reference_ids(!l_result.reference_filter_results.empty() ? l_result : r_result, result);
+ }
}
+
return Option(true);
}
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index b25439e6..6df9d397 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -551,6 +551,214 @@ TEST_F(CollectionJoinTest, FilterByReference_MultipleMatch) {
collectionManager.drop_collection("Links");
}
+TEST_F(CollectionJoinTest, FilterByNReferences) {
+ auto schema_json =
+ R"({
+ "name": "Users",
+ "fields": [
+ {"name": "user_id", "type": "string"},
+ {"name": "user_name", "type": "string"}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "user_id": "user_a",
+ "user_name": "Roshan"
+ })"_json,
+ R"({
+ "user_id": "user_b",
+ "user_name": "Ruby"
+ })"_json,
+ R"({
+ "user_id": "user_c",
+ "user_name": "Joe"
+ })"_json,
+ R"({
+ "user_id": "user_d",
+ "user_name": "Aby"
+ })"_json
+ };
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Repos",
+ "fields": [
+ {"name": "repo_id", "type": "string"},
+ {"name": "repo_content", "type": "string"},
+ {"name": "repo_stars", "type": "int32"},
+ {"name": "repo_is_private", "type": "bool"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "repo_content": "body1",
+ "repo_stars": 431,
+ "repo_is_private": true
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "repo_content": "body2",
+ "repo_stars": 4562,
+ "repo_is_private": false
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "repo_content": "body3",
+ "repo_stars": 945,
+ "repo_is_private": false
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Links",
+ "fields": [
+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id"},
+ {"name": "user_id", "type": "string", "reference": "Users.user_id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_d"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_d"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Organizations",
+ "fields": [
+ {"name": "org_id", "type": "string"},
+ {"name": "org_name", "type": "string"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "org_id": "org_a",
+ "org_name": "Typesense"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Participants",
+ "fields": [
+ {"name": "user_id", "type": "string", "reference": "Users.user_id"},
+ {"name": "org_id", "type": "string", "reference": "Organizations.org_id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "user_id": "user_a",
+ "org_id": "org_a"
+ })"_json,
+ R"({
+ "user_id": "user_b",
+ "org_id": "org_a"
+ })"_json,
+ R"({
+ "user_id": "user_d",
+ "org_id": "org_a"
+ })"_json,
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ auto coll = collectionManager.get_collection_unsafe("Users");
+
+ // Search for users within an organization with access to a particular repo.
+ auto result = coll->search("R", {"user_name"}, "$Participants(org_id:=org_a) && $Links(repo_id:=repo_b)", {}, {}, {0},
+ 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD).get();
+
+ ASSERT_EQ(2, result["found"].get<size_t>());
+ ASSERT_EQ(2, result["hits"].size());
+ ASSERT_EQ("user_b", result["hits"][0]["document"]["user_id"].get<std::string>());
+ ASSERT_EQ("user_a", result["hits"][1]["document"]["user_id"].get<std::string>());
+
+ collectionManager.drop_collection("Users");
+ collectionManager.drop_collection("Repos");
+ collectionManager.drop_collection("Links");
+}
+
TEST_F(CollectionJoinTest, IncludeFieldsByReference_SingleMatch) {
auto schema_json =
R"({