Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `and_filter_result` function.
Harpreet Sangar391d693ffa45
diff --git a/src/index.cpp b/src/index.cppindex bb5ca2c4..03bc00ab 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2095,6 +2095,66 @@ Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,return Option(true);}+void and_filter_result(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {+ auto lenA = a.count, lenB = b.count;+ if (lenA == 0 || lenB == 0) {+ return;+ }++ result.docs = new uint32_t[std::min(lenA, lenB)];++ auto A = a.docs, B = b.docs, out = result.docs;+ const uint32_t *endA = A + lenA;+ const uint32_t *endB = B + lenB;++ for (auto const& item: a.reference_filter_results) {+ result.reference_filter_results[item.first];+ }+ for (auto const& item: b.reference_filter_results) {+ result.reference_filter_results[item.first];+ }+ for (auto& item: result.reference_filter_results) {+ item.second = new reference_filter_result_t[std::min(lenA, lenB)];+ }++ while (true) {+ while (*A < *B) {+ SKIP_FIRST_COMPARE:+ if (++A == endA) {+ result.count = out - result.docs;+ return;+ }+ }+ while (*A > *B) {+ if (++B == endB) {+ result.count = out - result.docs;+ return;+ }+ }+ if (*A == *B) {+ *out = *A;++ for (auto const& item: a.reference_filter_results) {+ result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];+ item.second[A - a.docs].docs = nullptr;+ }+ for (auto const& item: b.reference_filter_results) {+ result.reference_filter_results[item.first][out - result.docs] = item.second[B - b.docs];+ item.second[B - b.docs].docs = nullptr;+ }++ out++;++ if (++A == endA || ++B == endB) {+ result.count = out - result.docs;+ return;+ }+ } else {+ goto SKIP_FIRST_COMPARE;+ }+ }+}+void copy_reference_ids(filter_result_t& from, filter_result_t& to) {if (to.count > 0 && !from.reference_filter_results.empty()) {for (const auto &item: from.reference_filter_results) {@@ -2144,22 +2204,21 @@ Option<bool> Index::recursive_filter(filter_node_t* const root,}}- uint32_t* filtered_results = nullptr;if (root->filter_operator == AND) {- result.count = ArrayUtils::and_scalar(- l_result.docs, l_result.count, r_result.docs,- r_result.count, &filtered_results);+ and_filter_result(l_result, r_result, result);} else {+ uint32_t* filtered_results = nullptr;result.count = ArrayUtils::or_scalar(l_result.docs, l_result.count, r_result.docs,r_result.count, &filtered_results);- }- result.docs = filtered_results;- if (!l_result.reference_filter_results.empty() || !r_result.reference_filter_results.empty()) {- copy_reference_ids(!l_result.reference_filter_results.empty() ? l_result : r_result, result);+ result.docs = filtered_results;+ if (!l_result.reference_filter_results.empty() || !r_result.reference_filter_results.empty()) {+ copy_reference_ids(!l_result.reference_filter_results.empty() ? l_result : r_result, result);+ }}+return Option(true);}diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex b25439e6..6df9d397 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -551,6 +551,214 @@ TEST_F(CollectionJoinTest, FilterByReference_MultipleMatch) {collectionManager.drop_collection("Links");}+TEST_F(CollectionJoinTest, FilterByNReferences) {+ auto schema_json =+ R"({+ "name": "Users",+ "fields": [+ {"name": "user_id", "type": "string"},+ {"name": "user_name", "type": "string"}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "user_id": "user_a",+ "user_name": "Roshan"+ })"_json,+ R"({+ "user_id": "user_b",+ "user_name": "Ruby"+ })"_json,+ R"({+ "user_id": "user_c",+ "user_name": "Joe"+ })"_json,+ R"({+ "user_id": "user_d",+ "user_name": "Aby"+ })"_json+ };+ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Repos",+ "fields": [+ {"name": "repo_id", "type": "string"},+ {"name": "repo_content", "type": "string"},+ {"name": "repo_stars", "type": "int32"},+ {"name": "repo_is_private", "type": "bool"}+ ]+ })"_json;+ documents = {+ R"({+ "repo_id": "repo_a",+ "repo_content": "body1",+ "repo_stars": 431,+ "repo_is_private": true+ })"_json,+ R"({+ "repo_id": "repo_b",+ "repo_content": "body2",+ "repo_stars": 4562,+ "repo_is_private": false+ })"_json,+ R"({+ "repo_id": "repo_c",+ "repo_content": "body3",+ "repo_stars": 945,+ "repo_is_private": false+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Links",+ "fields": [+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id"},+ {"name": "user_id", "type": "string", "reference": "Users.user_id"}+ ]+ })"_json;+ documents = {+ R"({+ "repo_id": "repo_a",+ "user_id": "user_b"+ })"_json,+ R"({+ "repo_id": "repo_a",+ "user_id": "user_c"+ })"_json,+ R"({+ "repo_id": "repo_b",+ "user_id": "user_a"+ })"_json,+ R"({+ "repo_id": "repo_b",+ "user_id": "user_b"+ })"_json,+ R"({+ "repo_id": "repo_b",+ "user_id": "user_d"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_a"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_b"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_c"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_d"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Organizations",+ "fields": [+ {"name": "org_id", "type": "string"},+ {"name": "org_name", "type": "string"}+ ]+ })"_json;+ documents = {+ R"({+ "org_id": "org_a",+ "org_name": "Typesense"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Participants",+ "fields": [+ {"name": "user_id", "type": "string", "reference": "Users.user_id"},+ {"name": "org_id", "type": "string", "reference": "Organizations.org_id"}+ ]+ })"_json;+ documents = {+ R"({+ "user_id": "user_a",+ "org_id": "org_a"+ })"_json,+ R"({+ "user_id": "user_b",+ "org_id": "org_a"+ })"_json,+ R"({+ "user_id": "user_d",+ "org_id": "org_a"+ })"_json,+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ auto coll = collectionManager.get_collection_unsafe("Users");++ // Search for users within an organization with access to a particular repo.+ auto result = coll->search("R", {"user_name"}, "$Participants(org_id:=org_a) && $Links(repo_id:=repo_b)", {}, {}, {0},+ 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD).get();++ ASSERT_EQ(2, result["found"].get<size_t>());+ ASSERT_EQ(2, result["hits"].size());+ ASSERT_EQ("user_b", result["hits"][0]["document"]["user_id"].get<std::string>());+ ASSERT_EQ("user_a", result["hits"][1]["document"]["user_id"].get<std::string>());++ collectionManager.drop_collection("Users");+ collectionManager.drop_collection("Repos");+ collectionManager.drop_collection("Links");+}+TEST_F(CollectionJoinTest, IncludeFieldsByReference_SingleMatch) {auto schema_json =R"({