Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow join on referenced collection.
Harpreet Sangar8d654c56eeb6
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 5c892c7f..a3e0650a 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -601,22 +601,63 @@ void filter_result_iterator_t::init() {}auto coll = cm.get_collection(collection_name);- if (coll->referenced_in.count(ref_collection_name) == 0 || coll->referenced_in.at(ref_collection_name).empty()) {- status = Option<bool>(400, "Could not find a reference to `" + collection_name + "` in `" +- ref_collection_name + "` collection.");+ bool is_referenced = coll->referenced_in.count(ref_collection_name) > 0,+ has_reference = ref_collection->is_referenced_in(collection_name);+ if (!is_referenced && !has_reference) {+ status = Option<bool>(400, "Failed to join on `" + ref_collection_name + "`: No reference field found.");is_valid = false;return;}- auto const& field_name = coll->referenced_in.at(ref_collection_name);- auto reference_filter_op = ref_collection->get_reference_filter_ids(a_filter.field_name,- filter_result,- field_name);- if (!reference_filter_op.ok()) {- status = Option<bool>(400, "Failed to apply reference filter on `" + a_filter.referenced_collection_name- + "` collection: " + reference_filter_op.error());- is_valid = false;- return;+ if (is_referenced) {+ auto const& field_name = coll->referenced_in.at(ref_collection_name);+ auto reference_filter_op = ref_collection->get_reference_filter_ids(a_filter.field_name,+ filter_result,+ field_name);+ if (!reference_filter_op.ok()) {+ status = Option<bool>(400, "Failed to join on `" + a_filter.referenced_collection_name+ + "` collection: " + reference_filter_op.error());+ is_valid = false;+ return;+ }+ } else if (has_reference) {+ // Get the doc ids of reference collection matching the filter then apply filter on the current collection's+ // reference helper field.+ filter_result_t result;+ auto reference_filter_op = ref_collection->get_filter_ids(a_filter.field_name, result);+ if (!reference_filter_op.ok()) {+ status = Option<bool>(400, "Failed to join on `" + a_filter.referenced_collection_name+ + "` collection: " + reference_filter_op.error());+ is_valid = false;+ return;+ }++ auto get_reference_field_op = ref_collection->get_reference_field(collection_name);+ if (!get_reference_field_op.ok()) {+ status = Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());+ is_valid = false;+ return;+ }++ std::vector<std::string> values;+ for (uint32_t i = 0; i < result.count; i++) {+ values.push_back(std::to_string(result.docs[i]));+ }++ filter filter_exp = {get_reference_field_op.get(), std::move(values), {EQUALS}};++ auto filter_tree_root = new filter_node_t(filter_exp);+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);++ auto fit = filter_result_iterator_t(collection_name, index, filter_tree_root);+ auto filter_init_op = fit.init_status();+ if (!filter_init_op.ok()) {+ status = Option<bool>(filter_init_op.code(), filter_init_op.error());+ is_valid = false;+ return;+ }++ filter_result = std::move(fit.filter_result);}if (filter_result.count == 0) {diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex bc8dea8c..212e720e 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -333,19 +333,22 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {"fields": [{"name": "product_id", "type": "string"},{"name": "product_name", "type": "string"},- {"name": "product_description", "type": "string"}+ {"name": "product_description", "type": "string"},+ {"name": "rating", "type": "int32"}]})"_json;std::vector<nlohmann::json> documents = {R"({"product_id": "product_a","product_name": "shampoo",- "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",+ "rating": "2"})"_json,R"({"product_id": "product_b","product_name": "soap",- "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",+ "rating": "4"})"_json};auto collection_create_op = collectionManager.create_collection(schema_json);@@ -404,6 +407,16 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {ASSERT_TRUE(add_op.ok());}+ schema_json =+ R"({+ "name": "Dummy",+ "fields": [+ {"name": "dummy_id", "type": "string"}+ ]+ })"_json;+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+auto coll = collectionManager.get_collection_unsafe("Products");auto search_op = coll->search("s", {"product_name"}, "$foo:=customer_a", {}, {}, {0},10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD);@@ -420,10 +433,15 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {ASSERT_FALSE(search_op.ok());ASSERT_EQ(search_op.error(), "Referenced collection `foo` not found.");+ search_op = coll->search("s", {"product_name"}, "$Dummy(dummy_id:=customer_a)", {}, {}, {0},+ 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ(search_op.error(), "Failed to join on `Dummy`: No reference field found.");+search_op = coll->search("s", {"product_name"}, "$Customers(foo:=customer_a)", {}, {}, {0},10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD);ASSERT_FALSE(search_op.ok());- ASSERT_EQ(search_op.error(), "Failed to apply reference filter on `Customers` collection: Could not find a filter "+ ASSERT_EQ(search_op.error(), "Failed to join on `Customers` collection: Could not find a filter ""field named `foo` in the schema.");auto result = coll->search("s", {"product_name"}, "$Customers(customer_id:=customer_a && product_price:<100)", {},@@ -433,6 +451,37 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {ASSERT_EQ(1, result["hits"].size());ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());+ std::map<std::string, std::string> req_params = {+ {"collection", "Customers"},+ {"q", "Dan"},+ {"query_by", "customer_name"},+ {"filter_by", "$Products(foo:>3)"},+ };+ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ auto search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op_bool.ok());+ ASSERT_EQ(search_op_bool.error(), "Failed to join on `Products` collection: Could not find a filter "+ "field named `foo` in the schema.");++ req_params = {+ {"collection", "Customers"},+ {"q", "Dan"},+ {"query_by", "customer_name"},+ {"filter_by", "$Products(rating:>3)"},+ };++ search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op_bool.ok());++ auto res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());+collectionManager.drop_collection("Customers");collectionManager.drop_collection("Products");}@@ -1055,19 +1104,22 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {{"name": "product_id", "type": "string"},{"name": "product_name", "type": "string", "infix": true},{"name": "product_description", "type": "string"},- {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}},+ {"name": "rating", "type": "int32"}]})"_json;std::vector<nlohmann::json> documents = {R"({"product_id": "product_a","product_name": "shampoo",- "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",+ "rating": "2"})"_json,R"({"product_id": "product_b","product_name": "soap",- "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",+ "rating": "4"})"_json};@@ -1167,12 +1219,13 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());// No fields are mentioned in `include_fields`, should include all fields of Products and Customers by default.- ASSERT_EQ(9, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(10, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));@@ -1192,12 +1245,13 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());// No fields of Products collection are mentioned in `include_fields`, should include all of its fields by default.- ASSERT_EQ(5, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(6, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));req_params = {{"collection", "Products"},@@ -1212,7 +1266,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());- ASSERT_EQ(6, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(7, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));@@ -1229,7 +1283,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());- ASSERT_EQ(7, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(8, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));@@ -1248,8 +1302,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());- // 5 fields in Products document and 2 fields from Customers document- ASSERT_EQ(7, res_obj["hits"][0]["document"].size());+ // 6 fields in Products document and 2 fields from Customers document+ ASSERT_EQ(8, res_obj["hits"][0]["document"].size());req_params = {{"collection", "Products"},@@ -1264,8 +1318,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());- // 5 fields in Products document and 2 fields from Customers document- ASSERT_EQ(7, res_obj["hits"][0]["document"].size());+ // 6 fields in Products document and 2 fields from Customers document+ ASSERT_EQ(8, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id_sequence_id"));@@ -1283,8 +1337,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {res_obj = nlohmann::json::parse(json_res);ASSERT_EQ(1, res_obj["found"].get<size_t>());ASSERT_EQ(1, res_obj["hits"].size());- // 5 fields in Products document and 1 fields from Customers document- ASSERT_EQ(6, res_obj["hits"][0]["document"].size());+ // 6 fields in Products document and 1 fields from Customers document+ ASSERT_EQ(7, res_obj["hits"][0]["document"].size());ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));@@ -1491,6 +1545,26 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ req_params = {+ {"collection", "Customers"},+ {"q", "Dan"},+ {"query_by", "customer_name"},+ {"filter_by", "$Products(rating:>3)"},+ {"include_fields", "$Products(product_name), product_price"}+ };++ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(140, res_obj["hits"][0]["document"].at("product_price"));+// Reference include_by without joinreq_params = {{"collection", "Customers"},