Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow join on referenced collection.

Harpreet Sangar8d654c56eeb6
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 5c892c7f..a3e0650a 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -601,22 +601,63 @@ void filter_result_iterator_t::init() {
}
auto coll = cm.get_collection(collection_name);
- if (coll->referenced_in.count(ref_collection_name) == 0 || coll->referenced_in.at(ref_collection_name).empty()) {
- status = Option<bool>(400, "Could not find a reference to `" + collection_name + "` in `" +
- ref_collection_name + "` collection.");
+ bool is_referenced = coll->referenced_in.count(ref_collection_name) > 0,
+ has_reference = ref_collection->is_referenced_in(collection_name);
+ if (!is_referenced && !has_reference) {
+ status = Option<bool>(400, "Failed to join on `" + ref_collection_name + "`: No reference field found.");
is_valid = false;
return;
}
- auto const& field_name = coll->referenced_in.at(ref_collection_name);
- auto reference_filter_op = ref_collection->get_reference_filter_ids(a_filter.field_name,
- filter_result,
- field_name);
- if (!reference_filter_op.ok()) {
- status = Option<bool>(400, "Failed to apply reference filter on `" + a_filter.referenced_collection_name
- + "` collection: " + reference_filter_op.error());
- is_valid = false;
- return;
+ if (is_referenced) {
+ auto const& field_name = coll->referenced_in.at(ref_collection_name);
+ auto reference_filter_op = ref_collection->get_reference_filter_ids(a_filter.field_name,
+ filter_result,
+ field_name);
+ if (!reference_filter_op.ok()) {
+ status = Option<bool>(400, "Failed to join on `" + a_filter.referenced_collection_name
+ + "` collection: " + reference_filter_op.error());
+ is_valid = false;
+ return;
+ }
+ } else if (has_reference) {
+ // Get the doc ids of reference collection matching the filter then apply filter on the current collection's
+ // reference helper field.
+ filter_result_t result;
+ auto reference_filter_op = ref_collection->get_filter_ids(a_filter.field_name, result);
+ if (!reference_filter_op.ok()) {
+ status = Option<bool>(400, "Failed to join on `" + a_filter.referenced_collection_name
+ + "` collection: " + reference_filter_op.error());
+ is_valid = false;
+ return;
+ }
+
+ auto get_reference_field_op = ref_collection->get_reference_field(collection_name);
+ if (!get_reference_field_op.ok()) {
+ status = Option<bool>(get_reference_field_op.code(), get_reference_field_op.error());
+ is_valid = false;
+ return;
+ }
+
+ std::vector<std::string> values;
+ for (uint32_t i = 0; i < result.count; i++) {
+ values.push_back(std::to_string(result.docs[i]));
+ }
+
+ filter filter_exp = {get_reference_field_op.get(), std::move(values), {EQUALS}};
+
+ auto filter_tree_root = new filter_node_t(filter_exp);
+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
+
+ auto fit = filter_result_iterator_t(collection_name, index, filter_tree_root);
+ auto filter_init_op = fit.init_status();
+ if (!filter_init_op.ok()) {
+ status = Option<bool>(filter_init_op.code(), filter_init_op.error());
+ is_valid = false;
+ return;
+ }
+
+ filter_result = std::move(fit.filter_result);
}
if (filter_result.count == 0) {
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index bc8dea8c..212e720e 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -333,19 +333,22 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
"fields": [
{"name": "product_id", "type": "string"},
{"name": "product_name", "type": "string"},
- {"name": "product_description", "type": "string"}
+ {"name": "product_description", "type": "string"},
+ {"name": "rating", "type": "int32"}
]
})"_json;
std::vector<nlohmann::json> documents = {
R"({
"product_id": "product_a",
"product_name": "shampoo",
- "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."
+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",
+ "rating": "2"
})"_json,
R"({
"product_id": "product_b",
"product_name": "soap",
- "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."
+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",
+ "rating": "4"
})"_json
};
auto collection_create_op = collectionManager.create_collection(schema_json);
@@ -404,6 +407,16 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
ASSERT_TRUE(add_op.ok());
}
+ schema_json =
+ R"({
+ "name": "Dummy",
+ "fields": [
+ {"name": "dummy_id", "type": "string"}
+ ]
+ })"_json;
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
auto coll = collectionManager.get_collection_unsafe("Products");
auto search_op = coll->search("s", {"product_name"}, "$foo:=customer_a", {}, {}, {0},
10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD);
@@ -420,10 +433,15 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
ASSERT_FALSE(search_op.ok());
ASSERT_EQ(search_op.error(), "Referenced collection `foo` not found.");
+ search_op = coll->search("s", {"product_name"}, "$Dummy(dummy_id:=customer_a)", {}, {}, {0},
+ 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ(search_op.error(), "Failed to join on `Dummy`: No reference field found.");
+
search_op = coll->search("s", {"product_name"}, "$Customers(foo:=customer_a)", {}, {}, {0},
10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD);
ASSERT_FALSE(search_op.ok());
- ASSERT_EQ(search_op.error(), "Failed to apply reference filter on `Customers` collection: Could not find a filter "
+ ASSERT_EQ(search_op.error(), "Failed to join on `Customers` collection: Could not find a filter "
"field named `foo` in the schema.");
auto result = coll->search("s", {"product_name"}, "$Customers(customer_id:=customer_a && product_price:<100)", {},
@@ -433,6 +451,37 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
ASSERT_EQ(1, result["hits"].size());
ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());
+ std::map<std::string, std::string> req_params = {
+ {"collection", "Customers"},
+ {"q", "Dan"},
+ {"query_by", "customer_name"},
+ {"filter_by", "$Products(foo:>3)"},
+ };
+ nlohmann::json embedded_params;
+ std::string json_res;
+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ auto search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op_bool.ok());
+ ASSERT_EQ(search_op_bool.error(), "Failed to join on `Products` collection: Could not find a filter "
+ "field named `foo` in the schema.");
+
+ req_params = {
+ {"collection", "Customers"},
+ {"q", "Dan"},
+ {"query_by", "customer_name"},
+ {"filter_by", "$Products(rating:>3)"},
+ };
+
+ search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op_bool.ok());
+
+ auto res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ("soap", result["hits"][0]["document"]["product_name"].get<std::string>());
+
collectionManager.drop_collection("Customers");
collectionManager.drop_collection("Products");
}
@@ -1055,19 +1104,22 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"name": "product_id", "type": "string"},
{"name": "product_name", "type": "string", "infix": true},
{"name": "product_description", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}},
+ {"name": "rating", "type": "int32"}
]
})"_json;
std::vector<nlohmann::json> documents = {
R"({
"product_id": "product_a",
"product_name": "shampoo",
- "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."
+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",
+ "rating": "2"
})"_json,
R"({
"product_id": "product_b",
"product_name": "soap",
- "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."
+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",
+ "rating": "4"
})"_json
};
@@ -1167,12 +1219,13 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
// No fields are mentioned in `include_fields`, should include all fields of Products and Customers by default.
- ASSERT_EQ(9, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(10, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
@@ -1192,12 +1245,13 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
// No fields of Products collection are mentioned in `include_fields`, should include all of its fields by default.
- ASSERT_EQ(5, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(6, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));
req_params = {
{"collection", "Products"},
@@ -1212,7 +1266,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
- ASSERT_EQ(6, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(7, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
@@ -1229,7 +1283,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
- ASSERT_EQ(7, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(8, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));
@@ -1248,8 +1302,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
- // 5 fields in Products document and 2 fields from Customers document
- ASSERT_EQ(7, res_obj["hits"][0]["document"].size());
+ // 6 fields in Products document and 2 fields from Customers document
+ ASSERT_EQ(8, res_obj["hits"][0]["document"].size());
req_params = {
{"collection", "Products"},
@@ -1264,8 +1318,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
- // 5 fields in Products document and 2 fields from Customers document
- ASSERT_EQ(7, res_obj["hits"][0]["document"].size());
+ // 6 fields in Products document and 2 fields from Customers document
+ ASSERT_EQ(8, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id_sequence_id"));
@@ -1283,8 +1337,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
- // 5 fields in Products document and 1 fields from Customers document
- ASSERT_EQ(6, res_obj["hits"][0]["document"].size());
+ // 6 fields in Products document and 1 fields from Customers document
+ ASSERT_EQ(7, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));
@@ -1491,6 +1545,26 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ req_params = {
+ {"collection", "Customers"},
+ {"q", "Dan"},
+ {"query_by", "customer_name"},
+ {"filter_by", "$Products(rating:>3)"},
+ {"include_fields", "$Products(product_name), product_price"}
+ };
+
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
+ ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("product_name"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(140, res_obj["hits"][0]["document"].at("product_price"));
+
// Reference include_by without join
req_params = {
{"collection", "Customers"},