Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Remove `Collection::validate_reference_filter`. Add comments.

Harpreet Sangar9641d423152d
diff --git a/include/collection.h b/include/collection.h
index dc8f41db..d0fdfec9 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -128,7 +128,7 @@ private:
SynonymIndex* synonym_index;
- // "field name" -> reference_pair
+ /// "field name" -> reference_pair(referenced_collection_name, referenced_field_name)
spp::sparse_hash_map<std::string, reference_pair> reference_fields;
// Keep index as the last field since it is initialized in the constructor via init_index(). Add a new field before it.
@@ -465,8 +465,6 @@ public:
filter_result_t& filter_result,
const std::string& collection_name) const;
- Option<bool> validate_reference_filter(const std::string& filter_query) const;
-
Option<nlohmann::json> get(const std::string & id) const;
Option<std::string> remove(const std::string & id, bool remove_from_store = true);
diff --git a/include/field.h b/include/field.h
index 0af1db89..8db3ce1c 100644
--- a/include/field.h
+++ b/include/field.h
@@ -659,6 +659,7 @@ struct filter_result_t {
docs = new uint32_t[count];
memcpy(docs, obj.docs, count * sizeof(uint32_t));
+ // Copy every collection's references.
for (const auto &item: obj.reference_filter_results) {
reference_filter_results[item.first] = new reference_filter_result_t[count];
diff --git a/src/collection.cpp b/src/collection.cpp
index d35e2e10..26032e7f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -137,12 +137,15 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::
"` in the collection `" + reference_collection_name + "` must be indexed.");
}
+ // Get the doc id of the referenced document.
auto value = document[field_name].get<std::string>();
filter_result_t filter_result;
collection->get_filter_ids(reference_field_name + ":=" + value, filter_result);
if (filter_result.count != 1) {
auto match = " `" + reference_field_name + ": " + value + "` ";
+
+ // Constraints similar to foreign key apply here. The reference match must be unique and not null.
return Option<doc_seq_id_t>(400, filter_result.count < 1 ?
"Referenced document having" + match + "not found in the collection `"
+ reference_collection_name + "`." :
@@ -2578,21 +2581,6 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_que
return index->do_reference_filtering_with_lock(filter_tree_root, filter_result, name, field_name);
}
-Option<bool> Collection::validate_reference_filter(const std::string& filter_query) const {
- std::shared_lock lock(mutex);
-
- const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
- filter_node_t* filter_tree_root = nullptr;
- Option<bool> filter_op = filter::parse_filter_query(filter_query, search_schema,
- store, doc_id_prefix, filter_tree_root);
- if(!filter_op.ok()) {
- return filter_op;
- }
-
- delete filter_tree_root;
- return Option<bool>(true);
-}
-
bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,
const nlohmann::json &document, std::string &value) const {
diff --git a/src/field.cpp b/src/field.cpp
index 0a0380b5..60a8695a 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -431,14 +431,6 @@ Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,
filter_exp = {expression.substr(parenthesis_index + 1, expression.size() - parenthesis_index - 2)};
filter_exp.referenced_collection_name = collection_name;
-
- auto op = collection->validate_reference_filter(filter_exp.field_name);
- if (!op.ok()) {
- is_successful = false;
- error_message = "Failed to parse reference filter on `" + collection_name + "` collection: " +
- op.error();
- break;
- }
} else {
Option<bool> toFilter_op = toFilter(expression, filter_exp, search_schema, store, doc_id_prefix);
if (!toFilter_op.ok()) {
@@ -790,6 +782,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
);
if (!field_json[fields::reference].get<std::string>().empty()) {
+ // Add a reference helper field in the schema. It stores the doc id of the document it references to reduce the
+ // computation while searching.
the_fields.emplace_back(
field(field_json[fields::name].get<std::string>() + Collection::REFERENCE_HELPER_FIELD_SUFFIX,
"int64", false, field_json[fields::optional], true)
@@ -996,6 +990,7 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_
const uint32_t *endA = A + lenA;
const uint32_t *endB = B + lenB;
+ // Add an entry of references in the result for each unique collection in a and b.
for (auto const& item: a.reference_filter_results) {
if (result.reference_filter_results.count(item.first) == 0) {
result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];
@@ -1024,6 +1019,7 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_
if (*A == *B) {
*out = *A;
+ // Copy the references of the document from every collection into result.
for (auto const& item: a.reference_filter_results) {
result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];
}
@@ -1048,11 +1044,11 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
return;
}
+ // If either one of a or b does not have any matches, copy other into result.
if (a.count == 0) {
result = b;
return;
}
-
if (b.count == 0) {
result = a;
return;
@@ -1061,6 +1057,7 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
size_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;
result.docs = new uint32_t[lenA + lenB];
+ // Add an entry of references in the result for each unique collection in a and b.
for (auto const& item: a.reference_filter_results) {
if (result.reference_filter_results.count(item.first) == 0) {
result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];
@@ -1080,6 +1077,7 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_r
res_index++;
}
+ // Copy references of the last result document from every collection in a.
for (auto const& item: a.reference_filter_results) {
result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];
}
diff --git a/src/index.cpp b/src/index.cpp
index f20efe47..4133a821 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1513,7 +1513,8 @@ Option<bool> Index::do_filtering(filter_node_t* const root,
reference_filter_result,
collection_name);
if (!reference_filter_op.ok()) {
- return reference_filter_op;
+ return Option<bool>(400, "Failed to parse reference filter on `" + a_filter.referenced_collection_name
+ +"` collection: " + reference_filter_op.error());
}
if (context_ids_length != 0) {
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 9fa65b4c..a5ce52af 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -355,7 +355,7 @@ Option<bool> parse_reference_filter(const std::string& filter_query, std::queue<
return error;
}
- int start_index = index;
+ size_t start_index = index;
auto size = filter_query.size();
while(++index < size && filter_query[index] != '(') {}
@@ -363,6 +363,7 @@ Option<bool> parse_reference_filter(const std::string& filter_query, std::queue<
return error;
}
+ // The reference filter could have parenthesis inside it. $Foo((X && Y) || Z)
int parenthesis_count = 1;
while (++index < size && parenthesis_count > 0) {
if (filter_query[index] == '(') {
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index d3d2ce23..fe1c32fa 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -137,8 +137,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
nlohmann::json customer_json = R"({
"customer_id": "customer_a",
"customer_name": "Joe",
- "product_price": 143,
- "product_id": "a"
+ "product_price": 143
})"_json;
auto add_doc_op = customer_collection->add(customer_json.dump());
@@ -591,6 +590,7 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {
for (size_t i = 0; i < a.count; i++) {
a.docs[i] = i;
+ // Having only one reference of each document for brevity.
auto& reference = a.reference_filter_results["foo"][i];
reference.count = 1;
reference.docs = new uint32_t[1];
@@ -627,6 +627,7 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {
for(size_t i = 0; i < result.count; i++) {
ASSERT_EQ(docs[i], result.docs[i]);
+ // result should contain correct references to the foo and bar collection.
ASSERT_EQ(1, result.reference_filter_results["foo"][i].count);
ASSERT_EQ(foo_reference[i], result.reference_filter_results["foo"][i].docs[0]);
ASSERT_EQ(1, result.reference_filter_results["bar"][i].count);
@@ -746,6 +747,8 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
ASSERT_EQ(10, result2.count);
expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+
+ // doc_id -> reference_id
std::map<uint32_t, uint32_t> foo_map = {{3, 6}, {6, 12}, {9, 18}}, bar_map = {{0, 10}, {1, 9}, {2, 8}, {3, 7},
{4, 6}, {5, 5}, {6, 4}, {7, 3}, {8, 2}};
for (size_t i = 0; i < result2.count; i++) {
@@ -755,6 +758,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {
ASSERT_EQ(1, result2.reference_filter_results["foo"][i].count);
ASSERT_EQ(foo_map[i], result2.reference_filter_results["foo"][i].docs[0]);
} else {
+ // Reference count should be 0 for the docs that were not present in the a result.
ASSERT_EQ(0, result2.reference_filter_results["foo"][i].count);
}