Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Remove `Collection::validate_reference_filter`. Add comments.
Harpreet Sangar9641d423152d
diff --git a/include/collection.h b/include/collection.hindex dc8f41db..d0fdfec9 100644--- a/include/collection.h+++ b/include/collection.h@@ -128,7 +128,7 @@ private:SynonymIndex* synonym_index;- // "field name" -> reference_pair+ /// "field name" -> reference_pair(referenced_collection_name, referenced_field_name)spp::sparse_hash_map<std::string, reference_pair> reference_fields;// Keep index as the last field since it is initialized in the constructor via init_index(). Add a new field before it.@@ -465,8 +465,6 @@ public:filter_result_t& filter_result,const std::string& collection_name) const;- Option<bool> validate_reference_filter(const std::string& filter_query) const;-Option<nlohmann::json> get(const std::string & id) const;Option<std::string> remove(const std::string & id, bool remove_from_store = true);diff --git a/include/field.h b/include/field.hindex 0af1db89..8db3ce1c 100644--- a/include/field.h+++ b/include/field.h@@ -659,6 +659,7 @@ struct filter_result_t {docs = new uint32_t[count];memcpy(docs, obj.docs, count * sizeof(uint32_t));+ // Copy every collection's references.for (const auto &item: obj.reference_filter_results) {reference_filter_results[item.first] = new reference_filter_result_t[count];diff --git a/src/collection.cpp b/src/collection.cppindex d35e2e10..26032e7f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -137,12 +137,15 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::"` in the collection `" + reference_collection_name + "` must be indexed.");}+ // Get the doc id of the referenced document.auto value = document[field_name].get<std::string>();filter_result_t filter_result;collection->get_filter_ids(reference_field_name + ":=" + value, filter_result);if (filter_result.count != 1) {auto match = " `" + reference_field_name + ": " + value + "` ";++ // Constraints similar to foreign key apply here. The reference match must be unique and not null.return Option<doc_seq_id_t>(400, filter_result.count < 1 ?"Referenced document having" + match + "not found in the collection `"+ reference_collection_name + "`." :@@ -2578,21 +2581,6 @@ Option<bool> Collection::get_reference_filter_ids(const std::string & filter_quereturn index->do_reference_filtering_with_lock(filter_tree_root, filter_result, name, field_name);}-Option<bool> Collection::validate_reference_filter(const std::string& filter_query) const {- std::shared_lock lock(mutex);-- const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";- filter_node_t* filter_tree_root = nullptr;- Option<bool> filter_op = filter::parse_filter_query(filter_query, search_schema,- store, doc_id_prefix, filter_tree_root);- if(!filter_op.ok()) {- return filter_op;- }-- delete filter_tree_root;- return Option<bool>(true);-}-bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,const nlohmann::json &document, std::string &value) const {diff --git a/src/field.cpp b/src/field.cppindex 0a0380b5..60a8695a 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -431,14 +431,6 @@ Option<bool> toParseTree(std::queue<std::string>& postfix, filter_node_t*& root,filter_exp = {expression.substr(parenthesis_index + 1, expression.size() - parenthesis_index - 2)};filter_exp.referenced_collection_name = collection_name;-- auto op = collection->validate_reference_filter(filter_exp.field_name);- if (!op.ok()) {- is_successful = false;- error_message = "Failed to parse reference filter on `" + collection_name + "` collection: " +- op.error();- break;- }} else {Option<bool> toFilter_op = toFilter(expression, filter_exp, search_schema, store, doc_id_prefix);if (!toFilter_op.ok()) {@@ -790,6 +782,8 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso);if (!field_json[fields::reference].get<std::string>().empty()) {+ // Add a reference helper field in the schema. It stores the doc id of the document it references to reduce the+ // computation while searching.the_fields.emplace_back(field(field_json[fields::name].get<std::string>() + Collection::REFERENCE_HELPER_FIELD_SUFFIX,"int64", false, field_json[fields::optional], true)@@ -996,6 +990,7 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_const uint32_t *endA = A + lenA;const uint32_t *endB = B + lenB;+ // Add an entry of references in the result for each unique collection in a and b.for (auto const& item: a.reference_filter_results) {if (result.reference_filter_results.count(item.first) == 0) {result.reference_filter_results[item.first] = new reference_filter_result_t[std::min(lenA, lenB)];@@ -1024,6 +1019,7 @@ void filter_result_t::and_filter_results(const filter_result_t& a, const filter_if (*A == *B) {*out = *A;+ // Copy the references of the document from every collection into result.for (auto const& item: a.reference_filter_results) {result.reference_filter_results[item.first][out - result.docs] = item.second[A - a.docs];}@@ -1048,11 +1044,11 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rreturn;}+ // If either one of a or b does not have any matches, copy other into result.if (a.count == 0) {result = b;return;}-if (b.count == 0) {result = a;return;@@ -1061,6 +1057,7 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rsize_t indexA = 0, indexB = 0, res_index = 0, lenA = a.count, lenB = b.count;result.docs = new uint32_t[lenA + lenB];+ // Add an entry of references in the result for each unique collection in a and b.for (auto const& item: a.reference_filter_results) {if (result.reference_filter_results.count(item.first) == 0) {result.reference_filter_results[item.first] = new reference_filter_result_t[lenA + lenB];@@ -1080,6 +1077,7 @@ void filter_result_t::or_filter_results(const filter_result_t& a, const filter_rres_index++;}+ // Copy references of the last result document from every collection in a.for (auto const& item: a.reference_filter_results) {result.reference_filter_results[item.first][res_index - 1] = item.second[indexA];}diff --git a/src/index.cpp b/src/index.cppindex f20efe47..4133a821 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1513,7 +1513,8 @@ Option<bool> Index::do_filtering(filter_node_t* const root,reference_filter_result,collection_name);if (!reference_filter_op.ok()) {- return reference_filter_op;+ return Option<bool>(400, "Failed to parse reference filter on `" + a_filter.referenced_collection_name+ +"` collection: " + reference_filter_op.error());}if (context_ids_length != 0) {diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 9fa65b4c..a5ce52af 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -355,7 +355,7 @@ Option<bool> parse_reference_filter(const std::string& filter_query, std::queue<return error;}- int start_index = index;+ size_t start_index = index;auto size = filter_query.size();while(++index < size && filter_query[index] != '(') {}@@ -363,6 +363,7 @@ Option<bool> parse_reference_filter(const std::string& filter_query, std::queue<return error;}+ // The reference filter could have parenthesis inside it. $Foo((X && Y) || Z)int parenthesis_count = 1;while (++index < size && parenthesis_count > 0) {if (filter_query[index] == '(') {diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex d3d2ce23..fe1c32fa 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -137,8 +137,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {nlohmann::json customer_json = R"({"customer_id": "customer_a","customer_name": "Joe",- "product_price": 143,- "product_id": "a"+ "product_price": 143})"_json;auto add_doc_op = customer_collection->add(customer_json.dump());@@ -591,6 +590,7 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {for (size_t i = 0; i < a.count; i++) {a.docs[i] = i;+ // Having only one reference of each document for brevity.auto& reference = a.reference_filter_results["foo"][i];reference.count = 1;reference.docs = new uint32_t[1];@@ -627,6 +627,7 @@ TEST_F(CollectionJoinTest, AndFilterResults_WithReferences) {for(size_t i = 0; i < result.count; i++) {ASSERT_EQ(docs[i], result.docs[i]);+ // result should contain correct references to the foo and bar collection.ASSERT_EQ(1, result.reference_filter_results["foo"][i].count);ASSERT_EQ(foo_reference[i], result.reference_filter_results["foo"][i].docs[0]);ASSERT_EQ(1, result.reference_filter_results["bar"][i].count);@@ -746,6 +747,8 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {ASSERT_EQ(10, result2.count);expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};++ // doc_id -> reference_idstd::map<uint32_t, uint32_t> foo_map = {{3, 6}, {6, 12}, {9, 18}}, bar_map = {{0, 10}, {1, 9}, {2, 8}, {3, 7},{4, 6}, {5, 5}, {6, 4}, {7, 3}, {8, 2}};for (size_t i = 0; i < result2.count; i++) {@@ -755,6 +758,7 @@ TEST_F(CollectionJoinTest, OrFilterResults_WithReferences) {ASSERT_EQ(1, result2.reference_filter_results["foo"][i].count);ASSERT_EQ(foo_map[i], result2.reference_filter_results["foo"][i].docs[0]);} else {+ // Reference count should be 0 for the docs that were not present in the a result.ASSERT_EQ(0, result2.reference_filter_results["foo"][i].count);}