Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Return all included overrides if provided `filter_by` produces no match. (#1861)
Harpreet Sangarfc153ae19283
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex d35744cc..08bbb5a7 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -425,4 +425,8 @@ public:[[nodiscard]] bool _get_is_equals_iterator_valid() const {return is_equals_iterator_valid;}++ inline bool is_filter_provided() const {+ return filter_node != nullptr;+ }};diff --git a/src/index.cpp b/src/index.cppindex f958cc00..6e240e9d 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2756,10 +2756,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consreturn filter_init_op;}- if (filter_tree_root != nullptr && filter_result_iterator->validity != filter_result_iterator_t::valid) {- return Option(true);- }-#ifdef TEST_BUILDif (filter_result_iterator->approx_filter_ids_length > 20) {@@ -2789,6 +2785,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());+ bool const& filter_by_provided = filter_tree_root != nullptr;+ bool const& no_filter_by_matches = filter_by_provided && filter_result_iterator->approx_filter_ids_length == 0;++ // If curation is not involved and there are no filter matches, return early.+ if (curated_ids_sorted.empty() && no_filter_by_matches) {+ return Option(true);+ }+// Order of `fields` are used to sort results// auto begin = std::chrono::high_resolution_clock::now();uint32_t* all_result_ids = nullptr;@@ -2816,8 +2820,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons// phrase queries are handled as a filtering querybool is_wildcard_non_phrase_query = is_wildcard_query && field_query_tokens[0].q_phrases.empty();-- bool no_filters_provided = (filter_tree_root == nullptr && !filter_result_iterator->validity == filter_result_iterator_t::valid);// handle phrase searchesif (!field_query_tokens[0].q_phrases.empty()) {@@ -2845,7 +2847,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}// for phrase query, parser will set field_query_tokens to "*", need to handle thatif (is_wildcard_non_phrase_query) {- if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&+ if(!filter_by_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&sort_fields_std[0].order == sort_field_const::desc) {// optimize for this path specifically@@ -2917,7 +2919,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfilter_result_iterator->compute_iterators();uint32_t filter_id_count = filter_result_iterator->approx_filter_ids_length;- if (!no_filters_provided && filter_id_count < vector_query.flat_search_cutoff) {+ if (filter_by_provided && filter_id_count < vector_query.flat_search_cutoff) {while (filter_result_iterator->validity == filter_result_iterator_t::valid) {auto seq_id = filter_result_iterator->seq_id;auto filter_result = single_filter_result_t(seq_id, std::move(filter_result_iterator->reference));@@ -2948,7 +2950,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfilter_result_iterator->reset();search_cutoff = search_cutoff || filter_result_iterator->validity == filter_result_iterator_t::timed_out;- if(no_filters_provided ||+ if(!filter_by_provided ||(filter_id_count >= vector_query.flat_search_cutoff && filter_result_iterator->validity == filter_result_iterator_t::valid)) {dist_results.clear();@@ -3060,7 +3062,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}} else {// if filters were not provided, use the seq_ids index to generate the list of all document ids- if (no_filters_provided) {+ if (!filter_by_provided) {filter_result_iterator = new filter_result_iterator_t(seq_ids->uncompress(), seq_ids->num_ids(),search_begin_us, search_stop_us);filter_iterator_guard.reset(filter_result_iterator);@@ -3428,7 +3430,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consauto& vec_result = vec_results[res_index];auto seq_id = vec_result.first;- if (!no_filters_provided && filter_result_iterator->is_valid(seq_id) != 1) {+ if (filter_by_provided && filter_result_iterator->is_valid(seq_id) != 1) {continue;}auto references = std::move(filter_result_iterator->reference);@@ -3544,7 +3546,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consbool estimate_facets = (facet_sample_percent > 0 && facet_sample_percent < 100 &&all_result_ids_len > facet_sample_threshold);- bool is_wildcard_no_filter_query = is_wildcard_non_phrase_query && no_filters_provided && vector_query.field_name.empty();+ bool is_wildcard_no_filter_query = is_wildcard_non_phrase_query && !filter_by_provided && vector_query.field_name.empty();if(!facets.empty()) {const size_t num_threads = std::min(concurrency, all_result_ids_len);@@ -3931,6 +3933,12 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fbool enable_typos_for_numerical_tokens,bool enable_typos_for_alpha_numerical_tokens) const {+ // Return early in case filter_by is provided but it matches no docs.+ if (filter_result_iterator != nullptr && filter_result_iterator->is_filter_provided() &&+ filter_result_iterator->approx_filter_ids_length == 0) {+ return Option<bool>(true);+ }+// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens// To prevent us from doing ART search repeatedly as we iterate through possible correctionsdiff --git a/src/posting_list.cpp b/src/posting_list.cppindex e9bb2610..3d684ac0 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -2154,7 +2154,7 @@ uint32_t posting_list_t::iterator_t::get_field_id() const {}bool result_iter_state_t::is_filter_provided() const {- return filter_ids_length > 0 || (fit != nullptr && fit->approx_filter_ids_length > 0);+ return filter_ids_length > 0 || (fit != nullptr && fit->is_filter_provided());}bool result_iter_state_t::is_filter_valid() const {diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 54ad410d..c9a87d62 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -78,6 +78,9 @@ TEST_F(CollectionFilteringTest, FilterOnTextFields) {results = coll_array_fields->search("Jeremy", query_fields, "tags : fine PLATINUM", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(1, results["hits"].size());+ results = coll_array_fields->search("Jeremy", query_fields, "tags : foobarbaz", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(0, results["hits"].size());+// using just ":", filtering should return documents that contain ALL tokens in the filter expressionresults = coll_array_fields->search("Jeremy", query_fields, "tags : PLATINUM", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(1, results["hits"].size());diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex ea5c7abf..2d14421e 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -786,6 +786,14 @@ TEST_F(CollectionOverrideTest, IncludeOverrideWithFilterBy) {ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+ // when filter by does not match any result, curated result should still show up+ // because `filter_curated_hits` is false+ results = coll1->search("shoes", {"name"}, "points:1000",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+// when bad filter by clause is used in overrideoverride_json_include = {{"id", "include-rule-2"},@@ -4280,7 +4288,44 @@ TEST_F(CollectionOverrideTest, FilterPinnedHits) {ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());ASSERT_EQ("2", results["hits"][4]["document"]["id"].get<std::string>());+ // when filter does not match, we should return only curated results+ results = coll3->search("2023", {"title"}, "title: foobarbaz", {}, {},+ {0}, 50, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10,+ "", 30, 5, "",+ 10, pinned_hits, {}, {}, 3,+ "<mark>", "</mark>", {}, UINT_MAX,+ true, false, true, "",+ false, 6000 * 1000, 4, 7,+ fallback, 4, {off}, INT16_MAX,+ INT16_MAX, 2, filter_curated_hits ).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("4", results["hits"][1]["document"]["id"].get<std::string>());+filter_curated_hits = true;+ // Filter should apply on curated results+ results = coll3->search("2023", {"title"}, "points: >70", {}, {},+ {0}, 50, 1, FREQUENCY,+ {false}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10,+ "", 30, 5, "",+ 10, pinned_hits, {}, {}, 3,+ "<mark>", "</mark>", {}, UINT_MAX,+ true, false, true, "",+ false, 6000 * 1000, 4, 7,+ fallback, 4, {off}, INT16_MAX,+ INT16_MAX, 2, filter_curated_hits ).get();++ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());+results = coll3->search("2023", {"title"}, "title: snapdragon", {}, {},{0}, 50, 1, FREQUENCY,{false}, Index::DROP_TOKENS_THRESHOLD,