Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Return all included overrides if provided `filter_by` produces no match. (#1861)

Harpreet Sangarfc153ae19283
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index d35744cc..08bbb5a7 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -425,4 +425,8 @@ public:
[[nodiscard]] bool _get_is_equals_iterator_valid() const {
return is_equals_iterator_valid;
}
+
+ inline bool is_filter_provided() const {
+ return filter_node != nullptr;
+ }
};
diff --git a/src/index.cpp b/src/index.cpp
index f958cc00..6e240e9d 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2756,10 +2756,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
return filter_init_op;
}
- if (filter_tree_root != nullptr && filter_result_iterator->validity != filter_result_iterator_t::valid) {
- return Option(true);
- }
-
#ifdef TEST_BUILD
if (filter_result_iterator->approx_filter_ids_length > 20) {
@@ -2789,6 +2785,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());
std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());
+ bool const& filter_by_provided = filter_tree_root != nullptr;
+ bool const& no_filter_by_matches = filter_by_provided && filter_result_iterator->approx_filter_ids_length == 0;
+
+ // If curation is not involved and there are no filter matches, return early.
+ if (curated_ids_sorted.empty() && no_filter_by_matches) {
+ return Option(true);
+ }
+
// Order of `fields` are used to sort results
// auto begin = std::chrono::high_resolution_clock::now();
uint32_t* all_result_ids = nullptr;
@@ -2816,8 +2820,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
// phrase queries are handled as a filtering query
bool is_wildcard_non_phrase_query = is_wildcard_query && field_query_tokens[0].q_phrases.empty();
-
- bool no_filters_provided = (filter_tree_root == nullptr && !filter_result_iterator->validity == filter_result_iterator_t::valid);
// handle phrase searches
if (!field_query_tokens[0].q_phrases.empty()) {
@@ -2845,7 +2847,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
// for phrase query, parser will set field_query_tokens to "*", need to handle that
if (is_wildcard_non_phrase_query) {
- if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
+ if(!filter_by_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&
sort_fields_std[0].order == sort_field_const::desc) {
// optimize for this path specifically
@@ -2917,7 +2919,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
filter_result_iterator->compute_iterators();
uint32_t filter_id_count = filter_result_iterator->approx_filter_ids_length;
- if (!no_filters_provided && filter_id_count < vector_query.flat_search_cutoff) {
+ if (filter_by_provided && filter_id_count < vector_query.flat_search_cutoff) {
while (filter_result_iterator->validity == filter_result_iterator_t::valid) {
auto seq_id = filter_result_iterator->seq_id;
auto filter_result = single_filter_result_t(seq_id, std::move(filter_result_iterator->reference));
@@ -2948,7 +2950,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
filter_result_iterator->reset();
search_cutoff = search_cutoff || filter_result_iterator->validity == filter_result_iterator_t::timed_out;
- if(no_filters_provided ||
+ if(!filter_by_provided ||
(filter_id_count >= vector_query.flat_search_cutoff && filter_result_iterator->validity == filter_result_iterator_t::valid)) {
dist_results.clear();
@@ -3060,7 +3062,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
} else {
// if filters were not provided, use the seq_ids index to generate the list of all document ids
- if (no_filters_provided) {
+ if (!filter_by_provided) {
filter_result_iterator = new filter_result_iterator_t(seq_ids->uncompress(), seq_ids->num_ids(),
search_begin_us, search_stop_us);
filter_iterator_guard.reset(filter_result_iterator);
@@ -3428,7 +3430,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto& vec_result = vec_results[res_index];
auto seq_id = vec_result.first;
- if (!no_filters_provided && filter_result_iterator->is_valid(seq_id) != 1) {
+ if (filter_by_provided && filter_result_iterator->is_valid(seq_id) != 1) {
continue;
}
auto references = std::move(filter_result_iterator->reference);
@@ -3544,7 +3546,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
bool estimate_facets = (facet_sample_percent > 0 && facet_sample_percent < 100 &&
all_result_ids_len > facet_sample_threshold);
- bool is_wildcard_no_filter_query = is_wildcard_non_phrase_query && no_filters_provided && vector_query.field_name.empty();
+ bool is_wildcard_no_filter_query = is_wildcard_non_phrase_query && !filter_by_provided && vector_query.field_name.empty();
if(!facets.empty()) {
const size_t num_threads = std::min(concurrency, all_result_ids_len);
@@ -3931,6 +3933,12 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
bool enable_typos_for_numerical_tokens,
bool enable_typos_for_alpha_numerical_tokens) const {
+ // Return early in case filter_by is provided but it matches no docs.
+ if (filter_result_iterator != nullptr && filter_result_iterator->is_filter_provided() &&
+ filter_result_iterator->approx_filter_ids_length == 0) {
+ return Option<bool>(true);
+ }
+
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index e9bb2610..3d684ac0 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -2154,7 +2154,7 @@ uint32_t posting_list_t::iterator_t::get_field_id() const {
}
bool result_iter_state_t::is_filter_provided() const {
- return filter_ids_length > 0 || (fit != nullptr && fit->approx_filter_ids_length > 0);
+ return filter_ids_length > 0 || (fit != nullptr && fit->is_filter_provided());
}
bool result_iter_state_t::is_filter_valid() const {
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 54ad410d..c9a87d62 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -78,6 +78,9 @@ TEST_F(CollectionFilteringTest, FilterOnTextFields) {
results = coll_array_fields->search("Jeremy", query_fields, "tags : fine PLATINUM", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
+ results = coll_array_fields->search("Jeremy", query_fields, "tags : foobarbaz", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(0, results["hits"].size());
+
// using just ":", filtering should return documents that contain ALL tokens in the filter expression
results = coll_array_fields->search("Jeremy", query_fields, "tags : PLATINUM", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index ea5c7abf..2d14421e 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -786,6 +786,14 @@ TEST_F(CollectionOverrideTest, IncludeOverrideWithFilterBy) {
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+ // when filter by does not match any result, curated result should still show up
+ // because `filter_curated_hits` is false
+ results = coll1->search("shoes", {"name"}, "points:1000",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
// when bad filter by clause is used in override
override_json_include = {
{"id", "include-rule-2"},
@@ -4280,7 +4288,44 @@ TEST_F(CollectionOverrideTest, FilterPinnedHits) {
ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][4]["document"]["id"].get<std::string>());
+ // when filter does not match, we should return only curated results
+ results = coll3->search("2023", {"title"}, "title: foobarbaz", {}, {},
+ {0}, 50, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10,
+ "", 30, 5, "",
+ 10, pinned_hits, {}, {}, 3,
+ "<mark>", "</mark>", {}, UINT_MAX,
+ true, false, true, "",
+ false, 6000 * 1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX,
+ INT16_MAX, 2, filter_curated_hits ).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("4", results["hits"][1]["document"]["id"].get<std::string>());
+
filter_curated_hits = true;
+ // Filter should apply on curated results
+ results = coll3->search("2023", {"title"}, "points: >70", {}, {},
+ {0}, 50, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10,
+ "", 30, 5, "",
+ 10, pinned_hits, {}, {}, 3,
+ "<mark>", "</mark>", {}, UINT_MAX,
+ true, false, true, "",
+ false, 6000 * 1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX,
+ INT16_MAX, 2, filter_curated_hits ).get();
+
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+
results = coll3->search("2023", {"title"}, "title: snapdragon", {}, {},
{0}, 50, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,