Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix phrase query messing up facet counts.

Kishore Nallan91415a649f4a
diff --git a/include/index.h b/include/index.h
index f89b05d9..2973abe3 100644
--- a/include/index.h
+++ b/include/index.h
@@ -426,7 +426,7 @@ private:
size_t group_limit, const std::vector<std::string>& group_by_fields,
const bool group_missing_values,
const uint32_t* result_ids, size_t results_size,
- int max_facet_count, bool is_wildcard_query, bool no_filters_provided,
+ int max_facet_count, bool is_wildcard_query,
facet_index_type_t facet_index_type) const;
bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,
diff --git a/src/index.cpp b/src/index.cpp
index 8d607ec4..f6b4abbb 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1301,7 +1301,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
const bool group_missing_values,
const uint32_t* result_ids, size_t results_size,
- int max_facet_count, bool is_wildcard_query, bool no_filters_provided,
+ int max_facet_count, bool is_wildcard_no_filter_query,
facet_index_type_t facet_index_type) const {
if(results_size == 0) {
@@ -1329,7 +1329,6 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
continue;
}
- bool is_wildcard_no_filter_query = is_wildcard_query && no_filters_provided;
bool facet_value_index_exists = facet_index_v4->has_value_index(facet_field.name);
#ifdef TEST_BUILD
@@ -2422,9 +2421,11 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto is_wildcard_query = !field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&
field_query_tokens[0].q_include_tokens[0].value == "*";
+
+ // phrase queries are handled as a filtering query
+ bool is_wildcard_non_phrase_query = is_wildcard_query && field_query_tokens[0].q_phrases.empty();
bool no_filters_provided = (filter_tree_root == nullptr && !filter_result_iterator->is_valid);
-
// handle phrase searches
if (!field_query_tokens[0].q_phrases.empty()) {
@@ -2451,7 +2452,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
}
// for phrase query, parser will set field_query_tokens to "*", need to handle that
- if (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {
+ if (is_wildcard_non_phrase_query) {
if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&
sort_fields_std[0].order == sort_field_const::desc) {
@@ -3108,7 +3109,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
delete [] excluded_result_ids;
bool estimate_facets = (facet_sample_percent < 100 && all_result_ids_len > facet_sample_threshold);
- bool is_wildcard_no_filter_query = is_wildcard_query && no_filters_provided;
+ bool is_wildcard_no_filter_query = is_wildcard_non_phrase_query && no_filters_provided;
if(!facets.empty()) {
const size_t num_threads = 1;
@@ -3160,8 +3161,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,
batch_result_ids, batch_res_len, &facet_infos, max_facet_values,
- is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,
- group_missing_values,
+ is_wildcard_no_filter_query, no_filters_provided, estimate_facets,
+ facet_sample_percent, group_missing_values,
&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
&num_processed, &m_process, &cv_process, facet_index_type]() {
search_begin_us = parent_search_begin;
@@ -3172,9 +3173,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<std::pair<std::string, uint32_t>> found_docs;
do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,
facet_infos, group_limit, group_by_fields, group_missing_values,
- batch_result_ids, batch_res_len, max_facet_values,
- is_wildcard_query, no_filters_provided,
- facet_index_type);
+ batch_result_ids, batch_res_len, max_facet_values,
+ is_wildcard_no_filter_query, facet_index_type);
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
@@ -3286,7 +3286,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
max_candidates, facet_infos, facet_index_type);
do_facets(facets, facet_query, estimate_facets, facet_sample_percent,
facet_infos, group_limit, group_by_fields, group_missing_values, &included_ids_vec[0],
- included_ids_vec.size(), max_facet_values, is_wildcard_query, no_filters_provided,
+ included_ids_vec.size(), max_facet_values, is_wildcard_no_filter_query,
facet_index_type);
all_result_ids_len += curated_topster->size;
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 36b20b37..1ac287df 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -1894,6 +1894,47 @@ TEST_F(CollectionFacetingTest, RangeFacetRangeNegativeRanges) {
ASSERT_EQ("good", results["facet_counts"][0]["counts"][2]["value"]);
}
+TEST_F(CollectionFacetingTest, FacetWithPhraseSearch) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("rating", field_types::FLOAT, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("rating", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "rating").get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "The Shawshank Redemption";
+ doc["rating"] = 9.3;
+
+ coll1->add(doc.dump());
+
+ doc["id"] = "1";
+ doc["title"] = "The Godfather";
+ doc["rating"] = 9.2;
+
+ coll1->add(doc.dump());
+
+ std::vector<std::string> facets = {"title"};
+
+ nlohmann::json results = coll1->search(R"("shawshank")", {"title"}, "", facets, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,
+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},
+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,
+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,
+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, HASH).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("The Shawshank Redemption", results["facet_counts"][0]["counts"][0]["value"]);
+}
+
TEST_F(CollectionFacetingTest, SampleFacetCounts) {
nlohmann::json schema = R"({
"name": "coll1",
diff --git a/test/collection_optimized_faceting_test.cpp b/test/collection_optimized_faceting_test.cpp
index e0f3d30f..f15da086 100644
--- a/test/collection_optimized_faceting_test.cpp
+++ b/test/collection_optimized_faceting_test.cpp
@@ -198,6 +198,8 @@ TEST_F(CollectionOptimizedFacetingTest, FacetCounts) {
ASSERT_EQ(3, results["hits"].size());
ASSERT_EQ(1, results["facet_counts"].size());
+ LOG(INFO) << results["facet_counts"][0];
+
ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
@@ -2382,6 +2384,47 @@ TEST_F(CollectionOptimizedFacetingTest, ValueIndexStatsMinMax) {
ASSERT_FLOAT_EQ(2, results["facet_counts"][0]["stats"]["total_values"].get<size_t>());
}
+TEST_F(CollectionOptimizedFacetingTest, FacetWithPhraseSearch) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("rating", field_types::FLOAT, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("rating", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "rating").get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "The Shawshank Redemption";
+ doc["rating"] = 9.3;
+
+ coll1->add(doc.dump());
+
+ doc["id"] = "1";
+ doc["title"] = "The Godfather";
+ doc["rating"] = 9.2;
+
+ coll1->add(doc.dump());
+
+ std::vector<std::string> facets = {"title"};
+
+ nlohmann::json results = coll1->search(R"("shawshank")", {"title"}, "", facets, sort_fields, {0}, 10, 1,
+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 2,"", 30UL, 4UL,
+ "", 1UL, "", "", {}, 3UL, "<mark>", "</mark>", {},
+ 4294967295UL, true, false, true, "", false, 6000000UL, 4UL,
+ 7UL, fallback, 4UL, {off}, 32767UL, 32767UL, 2UL, 2UL, false,
+ "", true, 0UL, max_score, 100UL, 0UL, 4294967295UL, VALUE).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("The Shawshank Redemption", results["facet_counts"][0]["counts"][0]["value"]);
+}
+
TEST_F(CollectionOptimizedFacetingTest, StringFacetsCountListOrderTest) {
//check if count list is ordering facets
Collection *coll1;