Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add flag for not keeping missing group field values distinct.

Kishore Nalland5d9fcad1de1
diff --git a/include/collection.h b/include/collection.h
index 38187eeb..6e20beb6 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -468,7 +468,8 @@ public:
const size_t page_offset = 0,
const size_t remote_embedding_timeout_ms = 30000,
const size_t remote_embedding_num_tries = 2,
- const bool prioritize_num_matching_fields = true) const;
+ const bool prioritize_num_matching_fields = true,
+ const bool group_missing_values = true) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --git a/include/index.h b/include/index.h
index 8e165e74..9ce88660 100644
--- a/include/index.h
+++ b/include/index.h
@@ -116,6 +116,7 @@ struct search_args {
size_t typo_tokens_threshold;
std::vector<std::string> group_by_fields;
size_t group_limit;
+ bool group_missing_values;
std::string default_sorting_field;
bool prioritize_exact_match;
bool prioritize_token_position;
@@ -154,6 +155,7 @@ struct search_args {
size_t max_facet_values, size_t max_hits, size_t per_page, size_t offset, token_ordering token_order,
const std::vector<bool>& prefixes, size_t drop_tokens_threshold, size_t typo_tokens_threshold,
const std::vector<std::string>& group_by_fields, size_t group_limit,
+ const bool group_missing_values,
const string& default_sorting_field, bool prioritize_exact_match,
const bool prioritize_token_position, const bool prioritize_num_matching_fields, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms,
@@ -167,7 +169,9 @@ struct search_args {
facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),
offset(offset), token_order(token_order), prefixes(prefixes),
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
- group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),
+ group_by_fields(group_by_fields), group_limit(group_limit),
+ group_missing_values(group_missing_values),
+ default_sorting_field(default_sorting_field),
prioritize_exact_match(prioritize_exact_match), prioritize_token_position(prioritize_token_position),
prioritize_num_matching_fields(prioritize_num_matching_fields),
all_result_ids_len(0), exhaustive_search(exhaustive_search), concurrency(concurrency),
@@ -369,6 +373,7 @@ private:
bool estimate_facets, size_t facet_sample_percent,
const std::vector<facet_info_t>& facet_infos,
size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const uint32_t* result_ids, size_t results_size) const;
bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,
@@ -403,6 +408,7 @@ private:
size_t& field_num_results,
size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
bool prioritize_exact_match,
size_t concurrency,
std::set<uint64>& query_hashes,
@@ -432,6 +438,7 @@ private:
const size_t typo_tokens_threshold,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const std::vector<token_t>& query_tokens,
const std::vector<uint32_t>& num_typos,
const std::vector<bool>& prefixes,
@@ -460,6 +467,7 @@ private:
size_t& field_num_results,
const size_t typo_tokens_threshold,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const std::vector<token_t>& query_tokens,
bool prioritize_exact_match,
bool exhaustive_search,
@@ -604,7 +612,9 @@ public:
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
const size_t group_limit,
- const std::vector<std::string> &group_by_fields, uint32_t token_bits,
+ const std::vector<std::string> &group_by_fields,
+ const bool group_missing_values,
+ uint32_t token_bits,
bool prioritize_exact_match,
bool single_exact_query_token,
int syn_orig_num_tokens,
@@ -627,7 +637,8 @@ public:
static float int64_t_to_float(int64_t n);
- uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;
+ uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields,
+ const uint32_t seq_id, const bool group_missing_values) const;
static void compute_token_offsets_facets(index_record& record,
const tsl::htrie_map<char, field>& search_schema,
@@ -659,6 +670,7 @@ public:
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const string& default_sorting_field, bool prioritize_exact_match,
const bool prioritize_token_position, const bool prioritize_num_matching_fields,
bool exhaustive_search,
@@ -744,7 +756,9 @@ public:
const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
+ const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
+ const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
@@ -786,7 +800,8 @@ public:
const std::vector<enable_t>& infixes,
const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
+ const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values, const size_t max_extra_prefix,
const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,
const uint32_t *filter_ids, size_t filter_ids_length,
const int sort_order[3],
@@ -804,7 +819,9 @@ public:
const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
const token_ordering& token_order,
const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
+ bool prioritize_exact_match,
const bool prioritize_token_position,
const bool prioritize_num_matching_fields,
const bool exhaustive_search, const size_t concurrency,
@@ -832,6 +849,7 @@ public:
const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
Topster* actual_topster,
const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
@@ -863,6 +881,7 @@ public:
Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t*& all_result_ids, size_t& all_result_ids_len,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
bool prioritize_exact_match,
const bool prioritize_token_position,
const bool prioritize_num_matching_fields,
@@ -903,6 +922,7 @@ public:
const std::vector<token_t>& dropped_tokens,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
bool prioritize_exact_match,
const bool search_all_candidates,
const bool prioritize_num_matching_fields,
@@ -958,7 +978,8 @@ public:
void
process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
- const size_t group_limit, const bool filter_curated_hits, const uint32_t* filter_ids,
+ const size_t group_limit, const bool group_missing_values,
+ const bool filter_curated_hits, const uint32_t* filter_ids,
uint32_t filter_ids_length, std::set<uint32_t>& curated_ids,
std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
std::vector<uint32_t>& included_ids_vec,
diff --git a/src/collection.cpp b/src/collection.cpp
index c0b3e6f3..3f9f90a6 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1115,7 +1115,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const size_t page_offset,
const size_t remote_embedding_timeout_ms,
const size_t remote_embedding_num_tries,
- const bool prioritize_num_matching_fields) const {
+ const bool prioritize_num_matching_fields,
+ const bool group_missing_values) const {
std::shared_lock lock(mutex);
@@ -1607,7 +1608,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
per_page, offset, token_order, prefixes,
drop_tokens_threshold, typo_tokens_threshold,
- group_by_fields, group_limit, default_sorting_field,
+ group_by_fields, group_limit, group_missing_values,
+ default_sorting_field,
prioritize_exact_match, prioritize_token_position,
prioritize_num_matching_fields,
exhaustive_search, 4,
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 6b52046f..657a906a 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -701,6 +701,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *GROUP_BY = "group_by";
const char *GROUP_LIMIT = "group_limit";
+ const char *GROUP_MISSING_VALUES = "group_missing_values";
const char *LIMIT_HITS = "limit_hits";
const char *PER_PAGE = "per_page";
@@ -835,6 +836,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::string hidden_hits_str;
std::vector<std::string> group_by_fields;
size_t group_limit = 3;
+ bool group_missing_values = true;
std::string highlight_start_tag = "<mark>";
std::string highlight_end_tag = "</mark>";
std::vector<uint32_t> query_by_weights;
@@ -909,6 +911,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
{ENABLE_OVERRIDES, &enable_overrides},
{ENABLE_HIGHLIGHT_V1, &enable_highlight_v1},
{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},
+ {GROUP_MISSING_VALUES, &group_missing_values},
};
std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {
@@ -1103,7 +1106,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
offset,
remote_embedding_timeout_ms,
remote_embedding_num_tries,
- prioritize_num_matching_fields
+ prioritize_num_matching_fields,
+ group_missing_values
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/src/index.cpp b/src/index.cpp
index e2eb5c99..cae1eb6b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1168,6 +1168,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
bool estimate_facets, size_t facet_sample_percent,
const std::vector<facet_info_t>& facet_infos,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const uint32_t* result_ids, size_t results_size) const {
// assumed that facet fields have already been validated upstream
@@ -1222,7 +1223,8 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
fhash = single_facet_map_it->second;
}
- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
+ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields,
+ doc_seq_id, group_missing_values) : 0;
if(((i + 1) % 16384) == 0) {
RETURN_CIRCUIT_BREAKER
@@ -1311,6 +1313,7 @@ void Index::search_all_candidates(const size_t num_search_fields,
const size_t typo_tokens_threshold,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const std::vector<token_t>& query_tokens,
const std::vector<uint32_t>& num_typos,
const std::vector<bool>& prefixes,
@@ -1366,6 +1369,7 @@ void Index::search_all_candidates(const size_t num_search_fields,
search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields, match_type,
sort_fields, topster,groups_processed,
searched_queries, qtoken_set, dropped_tokens, group_limit, group_by_fields,
+ group_missing_values,
prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,
filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,
exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
@@ -1390,6 +1394,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
const size_t typo_tokens_threshold,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const std::vector<token_t>& query_tokens,
bool prioritize_exact_match,
const bool exhaustive_search,
@@ -1476,7 +1481,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
total_cost, topster, query_suggestion, groups_processed,
seq_id, sort_order, field_values, geopoint_indices,
- group_limit, group_by_fields, token_bits,
+ group_limit, group_by_fields, group_missing_values, token_bits,
prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);
id_buff.push_back(seq_id);
@@ -2271,7 +2276,9 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& co
search_params->qtoken_set,
search_params->raw_result_kvs, search_params->override_result_kvs,
search_params->typo_tokens_threshold,
- search_params->group_limit, search_params->group_by_fields,
+ search_params->group_limit,
+ search_params->group_by_fields,
+ search_params->group_missing_values,
search_params->default_sorting_field,
search_params->prioritize_exact_match,
search_params->prioritize_token_position,
@@ -2589,7 +2596,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&
search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it.value(), field_name,
nullptr, 0, {}, sort_fields, -1, 0, searched_queries, topster, groups_processed,
&result_ids, result_ids_len, field_num_results, 0, group_by_fields,
- false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4);
+ true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4);
if(result_ids_len != 0) {
// we need to narraw onto the exact matches
@@ -2734,6 +2741,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const string& default_sorting_field, bool prioritize_exact_match,
const bool prioritize_token_position, const bool prioritize_num_matching_fields, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,
@@ -2768,7 +2776,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<uint32_t> included_ids_vec;
std::unordered_set<uint32_t> excluded_group_ids;
- process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, filter_curated_hits,
+ process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, group_missing_values,
+ filter_curated_hits,
filter_result.docs, filter_result.count, curated_ids, included_ids_map,
included_ids_vec, excluded_group_ids);
@@ -2804,7 +2813,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
// handle phrase searches
if (!field_query_tokens[0].q_phrases.empty()) {
do_phrase_search(num_search_fields, the_fields, field_query_tokens,
- sort_fields_std, searched_queries, group_limit, group_by_fields,
+ sort_fields_std, searched_queries, group_limit, group_by_fields, group_missing_values,
topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,
all_result_ids, all_result_ids_len, groups_processed, curated_ids,
excluded_result_ids, excluded_result_ids_size, excluded_group_ids, curated_topster,
@@ -2830,7 +2839,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
uint32_t seq_id = it.id();
uint64_t distinct_id = seq_id;
if (group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
continue;
}
@@ -2930,7 +2939,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
uint64_t distinct_id = seq_id;
if (group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
continue;
}
@@ -2967,7 +2976,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
} else {
search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,
curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
- curated_ids, curated_ids_sorted,
+ group_missing_values, curated_ids, curated_ids_sorted,
excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
all_result_ids, all_result_ids_len,
filter_result.docs, filter_result.count, concurrency,
@@ -3015,7 +3024,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,
excluded_group_ids,
sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,
+ group_missing_values, prioritize_exact_match,
prioritize_token_position, prioritize_num_matching_fields, query_hashes, token_order, prefixes,
typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,
@@ -3053,7 +3063,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,
excluded_group_ids,
sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,
+ group_missing_values, prioritize_exact_match,
prioritize_token_position, prioritize_num_matching_fields,
query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,
@@ -3064,7 +3075,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
// do synonym based searches
do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map, sort_fields_std,
curated_topster, token_order,
- 0, group_limit, group_by_fields, prioritize_exact_match, prioritize_token_position,
+ 0, group_limit, group_by_fields, group_missing_values,
+ prioritize_exact_match, prioritize_token_position,
prioritize_num_matching_fields,
exhaustive_search, concurrency, prefixes,
min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
@@ -3126,6 +3138,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
curated_ids_sorted, excluded_group_ids, sort_fields_std, num_typos, searched_queries,
qtoken_set, topster, groups_processed,
all_result_ids, all_result_ids_len, group_limit, group_by_fields,
+ group_missing_values,
prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,
query_hashes, token_order, prefixes, typo_tokens_threshold,
exhaustive_search, max_candidates, min_len_1typo,
@@ -3139,7 +3152,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std, searched_queries,
- group_limit, group_by_fields,
+ group_limit, group_by_fields, group_missing_values,
max_extra_prefix, max_extra_suffix,
field_query_tokens[0].q_include_tokens,
topster, filter_result.docs, filter_result.count,
@@ -3247,7 +3260,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
uint64_t distinct_id = seq_id;
if (group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
continue;
}
@@ -3326,7 +3339,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,
batch_result_ids, batch_res_len, &facet_infos,
- estimate_facets, facet_sample_percent,
+ estimate_facets, facet_sample_percent, group_missing_values,
&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
&num_processed, &m_process, &cv_process]() {
search_begin_us = parent_search_begin;
@@ -3335,7 +3348,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto fq = facet_query;
do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,
- facet_infos, group_limit, group_by_fields,
+ facet_infos, group_limit, group_by_fields, group_missing_values,
batch_result_ids, batch_res_len);
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
@@ -3412,7 +3425,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
compute_facet_infos(facets, facet_query, facet_query_num_typos,
&included_ids_vec[0], included_ids_vec.size(), group_by_fields, max_candidates, facet_infos);
do_facets(facets, facet_query, estimate_facets, facet_sample_percent,
- facet_infos, group_limit, group_by_fields, &included_ids_vec[0], included_ids_vec.size());
+ facet_infos, group_limit, group_by_fields, group_missing_values,
+ &included_ids_vec[0], included_ids_vec.size());
all_result_ids_len += curated_topster->size;
@@ -3428,6 +3442,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids,
const std::vector<std::string>& group_by_fields, const size_t group_limit,
+ const bool group_missing_values,
const bool filter_curated_hits, const uint32_t* filter_ids, uint32_t filter_ids_length,
std::set<uint32_t>& curated_ids,
std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
@@ -3441,7 +3456,7 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>
if(group_limit != 0) {
// if one `id` of a group is present in curated hits, we have to exclude that entire group from results
for(auto seq_id: included_ids_vec) {
- uint64_t distinct_id = get_distinct_id(group_by_fields, seq_id);
+ uint64_t distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
excluded_group_ids.emplace(distinct_id);
}
}
@@ -3531,6 +3546,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t*& all_result_ids, size_t & all_result_ids_len,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
bool prioritize_exact_match,
const bool prioritize_token_position,
const bool prioritize_num_matching_fields,
@@ -3777,7 +3793,8 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
sort_fields, token_candidates_vec, searched_queries, qtoken_set,
dropped_tokens, topster,
groups_processed, all_result_ids, all_result_ids_len,
- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
+ typo_tokens_threshold, group_limit, group_by_fields,
+ group_missing_values, query_tokens,
num_typos, prefixes, prioritize_exact_match, prioritize_token_position,
prioritize_num_matching_fields, exhaustive_search, max_candidates,
syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
@@ -3932,6 +3949,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
const std::vector<token_t>& dropped_tokens,
const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const bool prioritize_exact_match,
const bool prioritize_token_position,
const bool prioritize_num_matching_fields,
@@ -4142,7 +4160,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
return;
}
@@ -4472,6 +4490,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
Topster* actual_topster,
const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
@@ -4623,7 +4642,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
continue;
}
@@ -4650,7 +4669,9 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
const token_ordering& token_order,
const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
+ bool prioritize_exact_match,
const bool prioritize_token_position,
const bool prioritize_num_matching_fields,
const bool exhaustive_search, const size_t concurrency,
@@ -4678,7 +4699,8 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,
exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted, excluded_group_ids,
sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,
+ group_missing_values, prioritize_exact_match,
prioritize_token_position, prioritize_num_matching_fields,
query_hashes, token_order, prefixes, typo_tokens_threshold,
exhaustive_search, max_candidates, min_len_1typo,
@@ -4692,7 +4714,9 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
const std::vector<enable_t>& infixes,
const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
+ const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
+ const size_t max_extra_prefix,
const size_t max_extra_suffix,
const std::vector<token_t>& query_tokens, Topster* actual_topster,
const uint32_t *filter_ids, size_t filter_ids_length,
@@ -4758,7 +4782,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
continue;
}
@@ -4914,7 +4938,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
facet_field, facet_field.faceted_name(),
all_result_ids, all_result_ids_len, {}, sort_fields, -1, facet_query_num_typos, searched_queries, topster,
groups_processed, &field_result_ids, field_result_ids_len, field_num_results, 0, group_by_fields,
- false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates);
+ true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates);
//LOG(INFO) << "searched_queries.size: " << searched_queries.size();
@@ -5022,7 +5046,9 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
+ const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
+ const std::set<uint32_t>& curated_ids,
const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
@@ -5071,7 +5097,8 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
thread_id, &sort_fields, &searched_queries,
- &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,
+ &group_limit, &group_by_fields, group_missing_values,
+ &topsters, &tgroups_processed, &excluded_group_ids,
&sort_order, field_values, &geopoint_indices, &plists,
check_for_circuit_break,
batch_result_ids, batch_res_len,
@@ -5098,7 +5125,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
continue;
}
@@ -5206,6 +5233,7 @@ void Index::search_field(const uint8_t & field_id,
Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
bool prioritize_exact_match,
const size_t concurrency,
std::set<uint64>& query_hashes,
@@ -5337,7 +5365,8 @@ void Index::search_field(const uint8_t & field_id,
exclude_token_ids, exclude_token_ids_size,
curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,
groups_processed, all_result_ids, all_result_ids_len, field_num_results,
- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
+ typo_tokens_threshold, group_limit, group_by_fields,
+ group_missing_values, query_tokens,
prioritize_exact_match, exhaustive_search, syn_orig_num_tokens,
concurrency, query_hashes, id_buff);
@@ -5508,6 +5537,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
const uint32_t token_bits,
const bool prioritize_exact_match,
const bool single_exact_query_token,
@@ -5700,7 +5730,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
}
//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;
@@ -5716,8 +5746,9 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
// pre-filter group_by_fields such that we can avoid the find() check
uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
- const uint32_t seq_id) const {
- uint64_t distinct_id = 1; // some constant initial value
+ const uint32_t seq_id, const bool group_missing_values) const {
+ uint64_t distinct_id = 1;
+ bool found_group_value = false;
// calculate hash from group_by_fields
for(const auto& field: group_by_fields) {
@@ -5740,6 +5771,7 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
for(size_t i = 0; i < facet_hashes.size(); i++) {
distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);
+ found_group_value = true;
}
} else {
const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
@@ -5750,12 +5782,12 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
}
const auto& facet_hash = facet_hashes_it->second;
-
distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
+ found_group_value = true;
}
}
- return distinct_id;
+ return (!found_group_value && !group_missing_values) ? seq_id : distinct_id;
}
inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token_candidates_vec,
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 154e7555..8a4a3330 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -606,6 +606,77 @@ TEST_F(CollectionGroupingTest, RepeatedFieldNameGroupHitCount) {
ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());
}
+TEST_F(CollectionGroupingTest, ControlMissingValues) {
+ std::vector<field> fields = {
+ field("brand", field_types::STRING, true, true),
+ };
+
+ Collection* coll2 = collectionManager.get_collection("coll2").get();
+ if(coll2 == nullptr) {
+ coll2 = collectionManager.create_collection("coll2", 1, fields).get();
+ }
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["brand"] = "Omega";
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["brand"] = nullptr;
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "2";
+ doc["brand"] = nullptr;
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "3";
+ doc["brand"] = "Omega";
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ // disable null value aggregation
+
+ auto res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
+ 100, 0, 0, 30000, 2, true, false).get();
+
+ ASSERT_EQ(3, res["grouped_hits"].size());
+ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());
+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
+ ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
+ ASSERT_EQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ(0, res["grouped_hits"][2]["group_key"].size());
+ ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());
+ ASSERT_EQ("1", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>());
+
+ // with null value aggregation (default)
+ res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
+ 100, 0, 0, 30000, 2, true, true).get();
+
+ ASSERT_EQ(2, res["grouped_hits"].size());
+
+ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());
+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
+ ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());
+}
+
TEST_F(CollectionGroupingTest, SortingOnGroupCount) {
std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};