Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add flag for not keeping missing group field values distinct.
Kishore Nalland5d9fcad1de1
diff --git a/include/collection.h b/include/collection.hindex 38187eeb..6e20beb6 100644--- a/include/collection.h+++ b/include/collection.h@@ -468,7 +468,8 @@ public:const size_t page_offset = 0,const size_t remote_embedding_timeout_ms = 30000,const size_t remote_embedding_num_tries = 2,- const bool prioritize_num_matching_fields = true) const;+ const bool prioritize_num_matching_fields = true,+ const bool group_missing_values = true) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/index.h b/include/index.hindex 8e165e74..9ce88660 100644--- a/include/index.h+++ b/include/index.h@@ -116,6 +116,7 @@ struct search_args {size_t typo_tokens_threshold;std::vector<std::string> group_by_fields;size_t group_limit;+ bool group_missing_values;std::string default_sorting_field;bool prioritize_exact_match;bool prioritize_token_position;@@ -154,6 +155,7 @@ struct search_args {size_t max_facet_values, size_t max_hits, size_t per_page, size_t offset, token_ordering token_order,const std::vector<bool>& prefixes, size_t drop_tokens_threshold, size_t typo_tokens_threshold,const std::vector<std::string>& group_by_fields, size_t group_limit,+ const bool group_missing_values,const string& default_sorting_field, bool prioritize_exact_match,const bool prioritize_token_position, const bool prioritize_num_matching_fields, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms,@@ -167,7 +169,9 @@ struct search_args {facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),offset(offset), token_order(token_order), prefixes(prefixes),drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),- group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),+ group_by_fields(group_by_fields), group_limit(group_limit),+ group_missing_values(group_missing_values),+ default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), prioritize_token_position(prioritize_token_position),prioritize_num_matching_fields(prioritize_num_matching_fields),all_result_ids_len(0), exhaustive_search(exhaustive_search), concurrency(concurrency),@@ -369,6 +373,7 @@ private:bool estimate_facets, size_t facet_sample_percent,const std::vector<facet_info_t>& facet_infos,size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const uint32_t* result_ids, size_t results_size) const;bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,@@ -403,6 +408,7 @@ private:size_t& field_num_results,size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,bool prioritize_exact_match,size_t concurrency,std::set<uint64>& query_hashes,@@ -432,6 +438,7 @@ private:const size_t typo_tokens_threshold,const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const std::vector<token_t>& query_tokens,const std::vector<uint32_t>& num_typos,const std::vector<bool>& prefixes,@@ -460,6 +467,7 @@ private:size_t& field_num_results,const size_t typo_tokens_threshold,const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const std::vector<token_t>& query_tokens,bool prioritize_exact_match,bool exhaustive_search,@@ -604,7 +612,9 @@ public:std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const size_t group_limit,- const std::vector<std::string> &group_by_fields, uint32_t token_bits,+ const std::vector<std::string> &group_by_fields,+ const bool group_missing_values,+ uint32_t token_bits,bool prioritize_exact_match,bool single_exact_query_token,int syn_orig_num_tokens,@@ -627,7 +637,8 @@ public:static float int64_t_to_float(int64_t n);- uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;+ uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields,+ const uint32_t seq_id, const bool group_missing_values) const;static void compute_token_offsets_facets(index_record& record,const tsl::htrie_map<char, field>& search_schema,@@ -659,6 +670,7 @@ public:std::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,const size_t typo_tokens_threshold, const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const string& default_sorting_field, bool prioritize_exact_match,const bool prioritize_token_position, const bool prioritize_num_matching_fields,bool exhaustive_search,@@ -744,7 +756,9 @@ public:const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,+ const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,+ const std::set<uint32_t>& curated_ids,const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,@@ -786,7 +800,8 @@ public:const std::vector<enable_t>& infixes,const std::vector<sort_by>& sort_fields,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,+ const std::vector<std::string>& group_by_fields,+ const bool group_missing_values, const size_t max_extra_prefix,const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,const uint32_t *filter_ids, size_t filter_ids_length,const int sort_order[3],@@ -804,7 +819,9 @@ public:const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,const token_ordering& token_order,const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,+ bool prioritize_exact_match,const bool prioritize_token_position,const bool prioritize_num_matching_fields,const bool exhaustive_search, const size_t concurrency,@@ -832,6 +849,7 @@ public:const std::vector<sort_by>& sort_fields,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,Topster* actual_topster,const int sort_order[3],std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,@@ -863,6 +881,7 @@ public:Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,uint32_t*& all_result_ids, size_t& all_result_ids_len,const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,bool prioritize_exact_match,const bool prioritize_token_position,const bool prioritize_num_matching_fields,@@ -903,6 +922,7 @@ public:const std::vector<token_t>& dropped_tokens,const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,bool prioritize_exact_match,const bool search_all_candidates,const bool prioritize_num_matching_fields,@@ -958,7 +978,8 @@ public:voidprocess_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,- const size_t group_limit, const bool filter_curated_hits, const uint32_t* filter_ids,+ const size_t group_limit, const bool group_missing_values,+ const bool filter_curated_hits, const uint32_t* filter_ids,uint32_t filter_ids_length, std::set<uint32_t>& curated_ids,std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,std::vector<uint32_t>& included_ids_vec,diff --git a/src/collection.cpp b/src/collection.cppindex c0b3e6f3..3f9f90a6 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1115,7 +1115,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const size_t page_offset,const size_t remote_embedding_timeout_ms,const size_t remote_embedding_num_tries,- const bool prioritize_num_matching_fields) const {+ const bool prioritize_num_matching_fields,+ const bool group_missing_values) const {std::shared_lock lock(mutex);@@ -1607,7 +1608,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,per_page, offset, token_order, prefixes,drop_tokens_threshold, typo_tokens_threshold,- group_by_fields, group_limit, default_sorting_field,+ group_by_fields, group_limit, group_missing_values,+ default_sorting_field,prioritize_exact_match, prioritize_token_position,prioritize_num_matching_fields,exhaustive_search, 4,diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 6b52046f..657a906a 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -701,6 +701,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *GROUP_BY = "group_by";const char *GROUP_LIMIT = "group_limit";+ const char *GROUP_MISSING_VALUES = "group_missing_values";const char *LIMIT_HITS = "limit_hits";const char *PER_PAGE = "per_page";@@ -835,6 +836,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::string hidden_hits_str;std::vector<std::string> group_by_fields;size_t group_limit = 3;+ bool group_missing_values = true;std::string highlight_start_tag = "<mark>";std::string highlight_end_tag = "</mark>";std::vector<uint32_t> query_by_weights;@@ -909,6 +911,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{ENABLE_OVERRIDES, &enable_overrides},{ENABLE_HIGHLIGHT_V1, &enable_highlight_v1},{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},+ {GROUP_MISSING_VALUES, &group_missing_values},};std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {@@ -1103,7 +1106,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reoffset,remote_embedding_timeout_ms,remote_embedding_num_tries,- prioritize_num_matching_fields+ prioritize_num_matching_fields,+ group_missing_values);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex e2eb5c99..cae1eb6b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1168,6 +1168,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,bool estimate_facets, size_t facet_sample_percent,const std::vector<facet_info_t>& facet_infos,const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const uint32_t* result_ids, size_t results_size) const {// assumed that facet fields have already been validated upstream@@ -1222,7 +1223,8 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,fhash = single_facet_map_it->second;}- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;+ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields,+ doc_seq_id, group_missing_values) : 0;if(((i + 1) % 16384) == 0) {RETURN_CIRCUIT_BREAKER@@ -1311,6 +1313,7 @@ void Index::search_all_candidates(const size_t num_search_fields,const size_t typo_tokens_threshold,const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const std::vector<token_t>& query_tokens,const std::vector<uint32_t>& num_typos,const std::vector<bool>& prefixes,@@ -1366,6 +1369,7 @@ void Index::search_all_candidates(const size_t num_search_fields,search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields, match_type,sort_fields, topster,groups_processed,searched_queries, qtoken_set, dropped_tokens, group_limit, group_by_fields,+ group_missing_values,prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,exclude_token_ids, exclude_token_ids_size, excluded_group_ids,@@ -1390,6 +1394,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,const size_t typo_tokens_threshold,const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const std::vector<token_t>& query_tokens,bool prioritize_exact_match,const bool exhaustive_search,@@ -1476,7 +1481,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,score_results(sort_fields, searched_queries.size(), field_id, field_is_array,total_cost, topster, query_suggestion, groups_processed,seq_id, sort_order, field_values, geopoint_indices,- group_limit, group_by_fields, token_bits,+ group_limit, group_by_fields, group_missing_values, token_bits,prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);id_buff.push_back(seq_id);@@ -2271,7 +2276,9 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->qtoken_set,search_params->raw_result_kvs, search_params->override_result_kvs,search_params->typo_tokens_threshold,- search_params->group_limit, search_params->group_by_fields,+ search_params->group_limit,+ search_params->group_by_fields,+ search_params->group_missing_values,search_params->default_sorting_field,search_params->prioritize_exact_match,search_params->prioritize_token_position,@@ -2589,7 +2596,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it.value(), field_name,nullptr, 0, {}, sort_fields, -1, 0, searched_queries, topster, groups_processed,&result_ids, result_ids_len, field_num_results, 0, group_by_fields,- false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4);+ true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4);if(result_ids_len != 0) {// we need to narraw onto the exact matches@@ -2734,6 +2741,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<std::vector<KV*>>& raw_result_kvs, std::vector<std::vector<KV*>>& override_result_kvs,const size_t typo_tokens_threshold, const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const string& default_sorting_field, bool prioritize_exact_match,const bool prioritize_token_position, const bool prioritize_num_matching_fields, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,@@ -2768,7 +2776,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<uint32_t> included_ids_vec;std::unordered_set<uint32_t> excluded_group_ids;- process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, filter_curated_hits,+ process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, group_missing_values,+ filter_curated_hits,filter_result.docs, filter_result.count, curated_ids, included_ids_map,included_ids_vec, excluded_group_ids);@@ -2804,7 +2813,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons// handle phrase searchesif (!field_query_tokens[0].q_phrases.empty()) {do_phrase_search(num_search_fields, the_fields, field_query_tokens,- sort_fields_std, searched_queries, group_limit, group_by_fields,+ sort_fields_std, searched_queries, group_limit, group_by_fields, group_missing_values,topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,all_result_ids, all_result_ids_len, groups_processed, curated_ids,excluded_result_ids, excluded_result_ids_size, excluded_group_ids, curated_topster,@@ -2830,7 +2839,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consuint32_t seq_id = it.id();uint64_t distinct_id = seq_id;if (group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -2930,7 +2939,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consuint64_t distinct_id = seq_id;if (group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -2967,7 +2976,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons} else {search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,- curated_ids, curated_ids_sorted,+ group_missing_values, curated_ids, curated_ids_sorted,excluded_result_ids, excluded_result_ids_size, excluded_group_ids,all_result_ids, all_result_ids_len,filter_result.docs, filter_result.count, concurrency,@@ -3015,7 +3024,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consexcluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,excluded_group_ids,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,+ group_missing_values, prioritize_exact_match,prioritize_token_position, prioritize_num_matching_fields, query_hashes, token_order, prefixes,typo_tokens_threshold, exhaustive_search,max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,@@ -3053,7 +3063,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consexcluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,excluded_group_ids,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,+ group_missing_values, prioritize_exact_match,prioritize_token_position, prioritize_num_matching_fields,query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,@@ -3064,7 +3075,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons// do synonym based searchesdo_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map, sort_fields_std,curated_topster, token_order,- 0, group_limit, group_by_fields, prioritize_exact_match, prioritize_token_position,+ 0, group_limit, group_by_fields, group_missing_values,+ prioritize_exact_match, prioritize_token_position,prioritize_num_matching_fields,exhaustive_search, concurrency, prefixes,min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,@@ -3126,6 +3138,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscurated_ids_sorted, excluded_group_ids, sort_fields_std, num_typos, searched_queries,qtoken_set, topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields,+ group_missing_values,prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,query_hashes, token_order, prefixes, typo_tokens_threshold,exhaustive_search, max_candidates, min_len_1typo,@@ -3139,7 +3152,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std, searched_queries,- group_limit, group_by_fields,+ group_limit, group_by_fields, group_missing_values,max_extra_prefix, max_extra_suffix,field_query_tokens[0].q_include_tokens,topster, filter_result.docs, filter_result.count,@@ -3247,7 +3260,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consuint64_t distinct_id = seq_id;if (group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -3326,7 +3339,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consthread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,batch_result_ids, batch_res_len, &facet_infos,- estimate_facets, facet_sample_percent,+ estimate_facets, facet_sample_percent, group_missing_values,&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,&num_processed, &m_process, &cv_process]() {search_begin_us = parent_search_begin;@@ -3335,7 +3348,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consauto fq = facet_query;do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,- facet_infos, group_limit, group_by_fields,+ facet_infos, group_limit, group_by_fields, group_missing_values,batch_result_ids, batch_res_len);std::unique_lock<std::mutex> lock(m_process);num_processed++;@@ -3412,7 +3425,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscompute_facet_infos(facets, facet_query, facet_query_num_typos,&included_ids_vec[0], included_ids_vec.size(), group_by_fields, max_candidates, facet_infos);do_facets(facets, facet_query, estimate_facets, facet_sample_percent,- facet_infos, group_limit, group_by_fields, &included_ids_vec[0], included_ids_vec.size());+ facet_infos, group_limit, group_by_fields, group_missing_values,+ &included_ids_vec[0], included_ids_vec.size());all_result_ids_len += curated_topster->size;@@ -3428,6 +3442,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consvoid Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,const std::vector<uint32_t>& excluded_ids,const std::vector<std::string>& group_by_fields, const size_t group_limit,+ const bool group_missing_values,const bool filter_curated_hits, const uint32_t* filter_ids, uint32_t filter_ids_length,std::set<uint32_t>& curated_ids,std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,@@ -3441,7 +3456,7 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>if(group_limit != 0) {// if one `id` of a group is present in curated hits, we have to exclude that entire group from resultsfor(auto seq_id: included_ids_vec) {- uint64_t distinct_id = get_distinct_id(group_by_fields, seq_id);+ uint64_t distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);excluded_group_ids.emplace(distinct_id);}}@@ -3531,6 +3546,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,uint32_t*& all_result_ids, size_t & all_result_ids_len,const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,bool prioritize_exact_match,const bool prioritize_token_position,const bool prioritize_num_matching_fields,@@ -3777,7 +3793,8 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,sort_fields, token_candidates_vec, searched_queries, qtoken_set,dropped_tokens, topster,groups_processed, all_result_ids, all_result_ids_len,- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,+ typo_tokens_threshold, group_limit, group_by_fields,+ group_missing_values, query_tokens,num_typos, prefixes, prioritize_exact_match, prioritize_token_position,prioritize_num_matching_fields, exhaustive_search, max_candidates,syn_orig_num_tokens, sort_order, field_values, geopoint_indices,@@ -3932,6 +3949,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,const std::vector<token_t>& dropped_tokens,const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const bool prioritize_exact_match,const bool prioritize_token_position,const bool prioritize_num_matching_fields,@@ -4142,7 +4160,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,uint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {return;}@@ -4472,6 +4490,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<sconst std::vector<sort_by>& sort_fields,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,Topster* actual_topster,const int sort_order[3],std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,@@ -4623,7 +4642,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<suint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -4650,7 +4669,9 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,const token_ordering& token_order,const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,+ bool prioritize_exact_match,const bool prioritize_token_position,const bool prioritize_num_matching_fields,const bool exhaustive_search, const size_t concurrency,@@ -4678,7 +4699,8 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted, excluded_group_ids,sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,+ group_missing_values, prioritize_exact_match,prioritize_token_position, prioritize_num_matching_fields,query_hashes, token_order, prefixes, typo_tokens_threshold,exhaustive_search, max_candidates, min_len_1typo,@@ -4692,7 +4714,9 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<seconst std::vector<enable_t>& infixes,const std::vector<sort_by>& sort_fields,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,+ const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,+ const size_t max_extra_prefix,const size_t max_extra_suffix,const std::vector<token_t>& query_tokens, Topster* actual_topster,const uint32_t *filter_ids, size_t filter_ids_length,@@ -4758,7 +4782,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<seuint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -4914,7 +4938,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&facet_field, facet_field.faceted_name(),all_result_ids, all_result_ids_len, {}, sort_fields, -1, facet_query_num_typos, searched_queries, topster,groups_processed, &field_result_ids, field_result_ids_len, field_num_results, 0, group_by_fields,- false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates);+ true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates);//LOG(INFO) << "searched_queries.size: " << searched_queries.size();@@ -5022,7 +5046,9 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,+ const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,+ const std::set<uint32_t>& curated_ids,const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,@@ -5071,7 +5097,8 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,thread_id, &sort_fields, &searched_queries,- &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,+ &group_limit, &group_by_fields, group_missing_values,+ &topsters, &tgroups_processed, &excluded_group_ids,&sort_order, field_values, &geopoint_indices, &plists,check_for_circuit_break,batch_result_ids, batch_res_len,@@ -5098,7 +5125,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,uint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {continue;}@@ -5206,6 +5233,7 @@ void Index::search_field(const uint8_t & field_id,Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,bool prioritize_exact_match,const size_t concurrency,std::set<uint64>& query_hashes,@@ -5337,7 +5365,8 @@ void Index::search_field(const uint8_t & field_id,exclude_token_ids, exclude_token_ids_size,curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,groups_processed, all_result_ids, all_result_ids_len, field_num_results,- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,+ typo_tokens_threshold, group_limit, group_by_fields,+ group_missing_values, query_tokens,prioritize_exact_match, exhaustive_search, syn_orig_num_tokens,concurrency, query_hashes, id_buff);@@ -5508,6 +5537,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,const uint32_t token_bits,const bool prioritize_exact_match,const bool single_exact_query_token,@@ -5700,7 +5730,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16uint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);}//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;@@ -5716,8 +5746,9 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16// pre-filter group_by_fields such that we can avoid the find() checkuint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,- const uint32_t seq_id) const {- uint64_t distinct_id = 1; // some constant initial value+ const uint32_t seq_id, const bool group_missing_values) const {+ uint64_t distinct_id = 1;+ bool found_group_value = false;// calculate hash from group_by_fieldsfor(const auto& field: group_by_fields) {@@ -5740,6 +5771,7 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,for(size_t i = 0; i < facet_hashes.size(); i++) {distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);+ found_group_value = true;}} else {const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;@@ -5750,12 +5782,12 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,}const auto& facet_hash = facet_hashes_it->second;-distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);+ found_group_value = true;}}- return distinct_id;+ return (!found_group_value && !group_missing_values) ? seq_id : distinct_id;}inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token_candidates_vec,diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 154e7555..8a4a3330 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -606,6 +606,77 @@ TEST_F(CollectionGroupingTest, RepeatedFieldNameGroupHitCount) {ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());}+TEST_F(CollectionGroupingTest, ControlMissingValues) {+ std::vector<field> fields = {+ field("brand", field_types::STRING, true, true),+ };++ Collection* coll2 = collectionManager.get_collection("coll2").get();+ if(coll2 == nullptr) {+ coll2 = collectionManager.create_collection("coll2", 1, fields).get();+ }++ nlohmann::json doc;+ doc["id"] = "0";+ doc["brand"] = "Omega";+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["brand"] = nullptr;+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ doc["id"] = "2";+ doc["brand"] = nullptr;+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ doc["id"] = "3";+ doc["brand"] = "Omega";+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ // disable null value aggregation++ auto res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,+ {true}, 10,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"brand"}, 2,+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,+ 100, 0, 0, 30000, 2, true, false).get();++ ASSERT_EQ(3, res["grouped_hits"].size());+ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());++ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());+ ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());+ ASSERT_EQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());++ ASSERT_EQ(0, res["grouped_hits"][2]["group_key"].size());+ ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());+ ASSERT_EQ("1", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>());++ // with null value aggregation (default)+ res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,+ {true}, 10,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"brand"}, 2,+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,+ 100, 0, 0, 30000, 2, true, true).get();++ ASSERT_EQ(2, res["grouped_hits"].size());++ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());++ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());+ ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());+}+TEST_F(CollectionGroupingTest, SortingOnGroupCount) {std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};