Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.26-facets
Kishore Nallance4b8e3e2cfd
diff --cc include/collection.hindex 57107cd2,6e20beb6..920b4597--- a/include/collection.h+++ b/include/collection.h@@@ -492,13 -466,10 +492,15 @@@ publicconst size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,const size_t page_offset = 0,+ facet_index_type_t facet_index_type = HASH,const size_t remote_embedding_timeout_ms = 30000,const size_t remote_embedding_num_tries = 2,+ const std::string& stopwords_set="",+ const std::vector<std::string>& facet_return_parent = {},+ const std::vector<ref_include_fields>& ref_include_fields_vec = {},- const drop_tokens_mode_t drop_tokens_mode = right_to_left) const;++ const drop_tokens_mode_t drop_tokens_mode = right_to_left,+ const bool prioritize_num_matching_fields = true,+ const bool group_missing_values = true) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --cc include/index.hindex c4e65e16,a987f3f2..a78cf783--- a/include/index.h+++ b/include/index.h@@@ -382,9 -373,8 +388,10 @@@ privatebool estimate_facets, size_t facet_sample_percent,const std::vector<facet_info_t>& facet_infos,size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,- const uint32_t* result_ids, size_t results_size) const;+ const uint32_t* result_ids, size_t results_size,+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided,+ facet_index_type_t facet_index_type) const;bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,filter_node_t*& filter_tree_root) const;@@@ -430,36 -421,38 +438,38 @@@size_t min_len_2typo,size_t max_candidates) const;- void search_all_candidates(const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<search_field_t>& the_fields,- const uint32_t* filter_ids, size_t filter_ids_length,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by>& sort_fields,- std::vector<tok_candidates>& token_candidates_vec,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const size_t typo_tokens_threshold,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool exhaustive_search,- const bool prioritize_num_matching_fields,- const size_t max_candidates,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::set<uint64>& query_hashes,- std::vector<uint32_t>& id_buff) const;+ Option<bool> search_all_candidates(const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<search_field_t>& the_fields,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by>& sort_fields,+ std::vector<tok_candidates>& token_candidates_vec,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const size_t typo_tokens_threshold,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ bool prioritize_exact_match,+ const bool prioritize_token_position,+ const bool exhaustive_search,++ const bool prioritize_num_matching_fields,+ const size_t max_candidates,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::set<uint64>& query_hashes,+ std::vector<uint32_t>& id_buff, const std::string& collection_name = "") const;void search_candidates(const uint8_t & field_id,bool field_is_array,@@@ -675,7 -705,8 +691,10 @@@ publicconst std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,- const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true);- const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true,- const bool use_addition_fields = false, const tsl::htrie_map<char, field>& addition_fields = {});++ const bool do_validation, const size_t remote_embedding_batch_size = 200,++ const bool generate_embeddings = true,++ const bool use_addition_fields = false,++ const tsl::htrie_map<char, field>& addition_fields = {});void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);@@@ -704,21 -752,21 +723,22 @@@// the following methods are not synchronized because their parent calls are synchronized or they are const/static- void search_wildcard(filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,- uint32_t filter_ids_length, const size_t concurrency,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const;+ Option<bool> search_wildcard(filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,++ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values, const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ const size_t concurrency,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::string& collection_name = "") const;void search_infix(const std::string& query, const std::string& field_name, std::vector<uint32_t>& ids,size_t max_extra_prefix, size_t max_extra_suffix) const;@@@ -758,103 -797,107 +778,111 @@@const std::vector<search_field_t>& search_fields, uint32_t*& exclude_token_ids,size_t& exclude_token_ids_size) const;- void do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,- const std::vector<enable_t>& infixes,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values, const size_t max_extra_prefix,- const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,- const uint32_t *filter_ids, size_t filter_ids_length,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;-- void do_synonym_search(const std::vector<search_field_t>& the_fields,- const text_match_type_t match_type,- filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,- const token_ordering& token_order,- const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool prioritize_num_matching_fields,- const bool exhaustive_search, const size_t concurrency,- const std::vector<bool>& prefixes,- size_t min_len_1typo,- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* actual_topster,- std::vector<std::vector<token_t>>& q_pos_synonyms,- int syn_orig_num_tokens,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const uint32_t* filter_ids, uint32_t filter_ids_length,- std::set<uint64>& query_hashes,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- tsl::htrie_map<char, token_leaf>& qtoken_set) const;-- void do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,- std::vector<query_tokens_t>& field_query_tokens,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- Topster* actual_topster,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- const std::set<uint32_t>& curated_ids,- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* curated_topster,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- bool is_wildcard_query,- uint32_t*& filter_ids, uint32_t& filter_ids_length) const;-- void fuzzy_search_fields(const std::vector<search_field_t>& the_fields,- const std::vector<token_t>& query_tokens,- const std::vector<token_t>& dropped_tokens,- const text_match_type_t match_type,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const uint32_t* filter_ids, size_t filter_ids_length,- const std::vector<uint32_t>& curated_ids,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by>& sort_fields,- const std::vector<uint32_t>& num_typos,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const size_t group_limit, const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool prioritize_num_matching_fields,- std::set<uint64>& query_hashes,- const token_ordering token_order,- const std::vector<bool>& prefixes,- const size_t typo_tokens_threshold,- const bool exhaustive_search,- const size_t max_candidates,- size_t min_len_1typo,- size_t min_len_2typo,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const;+ Option<bool> do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,+ const std::vector<enable_t>& infixes,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,++ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,++ const size_t max_extra_prefix,+ const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,+ filter_result_iterator_t* const filter_result_iterator,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ const std::string& collection_name = "") const;++ [[nodiscard]] Option<bool> do_synonym_search(const std::vector<search_field_t>& the_fields,+ const text_match_type_t match_type,+ filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,+ const token_ordering& token_order,+ const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,++ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,++ bool prioritize_exact_match,+ const bool prioritize_token_position,++ const bool prioritize_num_matching_fields,+ const bool exhaustive_search, const size_t concurrency,+ const std::vector<bool>& prefixes,+ size_t min_len_1typo,+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* actual_topster,+ std::vector<std::vector<token_t>>& q_pos_synonyms,+ int syn_orig_num_tokens,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ std::set<uint64>& query_hashes,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::string& collection_name = "") const;++ Option<bool> do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,+ std::vector<query_tokens_t>& field_query_tokens,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ Topster* actual_topster,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ filter_result_iterator_t*& filter_result_iterator,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ const std::set<uint32_t>& curated_ids,+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* curated_topster,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ bool is_wildcard_query, const std::string& collection_name = "") const;++ [[nodiscard]] Option<bool> fuzzy_search_fields(const std::vector<search_field_t>& the_fields,+ const std::vector<token_t>& query_tokens,+ const std::vector<token_t>& dropped_tokens,+ const text_match_type_t match_type,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ filter_result_iterator_t* const filter_result_iterator,+ const std::vector<uint32_t>& curated_ids,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by>& sort_fields,+ const std::vector<uint32_t>& num_typos,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ bool prioritize_exact_match,+ const bool prioritize_token_position,++ const bool prioritize_num_matching_fields,+ std::set<uint64>& query_hashes,+ const token_ordering token_order,+ const std::vector<bool>& prefixes,+ const size_t typo_tokens_threshold,+ const bool exhaustive_search,+ const size_t max_candidates,+ size_t min_len_1typo,+ size_t min_len_2typo,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::string& collection_name = "") const;void find_across_fields(const token_t& previous_token,const std::string& previous_token_str,@@@ -866,34 -909,36 +894,36 @@@std::vector<uint32_t>& prev_token_doc_ids,std::vector<size_t>& top_prefix_field_ids) const;- void search_across_fields(const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- const std::vector<search_field_t>& the_fields,- const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<sort_by>& sort_fields,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- bool prioritize_exact_match,- const bool search_all_candidates,- const bool prioritize_num_matching_fields,- const uint32_t* filter_ids, uint32_t filter_ids_length,- const uint32_t total_cost,- const int syn_orig_num_tokens,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::vector<uint32_t>& id_buff,- uint32_t*& all_result_ids,- size_t& all_result_ids_len) const;+ Option<bool> search_across_fields(const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ const std::vector<search_field_t>& the_fields,+ const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<sort_by>& sort_fields,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ bool prioritize_exact_match,+ const bool search_all_candidates,++ const bool prioritize_num_matching_fields,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t total_cost,+ const int syn_orig_num_tokens,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::vector<uint32_t>& id_buff,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const std::string& collection_name = "") const;voidsearch_fields(const std::vector<filter>& filters,@@@ -923,26 -968,23 +953,27 @@@filter_node_t*& filter_tree_root,std::vector<const override_t*>& matched_dynamic_overrides) const;- void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices, uint32_t seq_id,- size_t filter_index,- int64_t max_field_match_score,- int64_t* scores,- int64_t& match_score_index, float vector_distance = 0) const;-- void- process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,- const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,- const size_t group_limit, const bool group_missing_values,- const bool filter_curated_hits, const uint32_t* filter_ids,- uint32_t filter_ids_length, std::set<uint32_t>& curated_ids,- std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- std::vector<uint32_t>& included_ids_vec,- std::unordered_set<uint32_t>& excluded_group_ids) const;+ Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices, uint32_t seq_id,+ const std::map<basic_string<char>, reference_filter_result_t>& references,+ size_t filter_index,+ int64_t max_field_match_score,+ int64_t* scores,+ int64_t& match_score_index, float vector_distance = 0,+ const std::string& collection_name = "") const;++ void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,- const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,- const size_t group_limit, const bool filter_curated_hits,++ const std::vector<uint32_t>& excluded_ids,++ const std::vector<std::string>& group_by_fields,++ const size_t group_limit, const bool group_missing_values, const bool filter_curated_hits,+ filter_result_iterator_t* const filter_result_iterator,+ std::set<uint32_t>& curated_ids,+ std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ std::vector<uint32_t>& included_ids_vec,+ std::unordered_set<uint32_t>& excluded_group_ids) const;++ int64_t get_doc_val_from_sort_index(sort_index_iterator it, uint32_t doc_seq_id) const;Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,std::vector<uint32_t>& outside_seq_ids);diff --cc include/topster.hindex 355300ea,fc9d3582..a471564c--- a/include/topster.h+++ b/include/topster.h@@@ -15,11 -14,13 +15,11 @@@ struct KVuint64_t key{};uint64_t distinct_key{};int64_t scores[3]{}; // match score + 2 custom attributes-+// only to be used in hybrid search- float vector_distance = 2.0f;+ float vector_distance = -1.0f;int64_t text_match_score = 0;- reference_filter_result_t* reference_filter_result = nullptr;-// to be used only in final aggregationuint64_t* query_indices = nullptr;diff --cc src/collection.cppindex 12f12e90,717f99ac..9eb782c3--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -1384,13 -1113,10 +1388,15 @@@ Option<nlohmann::json> Collection::searconst size_t facet_sample_percent,const size_t facet_sample_threshold,const size_t page_offset,+ facet_index_type_t facet_index_type,const size_t remote_embedding_timeout_ms,const size_t remote_embedding_num_tries,+ const std::string& stopwords_set,+ const std::vector<std::string>& facet_return_parent,+ const std::vector<ref_include_fields>& ref_include_fields_vec,- const drop_tokens_mode_t drop_tokens_mode) const {++ const drop_tokens_mode_t drop_tokens_mode,+ const bool prioritize_num_matching_fields,+ const bool group_missing_values) const {std::shared_lock lock(mutex);@@@ -4110,10 -3762,11 +4123,11 @@@ Option<bool> Collection::batch_alter_da// Update schema with additions (deletions can only be made later)std::vector<field> new_fields;tsl::htrie_map<char, field> schema_additions;-std::vector<std::string> nested_field_names;-+ bool found_embedding_field = false;+ std::unique_lock ulock(mutex);+for(auto& f: alter_fields) {if(f.name == ".*") {fields.push_back(f);@@@ -4194,9 -3845,25 +4209,26 @@@index->remove(seq_id, rec.doc, del_fields, true);}}-+- Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, embedding_fields,- fallback_field_type, token_separators, symbols_to_index, true, 200, false);+ Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,- fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field, true, schema_additions);++ fallback_field_type, token_separators, symbols_to_index, true, 200,++ found_embedding_field, true, schema_additions);+ if(found_embedding_field) {+ for(auto& index_record : iter_batch) {+ if(index_record.indexed.ok()) {+ remove_flat_fields(index_record.doc);+ const std::string& serialized_json = index_record.doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);+ bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);++ if(!write_ok) {+ LOG(ERROR) << "Inserting doc with new embedding field failed for seq id: " << index_record.seq_id;+ index_record.index_failure(500, "Could not write to on-disk storage.");+ } else {+ index_record.index_success();+ }+ }+ }+ }iter_batch.clear();}diff --cc src/collection_manager.cppindex 78c99a2d,657a906a..537bd9f1--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -976,7 -751,7 +977,8 @@@ Option<bool> CollectionManager::do_searconst char *FACET_SAMPLE_PERCENT = "facet_sample_percent";const char *FACET_SAMPLE_THRESHOLD = "facet_sample_threshold";+ const char *DROP_TOKENS_MODE = "drop_tokens_mode";+ const char *PRIORITIZE_NUM_MATCHING_FIELDS = "prioritize_num_matching_fields";// enrich params with values from embedded paramsfor(auto& item: embedded_params.items()) {@@@ -1098,7 -863,7 +1101,8 @@@size_t facet_sample_percent = 100;size_t facet_sample_threshold = 0;+ std::string drop_tokens_mode_str = "right_to_left";+ bool prioritize_num_matching_fields = true;std::unordered_map<std::string, size_t*> unsigned_int_values = {{MIN_LEN_1TYPO, &min_len_1typo},@@@ -1347,13 -1104,11 +1353,16 @@@facet_sample_percent,facet_sample_threshold,offset,+ HASH,remote_embedding_timeout_ms,remote_embedding_num_tries,+ stopwords_set,+ facet_return_parent,+ ref_include_fields_vec,- drop_tokens_mode);++ drop_tokens_mode,+ prioritize_num_matching_fields,+ group_missing_values+ );uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --cc src/index.cppindex 3b68cef9,3b3daea6..63d1ced1--- a/src/index.cpp+++ b/src/index.cpp@@@ -514,10 -511,9 +514,10 @@@ void Index::validate_and_preprocess(Ind}}-size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,+size_t Index::batch_memory_index(Index *index,+ std::vector<index_record>& iter_batch,const std::string & default_sorting_field,- const tsl::htrie_map<char, field> & search_schema,+ const tsl::htrie_map<char, field> & actual_search_schema,const tsl::htrie_map<char, field> & embedding_fields,const std::string& fallback_field_type,const std::vector<char>& token_separators,@@@ -1260,15 -1171,8 +1263,16 @@@ void Index::do_facets(std::vector<facetbool estimate_facets, size_t facet_sample_percent,const std::vector<facet_info_t>& facet_infos,const size_t group_limit, const std::vector<std::string>& group_by_fields,+ const bool group_missing_values,- const uint32_t* result_ids, size_t results_size) const {+ const uint32_t* result_ids, size_t results_size,+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided,+ facet_index_type_t facet_index_type) const {++ if(results_size == 0) {+ return ;+ }++ size_t total_docs = seq_ids->num_ids();// assumed that facet fields have already been validated upstreamfor(size_t findex=0; findex < facets.size(); findex++) {@@@ -1283,149 -1186,101 +1287,153 @@@size_t mod_value = 100 / facet_sample_percent;- facet_map_t::iterator facet_map_it;- single_val_facet_map_t::iterator single_facet_map_it;- uint64_t fhash = 0;- size_t facet_hash_count = 1;- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);-- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {- continue;+ auto num_facet_values = facet_index_v4->get_facet_count(facet_field.name);+ if(num_facet_values == 0) {+ continue;}- for(size_t i = 0; i < results_size; i++) {- // if sampling is enabled, we will skip a portion of the results to speed up things- if(estimate_facets) {- if(i % mod_value != 0) {- continue;+ bool is_wildcard_no_filter_query = is_wildcard_query && no_filters_provided;+ bool facet_value_index_exists = facet_index_v4->has_value_index(facet_field.name);++#ifdef TEST_BUILD+ if(facet_index_type == VALUE) {+#else+ if(facet_value_index_exists && use_value_index) {+#endif+ // LOG(INFO) << "Using intersection to find facets";+ a_facet.is_intersected = true;++ std::map<std::string, docid_count_t> facet_results;+ std::string sort_order = a_facet.is_sort_by_alpha ? a_facet.sort_order : "";++ facet_index_v4->intersect(a_facet, use_facet_query,+ facet_infos[findex].fvalue_searched_tokens, result_ids,+ results_size, max_facet_count, facet_results,+ is_wildcard_no_filter_query, sort_order);++ for(const auto& kv : facet_results) {+ //range facet processing+ if(a_facet.is_range_query) {+ const auto doc_val = kv.first;+ std::pair<int64_t , std::string> range_pair {};+ if(a_facet.get_range(std::stoll(doc_val), range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count = kv.second.count;+ }+ } else {+ facet_count_t& facet_count = a_facet.value_result_map[kv.first];+ facet_count.count = kv.second.count;+ facet_count.doc_id = kv.second.doc_id;}++ if(should_compute_stats) {+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;+ for(size_t i = 0; i < kv.second.count; ++i) {+ compute_facet_stats(a_facet, kv.first, facet_field.type);+ }+ }}-- uint32_t doc_seq_id = result_ids[i];- if(facet_field.is_array()) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }- facet_hash_count = facet_map_it->second.size();- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }- facet_hash_count = 1;- fhash = single_facet_map_it->second;+ } else {+ //LOG(INFO) << "Using hashing to find facets";+ bool facet_hash_index_exists = facet_index_v4->has_hash_index(facet_field.name);+ if(!facet_hash_index_exists) {+ continue;}-- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields,- doc_seq_id, group_missing_values) : 0;- if(((i + 1) % 16384) == 0) {- RETURN_CIRCUIT_BREAKER- }+ const auto& fhash_int64_map = facet_index_v4->get_fhash_int64_map(a_facet.field_name);- std::set<uint32_t> unique_facet_hashes;+ const auto facet_field_is_array = facet_field.is_array();- for(size_t j = 0; j < facet_hash_count; j++) {-- if(facet_field.is_array()) {- fhash = facet_map_it->second.hashes[j];+ const auto& facet_index = facet_index_v4->get_facet_hash_index(facet_field.name);+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();+ std::vector<uint32_t> facet_hashes;++ for(size_t i = 0; i < results_size; i++) {+ // if sampling is enabled, we will skip a portion of the results to speed up things+ if(estimate_facets) {+ if(i % mod_value != 0) {+ continue;+ }}- if(unique_facet_hashes.count(fhash) == 0) {- unique_facet_hashes.insert(fhash);- } else {+ uint32_t doc_seq_id = result_ids[i];+ facet_index_it.skip_to(doc_seq_id);++ if(!facet_index_it.valid()) {+ break;+ }++ if(facet_index_it.id() != doc_seq_id) {continue;}-- if(should_compute_stats) {- compute_facet_stats(a_facet, fhash, facet_field.type);++ facet_hashes.clear();+ posting_list_t::get_offsets(facet_index_it, facet_hashes);+- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;++ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id, group_missing_values) : 0;+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;+ if(((i + 1) % 16384) == 0) {+ RETURN_CIRCUIT_BREAKER}- if(a_facet.is_range_query) {- if(sort_index_it != sort_index.end()){- auto doc_id_val_map = sort_index_it->second;- auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);-- if(doc_seq_id_it != doc_id_val_map->end()){- int64_t doc_val = doc_seq_id_it->second;- std::pair<int64_t, std::string> range_pair {};- if(a_facet.get_range(doc_val, range_pair)) {- int64_t range_id = range_pair.first;- facet_count_t& facet_count = a_facet.result_map[range_id];- facet_count.count += 1;-- if(group_limit) {- a_facet.hash_groups[range_id].emplace(distinct_id);- }++ std::set<uint32_t> unique_facet_hashes;++ for(size_t j = 0; j < facet_hashes.size(); j++) {++ const auto& fhash = facet_hashes[j];++ if(unique_facet_hashes.count(fhash) == 0) {+ unique_facet_hashes.insert(fhash);+ } else {+ continue;+ }++ if(should_compute_stats) {+ int64_t val = fhash;+ if(facet_field.is_int64()) {+ if(fhash_int64_map.find(fhash) != fhash_int64_map.end()) {+ val = fhash_int64_map.at(fhash);+ } else {+ val = INT64_MAX;}}++ compute_facet_stats(a_facet, val, facet_field.type);}- } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {- facet_count_t& facet_count = a_facet.result_map[fhash];- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;- facet_count.doc_id = doc_seq_id;- facet_count.array_pos = j;- if(group_limit) {- a_facet.hash_groups[fhash].emplace(distinct_id);- } else {- facet_count.count += 1;- }- if(use_facet_query) {- a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);++ if(a_facet.is_range_query) {+ int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);++ std::pair<int64_t , std::string> range_pair {};+ if(a_facet.get_range(doc_val, range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count += 1;++++ if(group_limit) {++ a_facet.hash_groups[range_id].emplace(distinct_id);++ }+ }+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {+ facet_count_t& facet_count = a_facet.result_map[fhash];+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;+ facet_count.doc_id = doc_seq_id;+ facet_count.array_pos = j;+ if(group_limit) {+ a_facet.hash_groups[fhash].emplace(distinct_id);+ } else {+ facet_count.count += 1;+ }+ if(use_facet_query) {+ //LOG (INFO) << "adding hash tokens for hash " << fhash;+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);+ }+ if(!a_facet.sort_field.empty()) {+ sort_index_it = sort_index.find(a_facet.sort_field);+ facet_count.sort_field_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);+ //LOG(INFO) << "found sort_field val " << facet_count.sort_field;+ }}}}@@@ -1448,36 -1303,38 +1456,38 @@@ void Index::aggregate_topster(Topster*}}-void Index::search_all_candidates(const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<search_field_t>& the_fields,- const uint32_t* filter_ids, size_t filter_ids_length,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by>& sort_fields,- std::vector<tok_candidates>& token_candidates_vec,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const size_t typo_tokens_threshold,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool prioritize_num_matching_fields,- const bool exhaustive_search,- const size_t max_candidates,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::set<uint64>& query_hashes,- std::vector<uint32_t>& id_buff) const {+Option<bool> Index::search_all_candidates(const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<search_field_t>& the_fields,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by>& sort_fields,+ std::vector<tok_candidates>& token_candidates_vec,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const size_t typo_tokens_threshold,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ bool prioritize_exact_match,+ const bool prioritize_token_position,++ const bool prioritize_num_matching_fields,+ const bool exhaustive_search,+ const size_t max_candidates,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::set<uint64>& query_hashes,+ std::vector<uint32_t>& id_buff, const std::string& collection_name) const {/*if(!token_candidates_vec.empty()) {LOG(INFO) << "Prefix candidates size: " << token_candidates_vec.back().candidates.size();@@@ -1516,27 -1373,18 +1526,28 @@@//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;- search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields, match_type,- sort_fields, topster,groups_processed,- searched_queries, qtoken_set, dropped_tokens, group_limit, group_by_fields,- group_missing_values,- prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,- filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,- sort_order, field_values, geopoint_indices,- id_buff, all_result_ids, all_result_ids_len);+ auto search_across_fields_op = search_across_fields(query_suggestion, num_typos, prefixes, the_fields,+ num_search_fields, match_type,+ sort_fields, topster,groups_processed,+ searched_queries, qtoken_set, dropped_tokens,- group_limit, group_by_fields,++ group_limit, group_by_fields, group_missing_values,+ prioritize_exact_match, prioritize_token_position,++ prioritize_num_matching_fields,+ filter_result_iterator,+ total_cost, syn_orig_num_tokens,+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,+ sort_order, field_values, geopoint_indices,+ id_buff, all_result_ids, all_result_ids_len,+ collection_name);+ if (!search_across_fields_op.ok()) {+ return search_across_fields_op;+ }query_hashes.insert(qhash);+ filter_result_iterator->reset();}++ return Option<bool>(true);}void Index::search_candidates(const uint8_t & field_id, bool field_is_array,@@@ -1571,110 -1420,792 +1583,110 @@@populate_sort_mapping(sort_order, geopoint_indices, sort_fields, field_values);// escape hatch to prevent too much looping- size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;-- for (long long n = 0; n < N && n < combination_limit; ++n) {- RETURN_CIRCUIT_BREAKER-- // every element in `query_suggestion` contains a token and its associated hits- std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());-- // actual query suggestion preserves original order of tokens in query- std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());- uint64 qhash;-- uint32_t token_bits = 0;- uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,- query_suggestion, syn_orig_num_tokens, token_bits, qhash);-- if(query_hashes.find(qhash) != query_hashes.end()) {- // skip this query since it has already been processed before- continue;- }-- query_hashes.insert(qhash);-- //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;- //LOG(INFO) << "n: " << n;-- /*std::stringstream fullq;- for(const auto& qleaf : actual_query_suggestion) {- std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);- fullq << qtok << " ";- }- LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;*/-- // Prepare excluded document IDs that we can later remove from the result set- uint32_t* excluded_result_ids = nullptr;- size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,- &curated_ids[0], curated_ids.size(), &excluded_result_ids);-- std::vector<void*> posting_lists;-- for(auto& query_leaf : query_suggestion) {- posting_lists.push_back(query_leaf->values);- }-- result_iter_state_t iter_state(- excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length- );-- // We fetch offset positions only for multi token query- bool fetch_offsets = (query_suggestion.size() > 1);- bool single_exact_query_token = false;-- if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {- // does this candidate suggestion token match query token exactly?- single_exact_query_token = true;- }-- if(topster == nullptr) {- posting_t::block_intersector_t(posting_lists, iter_state)- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {- id_buff.push_back(seq_id);- });- } else {- posting_t::block_intersector_t(posting_lists, iter_state)- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {- score_results(sort_fields, searched_queries.size(), field_id, field_is_array,- total_cost, topster, query_suggestion, groups_processed,- seq_id, sort_order, field_values, geopoint_indices,- group_limit, group_by_fields, group_missing_values, token_bits,- prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);-- id_buff.push_back(seq_id);- });- }-- delete [] excluded_result_ids;- const size_t num_result_ids = id_buff.size();-- if(id_buff.size() > 100000) {- // prevents too many ORs during exhaustive searching- std::sort(id_buff.begin(), id_buff.end());- id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());-- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],- id_buff.size(), &new_all_result_ids);- delete[] *all_result_ids;- *all_result_ids = new_all_result_ids;- id_buff.clear();- }-- if(num_result_ids == 0) {- continue;- }-- field_num_results += num_result_ids;- searched_queries.push_back(actual_query_suggestion);- }-}--void Index::numeric_not_equals_filter(num_tree_t* const num_tree,- const int64_t value,- const uint32_t& context_ids_length,- uint32_t* const& context_ids,- size_t& ids_len,- uint32_t*& ids) const {- uint32_t* to_exclude_ids = nullptr;- size_t to_exclude_ids_len = 0;-- if (context_ids_length != 0) {- num_tree->contains(EQUALS, value, context_ids_length, context_ids, to_exclude_ids_len, to_exclude_ids);- } else {- num_tree->search(EQUALS, value, &to_exclude_ids, to_exclude_ids_len);- }-- auto all_ids = seq_ids->uncompress();- auto all_ids_size = seq_ids->num_ids();-- uint32_t* to_include_ids = nullptr;- size_t to_include_ids_len = 0;-- to_include_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, to_exclude_ids,- to_exclude_ids_len, &to_include_ids);-- delete[] all_ids;- delete[] to_exclude_ids;-- uint32_t* out = nullptr;- ids_len = ArrayUtils::or_scalar(ids, ids_len, to_include_ids, to_include_ids_len, &out);-- delete[] ids;- delete[] to_include_ids;-- ids = out;-}--bool Index::field_is_indexed(const std::string& field_name) const {- return search_index.count(field_name) != 0 ||- numerical_index.count(field_name) != 0 ||- geopoint_index.count(field_name) != 0;-}--Option<bool> Index::do_filtering(filter_node_t* const root,- filter_result_t& result,- const std::string& collection_name,- const uint32_t& context_ids_length,- uint32_t* const& context_ids) const {- // auto begin = std::chrono::high_resolution_clock::now();- const filter a_filter = root->filter_exp;-- bool is_referenced_filter = !a_filter.referenced_collection_name.empty();- if (is_referenced_filter) {- // Apply filter on referenced collection and get the sequence ids of current collection from the filtered documents.- auto& cm = CollectionManager::get_instance();- auto collection = cm.get_collection(a_filter.referenced_collection_name);- if (collection == nullptr) {- return Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");- }-- filter_result_t reference_filter_result;- auto reference_filter_op = collection->get_reference_filter_ids(a_filter.field_name,- reference_filter_result,- collection_name);- if (!reference_filter_op.ok()) {- return Option<bool>(400, "Failed to apply reference filter on `" + a_filter.referenced_collection_name- + "` collection: " + reference_filter_op.error());- }-- if (context_ids_length != 0) {- std::vector<uint32_t> include_indexes;- include_indexes.reserve(std::min(context_ids_length, reference_filter_result.count));-- size_t context_index = 0, reference_result_index = 0;- while (context_index < context_ids_length && reference_result_index < reference_filter_result.count) {- if (context_ids[context_index] == reference_filter_result.docs[reference_result_index]) {- include_indexes.push_back(reference_result_index);- context_index++;- reference_result_index++;- } else if (context_ids[context_index] < reference_filter_result.docs[reference_result_index]) {- context_index++;- } else {- reference_result_index++;- }- }-- result.count = include_indexes.size();- result.docs = new uint32_t[include_indexes.size()];- auto& result_references = result.reference_filter_results[a_filter.referenced_collection_name];- result_references = new reference_filter_result_t[include_indexes.size()];-- for (uint32_t i = 0; i < include_indexes.size(); i++) {- result.docs[i] = reference_filter_result.docs[include_indexes[i]];- result_references[i] = reference_filter_result.reference_filter_results[a_filter.referenced_collection_name][include_indexes[i]];- }-- return Option(true);- }-- result = std::move(reference_filter_result);- return Option(true);- }-- if (a_filter.field_name == "id") {- // we handle `ids` separately- std::vector<uint32> result_ids;- for (const auto& id_str : a_filter.values) {- result_ids.push_back(std::stoul(id_str));- }-- std::sort(result_ids.begin(), result_ids.end());-- auto result_array = new uint32[result_ids.size()];- std::copy(result_ids.begin(), result_ids.end(), result_array);-- if (context_ids_length != 0) {- uint32_t* out = nullptr;- result.count = ArrayUtils::and_scalar(context_ids, context_ids_length,- result_array, result_ids.size(), &out);-- delete[] result_array;-- result.docs = out;- return Option(true);- }-- result.docs = result_array;- result.count = result_ids.size();- return Option(true);- }-- if (!field_is_indexed(a_filter.field_name)) {- return Option(true);- }-- field f = search_schema.at(a_filter.field_name);-- uint32_t* result_ids = nullptr;- size_t result_ids_len = 0;-- if (f.is_integer()) {- auto num_tree = numerical_index.at(a_filter.field_name);-- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {- const std::string& filter_value = a_filter.values[fi];- int64_t value = (int64_t)std::stol(filter_value);-- if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {- const std::string& next_filter_value = a_filter.values[fi + 1];- auto const range_end_value = (int64_t)std::stol(next_filter_value);-- if (context_ids_length != 0) {- num_tree->range_inclusive_contains(value, range_end_value, context_ids_length, context_ids,- result_ids_len, result_ids);- } else {- num_tree->range_inclusive_search(value, range_end_value, &result_ids, result_ids_len);- }-- fi++;- } else if (a_filter.comparators[fi] == NOT_EQUALS) {- numeric_not_equals_filter(num_tree, value, context_ids_length, context_ids, result_ids_len, result_ids);- } else {- if (context_ids_length != 0) {- num_tree->contains(a_filter.comparators[fi], value,- context_ids_length, context_ids, result_ids_len, result_ids);- } else {- num_tree->search(a_filter.comparators[fi], value, &result_ids, result_ids_len);- }- }- }- } else if (f.is_float()) {- auto num_tree = numerical_index.at(a_filter.field_name);-- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {- const std::string& filter_value = a_filter.values[fi];- float value = (float)std::atof(filter_value.c_str());- int64_t float_int64 = float_to_int64_t(value);-- if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {- const std::string& next_filter_value = a_filter.values[fi+1];- int64_t range_end_value = float_to_int64_t((float) std::atof(next_filter_value.c_str()));-- if (context_ids_length != 0) {- num_tree->range_inclusive_contains(float_int64, range_end_value, context_ids_length, context_ids,- result_ids_len, result_ids);- } else {- num_tree->range_inclusive_search(float_int64, range_end_value, &result_ids, result_ids_len);- }-- fi++;- } else if (a_filter.comparators[fi] == NOT_EQUALS) {- numeric_not_equals_filter(num_tree, float_int64,- context_ids_length, context_ids, result_ids_len, result_ids);- } else {- if (context_ids_length != 0) {- num_tree->contains(a_filter.comparators[fi], float_int64,- context_ids_length, context_ids, result_ids_len, result_ids);- } else {- num_tree->search(a_filter.comparators[fi], float_int64, &result_ids, result_ids_len);- }- }- }- } else if (f.is_bool()) {- auto num_tree = numerical_index.at(a_filter.field_name);-- size_t value_index = 0;- for (const std::string& filter_value : a_filter.values) {- int64_t bool_int64 = (filter_value == "1") ? 1 : 0;- if (a_filter.comparators[value_index] == NOT_EQUALS) {- numeric_not_equals_filter(num_tree, bool_int64,- context_ids_length, context_ids, result_ids_len, result_ids);- } else {- if (context_ids_length != 0) {- num_tree->contains(a_filter.comparators[value_index], bool_int64,- context_ids_length, context_ids, result_ids_len, result_ids);- } else {- num_tree->search(a_filter.comparators[value_index], bool_int64, &result_ids, result_ids_len);- }- }-- value_index++;- }- } else if (f.is_geopoint()) {- for (const std::string& filter_value : a_filter.values) {- std::vector<uint32_t> geo_result_ids;-- std::vector<std::string> filter_value_parts;- StringUtils::split(filter_value, filter_value_parts, ","); // x, y, 2, km (or) list of points-- bool is_polygon = StringUtils::is_float(filter_value_parts.back());- S2Region* query_region;-- if (is_polygon) {- const int num_verts = int(filter_value_parts.size()) / 2;- std::vector<S2Point> vertices;- double sum = 0.0;-- for (size_t point_index = 0; point_index < size_t(num_verts);- point_index++) {- double lat = std::stod(filter_value_parts[point_index * 2]);- double lon = std::stod(filter_value_parts[point_index * 2 + 1]);- S2Point vertex = S2LatLng::FromDegrees(lat, lon).ToPoint();- vertices.emplace_back(vertex);- }-- auto loop = new S2Loop(vertices, S2Debug::DISABLE);- loop->Normalize(); // if loop is not CCW but CW, change to CCW.-- S2Error error;- if (loop->FindValidationError(&error)) {- LOG(ERROR) << "Query vertex is bad, skipping. Error: " << error;- delete loop;- continue;- } else {- query_region = loop;- }- } else {- double radius = std::stof(filter_value_parts[2]);- const auto& unit = filter_value_parts[3];-- if (unit == "km") {- radius *= 1000;- } else {- // assume "mi" (validated upstream)- radius *= 1609.34;- }-- S1Angle query_radius = S1Angle::Radians(S2Earth::MetersToRadians(radius));- double query_lat = std::stod(filter_value_parts[0]);- double query_lng = std::stod(filter_value_parts[1]);- S2Point center = S2LatLng::FromDegrees(query_lat, query_lng).ToPoint();- query_region = new S2Cap(center, query_radius);- }-- S2RegionTermIndexer::Options options;- options.set_index_contains_points_only(true);- S2RegionTermIndexer indexer(options);-- for (const auto& term : indexer.GetQueryTerms(*query_region, "")) {- auto geo_index = geopoint_index.at(a_filter.field_name);- const auto& ids_it = geo_index->find(term);- if(ids_it != geo_index->end()) {- geo_result_ids.insert(geo_result_ids.end(), ids_it->second.begin(), ids_it->second.end());- }- }-- gfx::timsort(geo_result_ids.begin(), geo_result_ids.end());- geo_result_ids.erase(std::unique( geo_result_ids.begin(), geo_result_ids.end() ), geo_result_ids.end());-- // `geo_result_ids` will contain all IDs that are within approximately within query radius- // we still need to do another round of exact filtering on them-- if (context_ids_length != 0) {- uint32_t *out = nullptr;- uint32_t count = ArrayUtils::and_scalar(context_ids, context_ids_length,- &geo_result_ids[0], geo_result_ids.size(), &out);-- geo_result_ids = std::vector<uint32_t>(out, out + count);- }-- std::vector<uint32_t> exact_geo_result_ids;-- if (f.is_single_geopoint()) {- spp::sparse_hash_map<uint32_t, int64_t>* sort_field_index = sort_index.at(f.name);-- for (auto result_id : geo_result_ids) {- // no need to check for existence of `result_id` because of indexer based pre-filtering above- int64_t lat_lng = sort_field_index->at(result_id);- S2LatLng s2_lat_lng;- GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);- if (query_region->Contains(s2_lat_lng.ToPoint())) {- exact_geo_result_ids.push_back(result_id);- }- }- } else {- spp::sparse_hash_map<uint32_t, int64_t*>* geo_field_index = geo_array_index.at(f.name);-- for (auto result_id : geo_result_ids) {- int64_t* lat_lngs = geo_field_index->at(result_id);-- bool point_found = false;-- // any one point should exist- for (size_t li = 0; li < lat_lngs[0]; li++) {- int64_t lat_lng = lat_lngs[li + 1];- S2LatLng s2_lat_lng;- GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);- if (query_region->Contains(s2_lat_lng.ToPoint())) {- point_found = true;- break;- }- }-- if (point_found) {- exact_geo_result_ids.push_back(result_id);- }- }- }-- uint32_t* out = nullptr;- result_ids_len = ArrayUtils::or_scalar(&exact_geo_result_ids[0], exact_geo_result_ids.size(),- result_ids, result_ids_len, &out);-- delete[] result_ids;- result_ids = out;-- delete query_region;- }- } else if (f.is_string()) {- art_tree* t = search_index.at(a_filter.field_name);-- uint32_t* or_ids = nullptr;- size_t or_ids_size = 0;-- // aggregates IDs across array of filter values and reduces excessive ORing- std::vector<uint32_t> f_id_buff;-- for (const std::string& filter_value : a_filter.values) {- std::vector<void*> posting_lists;-- // there could be multiple tokens in a filter value, which we have to treat as ANDs- // e.g. country: South Africa- Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);-- std::string str_token;- size_t token_index = 0;- std::vector<std::string> str_tokens;-- while (tokenizer.next(str_token, token_index)) {- if (str_token.size() > 100) {- str_token.erase(100);- }- str_tokens.push_back(str_token);-- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),- str_token.length()+1);- if (leaf == nullptr) {- continue;- }-- posting_lists.push_back(leaf->values);- }-- if (posting_lists.size() != str_tokens.size()) {- continue;- }-- if(a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {- // needs intersection + exact matching (unlike CONTAINS)- std::vector<uint32_t> result_id_vec;- posting_t::intersect(posting_lists, result_id_vec, context_ids_length, context_ids);-- if (result_id_vec.empty()) {- continue;- }-- // need to do exact match- uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];- size_t exact_str_ids_size = 0;- std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);-- posting_t::get_exact_matches(posting_lists, f.is_array(), result_id_vec.data(), result_id_vec.size(),- exact_str_ids, exact_str_ids_size);-- if (exact_str_ids_size == 0) {- continue;- }-- for (size_t ei = 0; ei < exact_str_ids_size; ei++) {- f_id_buff.push_back(exact_str_ids[ei]);- }- } else {- // CONTAINS- size_t before_size = f_id_buff.size();- posting_t::intersect(posting_lists, f_id_buff, context_ids_length, context_ids);- if (f_id_buff.size() == before_size) {- continue;- }- }-- if (f_id_buff.size() > 100000 || a_filter.values.size() == 1) {- gfx::timsort(f_id_buff.begin(), f_id_buff.end());- f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());-- uint32_t* out = nullptr;- or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);- delete[] or_ids;- or_ids = out;- std::vector<uint32_t>().swap(f_id_buff); // clears out memory- }- }-- if (!f_id_buff.empty()) {- gfx::timsort(f_id_buff.begin(), f_id_buff.end());- f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());-- uint32_t* out = nullptr;- or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);- delete[] or_ids;- or_ids = out;- std::vector<uint32_t>().swap(f_id_buff); // clears out memory- }-- result_ids = or_ids;- result_ids_len = or_ids_size;- }-- if (a_filter.apply_not_equals) {- auto all_ids = seq_ids->uncompress();- auto all_ids_size = seq_ids->num_ids();-- uint32_t* to_include_ids = nullptr;- size_t to_include_ids_len = 0;-- to_include_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, result_ids,- result_ids_len, &to_include_ids);-- delete[] all_ids;- delete[] result_ids;-- result_ids = to_include_ids;- result_ids_len = to_include_ids_len;-- if (context_ids_length != 0) {- uint32_t *out = nullptr;- result.count = ArrayUtils::and_scalar(context_ids, context_ids_length,- result_ids, result_ids_len, &out);-- delete[] result_ids;-- result.docs = out;- return Option(true);- }- }-- result.docs = result_ids;- result.count = result_ids_len;-- return Option(true);- /*long long int timeMillis =- std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now()- - begin).count();-- LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/-}--void Index::aproximate_numerical_match(num_tree_t* const num_tree,- const NUM_COMPARATOR& comparator,- const int64_t& value,- const int64_t& range_end_value,- uint32_t& filter_ids_length) const {- if (comparator == RANGE_INCLUSIVE) {- num_tree->approx_range_inclusive_search_count(value, range_end_value, filter_ids_length);- return;- }-- if (comparator == NOT_EQUALS) {- uint32_t to_exclude_ids_len = 0;- num_tree->approx_search_count(EQUALS, value, to_exclude_ids_len);-- auto all_ids_size = seq_ids->num_ids();- filter_ids_length += (all_ids_size - to_exclude_ids_len);- return;- }-- num_tree->approx_search_count(comparator, value, filter_ids_length);-}--Option<bool> Index::_approximate_filter_ids(const filter& a_filter,- uint32_t& filter_ids_length,- const std::string& collection_name) const {- if (!a_filter.referenced_collection_name.empty()) {- auto& cm = CollectionManager::get_instance();- auto collection = cm.get_collection(a_filter.referenced_collection_name);- if (collection == nullptr) {- return Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");- }-- return collection->get_approximate_reference_filter_ids(a_filter.field_name, filter_ids_length);- }-- if (a_filter.field_name == "id") {- filter_ids_length = a_filter.values.size();- return Option(true);- }-- if (!field_is_indexed(a_filter.field_name)) {- return Option(true);- }-- field f = search_schema.at(a_filter.field_name);-- if (f.is_integer()) {- auto num_tree = numerical_index.at(f.name);-- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {- const std::string& filter_value = a_filter.values[fi];- auto const value = (int64_t)std::stol(filter_value);-- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {- const std::string& next_filter_value = a_filter.values[fi + 1];- auto const range_end_value = (int64_t)std::stol(next_filter_value);-- aproximate_numerical_match(num_tree, a_filter.comparators[fi], value, range_end_value,- filter_ids_length);- fi++;- } else {- aproximate_numerical_match(num_tree, a_filter.comparators[fi], value, 0, filter_ids_length);- }- }- } else if (f.is_float()) {- auto num_tree = numerical_index.at(a_filter.field_name);+ size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {- const std::string& filter_value = a_filter.values[fi];- float value = (float)std::atof(filter_value.c_str());- int64_t float_int64 = float_to_int64_t(value);+ for (long long n = 0; n < N && n < combination_limit; ++n) {+ RETURN_CIRCUIT_BREAKER- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {- const std::string& next_filter_value = a_filter.values[fi + 1];- auto const range_end_value = float_to_int64_t((float) std::atof(next_filter_value.c_str()));+ // every element in `query_suggestion` contains a token and its associated hits+ std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());- aproximate_numerical_match(num_tree, a_filter.comparators[fi], float_int64, range_end_value,- filter_ids_length);- fi++;- } else {- aproximate_numerical_match(num_tree, a_filter.comparators[fi], float_int64, 0, filter_ids_length);- }- }- } else if (f.is_bool()) {- auto num_tree = numerical_index.at(a_filter.field_name);+ // actual query suggestion preserves original order of tokens in query+ std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());+ uint64 qhash;- size_t value_index = 0;- for (const std::string& filter_value : a_filter.values) {- int64_t bool_int64 = (filter_value == "1") ? 1 : 0;+ uint32_t token_bits = 0;+ uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,+ query_suggestion, syn_orig_num_tokens, token_bits, qhash);- aproximate_numerical_match(num_tree, a_filter.comparators[value_index], bool_int64, 0, filter_ids_length);- value_index++;+ if(query_hashes.find(qhash) != query_hashes.end()) {+ // skip this query since it has already been processed before+ continue;}- } else if (f.is_geopoint()) {- filter_ids_length = 100;- } else if (f.is_string()) {- art_tree* t = search_index.at(a_filter.field_name);-- for (const std::string& filter_value : a_filter.values) {- Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);- std::string str_token;- size_t token_index = 0;+ query_hashes.insert(qhash);- while (tokenizer.next(str_token, token_index)) {- auto const leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),- str_token.length()+1);- if (leaf == nullptr) {- continue;- }+ //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;+ //LOG(INFO) << "n: " << n;- filter_ids_length += posting_t::num_ids(leaf->values);- }+ /*std::stringstream fullq;+ for(const auto& qleaf : actual_query_suggestion) {+ std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);+ fullq << qtok << " ";}- }-- if (a_filter.apply_not_equals) {- auto all_ids_size = seq_ids->num_ids();- filter_ids_length = (all_ids_size - filter_ids_length);- }+ LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;*/- return Option(true);-}+ // Prepare excluded document IDs that we can later remove from the result set+ uint32_t* excluded_result_ids = nullptr;+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);-Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,- uint32_t& approx_filter_ids_length,- const std::string& collection_name) const {- if (root == nullptr) {- return Option(true);- }+ std::vector<void*> posting_lists;- if (root->isOperator) {- uint32_t l_filter_ids_length = 0;- if (root->left != nullptr) {- auto rearrange_op = rearrange_filter_tree(root->left, l_filter_ids_length, collection_name);- if (!rearrange_op.ok()) {- return rearrange_op;- }+ for(auto& query_leaf : query_suggestion) {+ posting_lists.push_back(query_leaf->values);}- uint32_t r_filter_ids_length = 0;- if (root->right != nullptr) {- auto rearrange_op = rearrange_filter_tree(root->right, r_filter_ids_length, collection_name);- if (!rearrange_op.ok()) {- return rearrange_op;- }- }+ result_iter_state_t iter_state(+ excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length+ );- if (root->filter_operator == AND) {- approx_filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);- } else {- approx_filter_ids_length = l_filter_ids_length + r_filter_ids_length;- }+ // We fetch offset positions only for multi token query+ bool single_exact_query_token = false;- if (l_filter_ids_length > r_filter_ids_length) {- std::swap(root->left, root->right);+ if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {+ // does this candidate suggestion token match query token exactly?+ single_exact_query_token = true;}- return Option(true);- }+ if(topster == nullptr) {+ posting_t::block_intersector_t(posting_lists, iter_state)+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {+ id_buff.push_back(seq_id);+ });+ } else {+ posting_t::block_intersector_t(posting_lists, iter_state)+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {+ score_results(sort_fields, searched_queries.size(), field_id, field_is_array,+ total_cost, topster, query_suggestion, groups_processed,+ seq_id, sort_order, field_values, geopoint_indices,- group_limit, group_by_fields, token_bits,++ group_limit, group_by_fields, group_missing_values, token_bits,+ prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);- _approximate_filter_ids(root->filter_exp, approx_filter_ids_length, collection_name);- return Option(true);-}+ id_buff.push_back(seq_id);+ });+ }-Option<bool> Index::recursive_filter(filter_node_t* const root,- filter_result_t& result,- const std::string& collection_name,- const uint32_t& context_ids_length,- uint32_t* const& context_ids) const {- if (root == nullptr) {- return Option(true);- }+ delete [] excluded_result_ids;+ const size_t num_result_ids = id_buff.size();- if (root->isOperator) {- filter_result_t l_result;- if (root->left != nullptr) {- auto filter_op = recursive_filter(root->left, l_result , collection_name, context_ids_length, context_ids);- if (!filter_op.ok()) {- return filter_op;- }- }+ if(id_buff.size() > 100000) {+ // prevents too many ORs during exhaustive searching+ std::sort(id_buff.begin(), id_buff.end());+ id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());- filter_result_t r_result;- if (root->right != nullptr) {- auto filter_op = recursive_filter(root->right, r_result , collection_name, context_ids_length, context_ids);- if (!filter_op.ok()) {- return filter_op;- }+ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],+ id_buff.size(), &new_all_result_ids);+ delete[] *all_result_ids;+ *all_result_ids = new_all_result_ids;+ id_buff.clear();}- if (root->filter_operator == AND) {- filter_result_t::and_filter_results(l_result, r_result, result);- } else {- filter_result_t::or_filter_results(l_result, r_result, result);+ if(num_result_ids == 0) {+ continue;}- return Option(true);+ field_num_results += num_result_ids;+ searched_queries.push_back(actual_query_suggestion);}+}- return do_filtering(root, result, collection_name, context_ids_length, context_ids);+bool Index::field_is_indexed(const std::string& field_name) const {+ return search_index.count(field_name) != 0 ||+ numerical_index.count(field_name) != 0 ||+ range_index.count(field_name) != 0 ||+ geo_range_index.count(field_name) != 0;}Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root,@@@ -2280,10 -2783,10 +2296,11 @@@ Option<bool> Index::search(std::vector<std::vector<uint32_t> included_ids_vec;std::unordered_set<uint32_t> excluded_group_ids;- process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, filter_curated_hits,- process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, group_missing_values,- filter_curated_hits,- filter_result.docs, filter_result.count, curated_ids, included_ids_map,++ process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit,++ group_missing_values, filter_curated_hits,+ filter_result_iterator, curated_ids, included_ids_map,included_ids_vec, excluded_group_ids);+ filter_result_iterator->reset();std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());@@@ -2318,24 -2819,14 +2335,25 @@@// handle phrase searchesif (!field_query_tokens[0].q_phrases.empty()) {- do_phrase_search(num_search_fields, the_fields, field_query_tokens,- sort_fields_std, searched_queries, group_limit, group_by_fields, group_missing_values,- topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,- all_result_ids, all_result_ids_len, groups_processed, curated_ids,- excluded_result_ids, excluded_result_ids_size, excluded_group_ids, curated_topster,- included_ids_map, is_wildcard_query,- filter_result.docs, filter_result.count);- if (filter_result.count == 0) {+ auto do_phrase_search_op = do_phrase_search(num_search_fields, the_fields, field_query_tokens,+ sort_fields_std, searched_queries, group_limit, group_by_fields,++ group_missing_values,+ topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,+ filter_result_iterator, all_result_ids, all_result_ids_len,+ groups_processed, curated_ids,+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,+ curated_topster, included_ids_map, is_wildcard_query,+ collection_name);++ filter_iterator_guard.release();+ filter_iterator_guard.reset(filter_result_iterator);++ if (!do_phrase_search_op.ok()) {+ delete [] all_result_ids;+ return do_phrase_search_op;+ }++ if (filter_result_iterator->approx_filter_ids_length == 0) {goto process_search_results;}}@@@ -2352,10 -2846,10 +2370,10 @@@uint32_t seq_id = it.id();uint64_t distinct_id = seq_id;if (group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {- continue;- }+ continue;+ }}int64_t scores[3] = {0};@@@ -2472,14 -2946,14 +2490,14 @@@uint64_t distinct_id = seq_id;if (group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {- continue;- }+ continue;+ }}- auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :- dist_label.first;+ auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_result.first) :+ dist_result.first;if(vec_dist_score > vector_query.distance_threshold) {continue;@@@ -2512,29 -2981,14 +2530,31 @@@all_result_ids_len = nearest_ids.size();}} else {- search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,- curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,- group_missing_values, curated_ids, curated_ids_sorted,- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,- all_result_ids, all_result_ids_len,- filter_result.docs, filter_result.count, concurrency,- sort_order, field_values, geopoint_indices);+ // if filters were not provided, use the seq_ids index to generate the list of all document ids+ if (no_filters_provided) {+ filter_result_iterator = new filter_result_iterator_t(seq_ids->uncompress(), seq_ids->num_ids());+ filter_iterator_guard.reset(filter_result_iterator);+ }++ auto search_wildcard_op = search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,+ curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,++ group_missing_values,+ curated_ids, curated_ids_sorted,+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,+ all_result_ids, all_result_ids_len,+ filter_result_iterator, concurrency,+ sort_order, field_values, geopoint_indices, collection_name);+ if (!search_wildcard_op.ok()) {+ return search_wildcard_op;+ }+++ filter_result_iterator->reset();}++ uint32_t _all_result_ids_len = all_result_ids_len;+ curate_filtered_ids(curated_ids, excluded_result_ids,+ excluded_result_ids_size, all_result_ids, _all_result_ids_len, curated_ids_sorted);+ all_result_ids_len = _all_result_ids_len;} else {// Non-wildcard// In multi-field searches, a record can be matched across different fields, so we use this for aggregation@@@ -2573,21 -3027,16 +2593,22 @@@}}- fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type, excluded_result_ids,- excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,- excluded_group_ids,- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields,- group_missing_values, prioritize_exact_match,- prioritize_token_position, prioritize_num_matching_fields, query_hashes, token_order, prefixes,- typo_tokens_threshold, exhaustive_search,- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,- field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type,+ excluded_result_ids, excluded_result_ids_size,+ filter_result_iterator, curated_ids_sorted,+ excluded_group_ids, sort_fields_std, num_typos,+ searched_queries, qtoken_set, topster, groups_processed,+ all_result_ids, all_result_ids_len,- group_limit, group_by_fields, prioritize_exact_match,- prioritize_token_position, query_hashes, token_order, prefixes,++ group_limit, group_by_fields, group_missing_values, prioritize_exact_match,++ prioritize_token_position, prioritize_num_matching_fields,++ query_hashes, token_order, prefixes,+ typo_tokens_threshold, exhaustive_search,+ max_candidates, min_len_1typo, min_len_2typo,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,+ collection_name);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }// try split/joining tokens if no results are foundif(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {@@@ -2617,40 -3066,33 +2638,43 @@@space_resolved_queries[0][j].size(), 0);}- fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,- excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,- excluded_group_ids,- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields,- group_missing_values, prioritize_exact_match,- prioritize_token_position, prioritize_num_matching_fields,- query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,- field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,+ excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,+ excluded_group_ids,+ sort_fields_std, num_typos, searched_queries,+ qtoken_set, topster, groups_processed,+ all_result_ids, all_result_ids_len,- group_limit, group_by_fields, prioritize_exact_match,- prioritize_token_position, query_hashes, token_order,++ group_limit, group_by_fields, group_missing_values,++ prioritize_exact_match, prioritize_token_position,++ prioritize_num_matching_fields,++ query_hashes, token_order,+ prefixes, typo_tokens_threshold, exhaustive_search,+ max_candidates, min_len_1typo, min_len_2typo,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,+ collection_name);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }}}// do synonym based searches- do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map, sort_fields_std,- curated_topster, token_order,- 0, group_limit, group_by_fields, group_missing_values,- prioritize_exact_match, prioritize_token_position,- prioritize_num_matching_fields,- exhaustive_search, concurrency, prefixes,- min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,- topster, q_pos_synonyms, syn_orig_num_tokens,- groups_processed, searched_queries, all_result_ids, all_result_ids_len,- filter_result.docs, filter_result.count, query_hashes,- sort_order, field_values, geopoint_indices,- qtoken_set);+ auto do_synonym_search_op = do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map,+ sort_fields_std, curated_topster, token_order, 0, group_limit,- group_by_fields, prioritize_exact_match, prioritize_token_position,- exhaustive_search, concurrency, prefixes,++ group_by_fields, group_missing_values, prioritize_exact_match, prioritize_token_position,++ prioritize_num_matching_fields, exhaustive_search, concurrency, prefixes,+ min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,+ topster, q_pos_synonyms, syn_orig_num_tokens,+ groups_processed, searched_queries, all_result_ids, all_result_ids_len,+ filter_result_iterator, query_hashes,+ sort_order, field_values, geopoint_indices,+ qtoken_set, collection_name);+ if (!do_synonym_search_op.ok()) {+ return do_synonym_search_op;+ }+++ filter_result_iterator->reset();// gather up both original query and synonym queries and do drop tokens@@@ -2705,22 -3139,17 +2729,23 @@@drop_token_prefixes.push_back(p && prefix_search);}- fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,- excluded_result_ids, excluded_result_ids_size,- filter_result.docs, filter_result.count,- curated_ids_sorted, excluded_group_ids, sort_fields_std, num_typos, searched_queries,- qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields,- group_missing_values,- prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,- query_hashes, token_order, prefixes, typo_tokens_threshold,- exhaustive_search, max_candidates, min_len_1typo,- min_len_2typo, -1, sort_order, field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,+ excluded_result_ids, excluded_result_ids_size,+ filter_result_iterator,+ curated_ids_sorted, excluded_group_ids,+ sort_fields_std, num_typos, searched_queries,- qtoken_set, topster, groups_processed,++ qtoken_set, topster, groups_processed,+ all_result_ids, all_result_ids_len,- group_limit, group_by_fields,- prioritize_exact_match, prioritize_token_position, query_hashes,++ group_limit, group_by_fields, group_missing_values,++ prioritize_exact_match, prioritize_token_position,++ prioritize_num_matching_fields, query_hashes,+ token_order, prefixes, typo_tokens_threshold,+ exhaustive_search, max_candidates, min_len_1typo,+ min_len_2typo, -1, sort_order, field_values, geopoint_indices,+ collection_name);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }} else {break;@@@ -2729,20 -3158,13 +2754,21 @@@}}- do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std, searched_queries,- group_limit, group_by_fields, group_missing_values,- max_extra_prefix, max_extra_suffix,- field_query_tokens[0].q_include_tokens,- topster, filter_result.docs, filter_result.count,- sort_order, field_values, geopoint_indices,- curated_ids_sorted, excluded_group_ids, all_result_ids, all_result_ids_len, groups_processed);+ auto do_infix_search_op = do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std,+ searched_queries,- group_limit, group_by_fields,++ group_limit, group_by_fields, group_missing_values,+ max_extra_prefix, max_extra_suffix,+ field_query_tokens[0].q_include_tokens,+ topster, filter_result_iterator,+ sort_order, field_values, geopoint_indices,+ curated_ids_sorted, excluded_group_ids,+ all_result_ids, all_result_ids_len, groups_processed,+ collection_name);+ if (!do_infix_search_op.ok()) {+ return do_infix_search_op;+ }+++ filter_result_iterator->reset();if(!vector_query.field_name.empty()) {// check at least one of sort fields is text match@@@ -2811,14 -3245,22 +2852,27 @@@for(size_t res_index = 0; res_index < vec_results.size(); res_index++) {auto& vec_result = vec_results[res_index];auto seq_id = vec_result.first;++ filter_result_iterator->skip_to(seq_id);+ auto references = std::move(filter_result_iterator->reference);+ filter_result_iterator->reset();- auto result_it = topster->kv_map.find(seq_id);-- if(result_it != topster->kv_map.end()) {- if(result_it->second->match_score_index < 0 || result_it->second->match_score_index > 2) {+++ KV* found_kv = nullptr;+ if(group_limit != 0) {+ for(auto& kv : kvs) {+ if(kv->key == seq_id) {+ found_kv = kv;+ break;+ }+ }+ } else {+ auto result_it = topster->kv_map.find(seq_id);+ if(result_it != topster->kv_map.end()) {+ found_kv = result_it->second;+ }+ }+ if(found_kv) {+ if(found_kv->match_score_index < 0 || found_kv->match_score_index > 2) {continue;}@@@ -2833,20 -3274,14 +2886,20 @@@((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT));int64_t match_score_index = -1;int64_t scores[3] = {0};-- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, seq_id, 0,- match_score, scores, match_score_index, vec_result.second);++ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,+ geopoint_indices, seq_id, references, 0,+ match_score, scores, match_score_index,+ vec_result.second, collection_name);+ if (!compute_sort_scores_op.ok()) {+ return compute_sort_scores_op;+ }for(int i = 0; i < 3; i++) {- kv->scores[i] = scores[i];+ found_kv->scores[i] = scores[i];}+- kv->match_score_index = match_score_index;+ found_kv->match_score_index = match_score_index;} else {// Result has been found only in vector search: we have to add it to both KV and result_ids@@@ -2870,20 -3298,16 +2923,24 @@@continue;}}-- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));kv.text_match_score = 0;kv.vector_distance = vec_result.second;++ if (filter_result_iterator->is_valid &&+ !filter_result_iterator->reference.empty()) {+ // The doc_id must be valid otherwise it would've been filtered out upstream.+ filter_result_iterator->skip_to(seq_id);+ kv.reference_filter_results = std::move(filter_result_iterator->reference);+ filter_result_iterator->reset();+ }+- topster->add(&kv);+ auto ret = topster->add(&kv);vec_search_ids.push_back(seq_id);++ if(group_limit != 0 && ret < 2) {+ groups_processed[distinct_id]++;+ }}}@@@ -2962,21 -3375,18 +3019,23 @@@num_queued++;thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,- batch_result_ids, batch_res_len, &facet_infos,- estimate_facets, facet_sample_percent, group_missing_values,+ batch_result_ids, batch_res_len, &facet_infos, max_facet_values,- is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,++ is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,++ group_missing_values,&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,- &num_processed, &m_process, &cv_process]() {+ &num_processed, &m_process, &cv_process, facet_index_type]() {search_begin_us = parent_search_begin;search_stop_us = parent_search_stop_ms;search_cutoff = parent_search_cutoff;auto fq = facet_query;+ std::vector<std::pair<std::string, uint32_t>> found_docs;do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,- facet_infos, group_limit, group_by_fields,+ facet_infos, group_limit, group_by_fields, group_missing_values,- batch_result_ids, batch_res_len);+ batch_result_ids, batch_res_len, max_facet_values,+ is_wildcard_query, no_filters_provided,+ facet_index_type);++std::unique_lock<std::mutex> lock(m_process);num_processed++;parent_search_cutoff = parent_search_cutoff || search_cutoff;@@@ -3079,16 -3457,13 +3138,16 @@@std::chrono::high_resolution_clock::now() - beginF).count();LOG(INFO) << "Time for faceting: " << timeMillisF;*/}-+ std::vector<std::pair<std::string, uint32_t>> found_docs;std::vector<facet_info_t> facet_infos(facets.size());compute_facet_infos(facets, facet_query, facet_query_num_typos,- &included_ids_vec[0], included_ids_vec.size(), group_by_fields, max_candidates, facet_infos);+ &included_ids_vec[0], included_ids_vec.size(), group_by_fields,+ group_limit, is_wildcard_no_filter_query,+ max_candidates, facet_infos, facet_index_type);do_facets(facets, facet_query, estimate_facets, facet_sample_percent,- facet_infos, group_limit, group_by_fields, &included_ids_vec[0],- facet_infos, group_limit, group_by_fields, group_missing_values,- &included_ids_vec[0], included_ids_vec.size());++ facet_infos, group_limit, group_by_fields, group_missing_values, &included_ids_vec[0],+ included_ids_vec.size(), max_facet_values, is_wildcard_query, no_filters_provided,+ facet_index_type);all_result_ids_len += curated_topster->size;@@@ -3103,8 -3478,9 +3162,9 @@@void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,const std::vector<uint32_t>& excluded_ids,-- const std::vector<std::string>& group_by_fields, const size_t group_limit,++ const std::vector<std::string>& group_by_fields, const size_t group_limit,+ const bool group_missing_values,- const bool filter_curated_hits, const uint32_t* filter_ids, uint32_t filter_ids_length,+ const bool filter_curated_hits, filter_result_iterator_t* const filter_result_iterator,std::set<uint32_t>& curated_ids,std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,std::vector<uint32_t>& included_ids_vec,@@@ -3189,37 -3567,38 +3249,39 @@@}}-void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,- const std::vector<token_t>& query_tokens,- const std::vector<token_t>& dropped_tokens,- const text_match_type_t match_type,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const uint32_t* filter_ids, size_t filter_ids_length,- const std::vector<uint32_t>& curated_ids,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by> & sort_fields,- const std::vector<uint32_t>& num_typos,- std::vector<std::vector<art_leaf*>> & searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t & all_result_ids_len,- const size_t group_limit, const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool prioritize_num_matching_fields,- std::set<uint64>& query_hashes,- const token_ordering token_order,- const std::vector<bool>& prefixes,- const size_t typo_tokens_threshold,- const bool exhaustive_search,- const size_t max_candidates,- size_t min_len_1typo,- size_t min_len_2typo,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const {+Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,+ const std::vector<token_t>& query_tokens,+ const std::vector<token_t>& dropped_tokens,+ const text_match_type_t match_type,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ filter_result_iterator_t* const filter_result_iterator,+ const std::vector<uint32_t>& curated_ids,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by> & sort_fields,+ const std::vector<uint32_t>& num_typos,+ std::vector<std::vector<art_leaf*>> & searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t & all_result_ids_len,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ bool prioritize_exact_match,+ const bool prioritize_token_position,++ const bool prioritize_num_matching_fields,+ std::set<uint64>& query_hashes,+ const token_ordering token_order,+ const std::vector<bool>& prefixes,+ const size_t typo_tokens_threshold,+ const bool exhaustive_search,+ const size_t max_candidates,+ size_t min_len_1typo,+ size_t min_len_2typo,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::string& collection_name) const {// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens@@@ -3449,21 -3825,17 +3511,21 @@@if(token_candidates_vec.size() == query_tokens.size()) {std::vector<uint32_t> id_buff;-- search_all_candidates(num_search_fields, match_type, the_fields, filter_ids, filter_ids_length,- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,- sort_fields, token_candidates_vec, searched_queries, qtoken_set,- dropped_tokens, topster,- groups_processed, all_result_ids, all_result_ids_len,- typo_tokens_threshold, group_limit, group_by_fields,- group_missing_values, query_tokens,- num_typos, prefixes, prioritize_exact_match, prioritize_token_position,- prioritize_num_matching_fields, exhaustive_search, max_candidates,- syn_orig_num_tokens, sort_order, field_values, geopoint_indices,- query_hashes, id_buff);+ auto search_all_candidates_op = search_all_candidates(num_search_fields, match_type, the_fields,+ filter_result_iterator,+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,+ sort_fields, token_candidates_vec, searched_queries, qtoken_set,+ dropped_tokens, topster,+ groups_processed, all_result_ids, all_result_ids_len,- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,++ typo_tokens_threshold, group_limit, group_by_fields,++ group_missing_values, query_tokens,+ num_typos, prefixes, prioritize_exact_match, prioritize_token_position,- exhaustive_search, max_candidates,++ prioritize_num_matching_fields, exhaustive_search, max_candidates,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,+ query_hashes, id_buff, collection_name);+ if (!search_all_candidates_op.ok()) {+ return search_all_candidates_op;+ }if(id_buff.size() > 1) {gfx::timsort(id_buff.begin(), id_buff.end());@@@ -3604,32 -3972,33 +3666,34 @@@ void Index::find_across_fields(const to}}-void Index::search_across_fields(const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- const std::vector<search_field_t>& the_fields,- const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<sort_by>& sort_fields,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const bool prioritize_exact_match,- const bool prioritize_token_position,- const bool prioritize_num_matching_fields,- const uint32_t* filter_ids, uint32_t filter_ids_length,- const uint32_t total_cost, const int syn_orig_num_tokens,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::vector<uint32_t>& id_buff,- uint32_t*& all_result_ids, size_t& all_result_ids_len) const {+Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ const std::vector<search_field_t>& the_fields,+ const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<sort_by>& sort_fields,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ const bool prioritize_exact_match,+ const bool prioritize_token_position,++ const bool prioritize_num_matching_fields,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t total_cost, const int syn_orig_num_tokens,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::vector<uint32_t>& id_buff,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const std::string& collection_name) const {std::vector<art_leaf*> query_suggestion;@@@ -3831,10 -4197,10 +3895,10 @@@uint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {- return;- }+ return;+ }}int64_t scores[3] = {0};@@@ -4454,34 -4520,34 +4522,35 @@@ Option<bool> Index::compute_sort_scoresscores[2] = -scores[2];}}-}-void Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,- std::vector<query_tokens_t>& field_query_tokens,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- Topster* actual_topster,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- const std::set<uint32_t>& curated_ids,- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* curated_topster,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- bool is_wildcard_query,- uint32_t*& filter_ids, uint32_t& filter_ids_length) const {+ return Option<bool>(true);+}+Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,+ std::vector<query_tokens_t>& field_query_tokens,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,+ Topster* actual_topster,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ filter_result_iterator_t*& filter_result_iterator,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ const std::set<uint32_t>& curated_ids,+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* curated_topster,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ bool is_wildcard_query, const std::string& collection_name) const {++ uint32_t* phrase_result_ids = nullptr;+ uint32_t phrase_result_count = 0;std::map<uint32_t, size_t> phrase_match_id_scores;- uint32_t* phrase_match_ids = nullptr;- size_t phrase_match_ids_size = 0;-for(size_t i = 0; i < num_search_fields; i++) {const std::string& field_name = search_fields[i].name;const size_t field_weight = search_fields[i].weight;@@@ -4622,79 -4695,75 +4691,87 @@@BREAK_CIRCUIT_BREAKER}}+ filter_result_iterator->reset();searched_queries.push_back({});+ return Option<bool>(true);}-void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,- const text_match_type_t match_type,- filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,- const token_ordering& token_order,- const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool prioritize_num_matching_fields,- const bool exhaustive_search, const size_t concurrency,- const std::vector<bool>& prefixes,- size_t min_len_1typo,- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* actual_topster,- std::vector<std::vector<token_t>>& q_pos_synonyms,- int syn_orig_num_tokens,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const uint32_t* filter_ids, const uint32_t filter_ids_length,- std::set<uint64>& query_hashes,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- tsl::htrie_map<char, token_leaf>& qtoken_set) const {+Option<bool> Index::do_synonym_search(const std::vector<search_field_t>& the_fields,+ const text_match_type_t match_type,+ filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,+ const token_ordering& token_order,+ const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,++ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,++ bool prioritize_exact_match,+ const bool prioritize_token_position,++ const bool prioritize_num_matching_fields,+ const bool exhaustive_search, const size_t concurrency,+ const std::vector<bool>& prefixes,+ size_t min_len_1typo,+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* actual_topster,+ std::vector<std::vector<token_t>>& q_pos_synonyms,+ int syn_orig_num_tokens,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ std::set<uint64>& query_hashes,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::string& collection_name) const {for (const auto& syn_tokens : q_pos_synonyms) {query_hashes.clear();- fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,- exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted, excluded_group_ids,- sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields,- group_missing_values, prioritize_exact_match,- prioritize_token_position, prioritize_num_matching_fields,- query_hashes, token_order, prefixes, typo_tokens_threshold,- exhaustive_search, max_candidates, min_len_1typo,- min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,+ exclude_token_ids_size, filter_result_iterator,+ curated_ids_sorted, excluded_group_ids, sort_fields_std, {0},+ searched_queries, qtoken_set, actual_topster, groups_processed,+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,- prioritize_exact_match, prioritize_token_position, query_hashes,++ group_missing_values,++ prioritize_exact_match, prioritize_token_position,++ prioritize_num_matching_fields,++ query_hashes,+ token_order, prefixes, typo_tokens_threshold, exhaustive_search,+ max_candidates, min_len_1typo, min_len_2typo,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,+ collection_name);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }}collate_included_ids({}, included_ids_map, curated_topster, searched_queries);+ return Option<bool>(true);}-void Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,- const std::vector<enable_t>& infixes,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const size_t max_extra_prefix,- const size_t max_extra_suffix,- const std::vector<token_t>& query_tokens, Topster* actual_topster,- const uint32_t *filter_ids, size_t filter_ids_length,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {+Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,+ const std::vector<enable_t>& infixes,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,++ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,++ const size_t max_extra_prefix,+ const size_t max_extra_suffix,+ const std::vector<token_t>& query_tokens, Topster* actual_topster,+ filter_result_iterator_t* const filter_result_iterator,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ const std::string& collection_name) const {for(size_t field_id = 0; field_id < num_search_fields; field_id++) {auto& field_name = the_fields[field_id].name;@@@ -4759,15 -4819,14 +4836,15 @@@uint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {- continue;- }+ continue;+ }}- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = actual_topster->add(&kv);+if(group_limit != 0 && ret < 2) {groups_processed[distinct_id]++;}@@@ -5047,27 -5078,24 +5124,29 @@@ void Index::curate_filtered_ids(const s}}-void Index::search_wildcard(filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool group_missing_values,- const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,- uint32_t filter_ids_length, const size_t concurrency,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const {+Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,++ const std::vector<std::string>& group_by_fields,++ const bool group_missing_values,++ const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ const size_t concurrency,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::string& collection_name) const {++ filter_result_iterator->compute_result();+ auto const& approx_filter_ids_length = filter_result_iterator->approx_filter_ids_length;uint32_t token_bits = 0;- const bool check_for_circuit_break = (filter_ids_length > 1000000);+ const bool check_for_circuit_break = (approx_filter_ids_length > 1000000);//auto beginF = std::chrono::high_resolution_clock::now();@@@ -5101,16 -5131,15 +5180,17 @@@searched_queries.push_back({});topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);+ auto& compute_sort_score_status = compute_sort_score_statuses[thread_id] = nullptr;thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,- thread_id, &sort_fields, &searched_queries,- &group_limit, &group_by_fields, group_missing_values,- &topsters, &tgroups_processed, &excluded_group_ids,- &sort_order, field_values, &geopoint_indices, &plists,- check_for_circuit_break,- batch_result_ids, batch_res_len,- &num_processed, &m_process, &cv_process]() {+ thread_id, &sort_fields, &searched_queries,- &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,++ &group_limit, &group_by_fields, group_missing_values,++ &topsters, &tgroups_processed, &excluded_group_ids,+ &sort_order, field_values, &geopoint_indices, &plists,+ check_for_circuit_break,+ batch_result,+ &num_processed, &m_process, &cv_process, &compute_sort_score_status, collection_name]() {+ std::unique_ptr<filter_result_t> batch_result_guard(batch_result);search_begin_us = parent_search_begin;search_stop_us = parent_search_stop_ms;@@@ -5143,15 -5162,15 +5223,15 @@@uint64_t distinct_id = seq_id;if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);if(excluded_group_ids.count(distinct_id) != 0) {- continue;- }+ continue;+ }}- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- int ret = topsters[thread_id]->add(&kv);+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));+ int ret = topsters[thread_id]->add(&kv);if(group_limit != 0 && ret < 2) {tgroups_processed[thread_id][distinct_id]++;}@@@ -5802,37 -5783,48 +5885,38 @@@ void Index::score_results(const std::ve// pre-filter group_by_fields such that we can avoid the find() checkuint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,- const uint32_t seq_id) const {- const uint32_t seq_id, const bool group_missing_values) const {- uint64_t distinct_id = 1;- bool found_group_value = false;++ const uint32_t seq_id, const bool group_missing_values) const {+ uint32_t distinct_id = 1; // some constant initial value// calculate hash from group_by_fields- for(const auto& field: group_by_fields) {- const auto& field_facet_mapping_it = facet_index_v3.find(field);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ for(const auto& field_name: group_by_fields) {+ if(!facet_index_v4->has_hash_index(field_name)) {continue;}- if(search_schema.at(field).is_array()) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);-- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }+ posting_list_t* facet_hash_index = facet_index_v4->get_facet_hash_index(field_name);+ std::vector<uint32_t> facet_hashes;+ posting_list_t::iterator_t facet_index_it = facet_hash_index->new_iterator();+ facet_index_it.skip_to(seq_id);- if(facet_index_it.valid()) {- const auto& facet_hashes = facet_hashes_it->second;-- for(size_t i = 0; i < facet_hashes.size(); i++) {- distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);- found_group_value = true;- }- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);++ if(facet_index_it.valid() && facet_index_it.id() == seq_id) {+ posting_list_t::get_offsets(facet_index_it, facet_hashes);- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- continue;+ if(search_schema.at(field_name).is_array()) {+ //LOG(INFO) << "combining hashes for facet array ";+ for(size_t i = 0; i < facet_hashes.size(); i++) {+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes[i]);+ }+ } else {+ const auto& facet_hash = facet_hashes[0];+ //LOG(INFO) << "combining hashes for facet ";+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);}-- const auto& facet_hash = facet_hashes_it->second;- distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);- found_group_value = true;}}- //LOG(INFO) << "distinct id " << distinct_id;- return distinct_id;+- return (!found_group_value && !group_missing_values) ? seq_id : distinct_id;++ //LOG(INFO) << "seq_id: " << seq_id << ", distinct_id: " << distinct_id;++ return (distinct_id == 1 && !group_missing_values) ? seq_id : distinct_id;}inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token_candidates_vec,diff --cc test/collection_faceting_test.cppindex 4ff66550,5e9c1e9d..3d95a45f--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@@ -1698,744 -1690,110 +1698,852 @@@ TEST_F(CollectionFacetingTest, FacetOnA}}+TEST_F(CollectionFacetingTest, FloatFieldValueTruncation) {+ std::vector<field> fields = {+ field("tags", field_types::STRING_ARRAY, true),+ field("points", field_types::FLOAT, true),+ };++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["tags"] = {"gamma"};+ doc["points"] = 300;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("*", {},+ "", {"points"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());++ ASSERT_EQ("300", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+}++TEST_F(CollectionFacetingTest, FacetingReturnParent) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value.color", "type": "string", "optional": false, "facet": true },+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "value": {+ "color": "red",+ "r": 255,+ "g": 0,+ "b": 0+ }+ })"_json;++ nlohmann::json doc2 = R"({+ "value": {+ "color": "blue",+ "r": 0,+ "g": 0,+ "b": 255+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto search_op = coll1->search("*", {},"", {"value.color"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"value.color"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["parent"].dump());+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["parent"].dump());+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);++ //not passing facet_fields in facet_return_parent list will only return facet value, not immediate parent for those field+ search_op = coll1->search("*", {},"", {"value.color"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);++ search_op = coll1->search("*", {},"", {"value.color", "value.r"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"value.r"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ results = search_op.get();+ ASSERT_EQ(2, results["facet_counts"].size());++ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);++ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][1]["counts"][0]["parent"].dump());+ ASSERT_EQ("0", results["facet_counts"][1]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][1]["parent"].dump());+ ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);+}++TEST_F(CollectionFacetingTest, FacetingReturnParentDeepNested) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "product.specification.detail.width", "type": "int32", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "product" : {+ "specification": {+ "detail" : {+ "width": 25+ }+ }+ }+ })"_json;++ nlohmann::json doc2 = R"({+ "product" : {+ "specification": {+ "detail" : {+ "width": 30+ }+ }+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto search_op = coll1->search("*", {},"", {"product.specification.detail.width"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"product.specification.detail.width"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":30}}}", results["facet_counts"][0]["counts"][0]["parent"].dump());+ ASSERT_EQ("30", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":25}}}", results["facet_counts"][0]["counts"][1]["parent"].dump());+ ASSERT_EQ("25", results["facet_counts"][0]["counts"][1]["value"]);+}++TEST_F(CollectionFacetingTest, FacetingReturnParentObject) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value", "type": "object", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "value": {+ "color": "red",+ "r": 255,+ "g": 0,+ "b": 0+ }+ })"_json;++ nlohmann::json doc2 = R"({+ "value": {+ "color": "blue",+ "r": 0,+ "g": 0,+ "b": 255+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto search_op = coll1->search("*", {},"", {"value.color"},+ {}, {2}, 10, 1,FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),10, "",+ 30, 4, "",+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},+ 3, "<mark>", "</mark>", {},+ UINT32_MAX, true, false, true,+ "", false, 6000*1000, 4, 7,+ fallback, 4, {off}, INT16_MAX, INT16_MAX,+ 2, 2, false, "",+ true, 0, max_score, 100,+ 0, 0, HASH, 30000,+ 2, "", {"value.color"});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["parent"].dump());+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["parent"].dump());+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);+}+++TEST_F(CollectionFacetingTest, FacetSortByAlpha) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "phone", "type": "string", "optional": false, "facet": true },+ {"name": "brand", "type": "string", "optional": false, "facet": true },+ {"name": "rating", "type": "float", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ nlohmann::json doc;++ doc["phone"] = "Oneplus 11R";+ doc["brand"] = "Oneplus";+ doc["rating"] = 4.6;+ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "Fusion Plus";+ doc["brand"] = "Moto";+ doc["rating"] = 4.2;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "S22 Ultra";+ doc["brand"] = "Samsung";+ doc["rating"] = 4.1;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "GT Master";+ doc["brand"] = "Realme";+ doc["rating"] = 4.4;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "T2";+ doc["brand"] = "Vivo";+ doc["rating"] = 4.0;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "Mi 6";+ doc["brand"] = "Xiaomi";+ doc["rating"] = 3.9;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "Z6 Lite";+ doc["brand"] = "Iqoo";+ doc["rating"] = 4.3;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ //sort facets by phone in asc order+ auto search_op = coll1->search("*", {}, "", {"phone(sort_by:_alpha:asc)"},+ {}, {2});++ if (!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }++ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(7, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Fusion Plus", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("GT Master", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("Mi 6", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("Oneplus 11R", results["facet_counts"][0]["counts"][3]["value"]);+ ASSERT_EQ("S22 Ultra", results["facet_counts"][0]["counts"][4]["value"]);+ ASSERT_EQ("T2", results["facet_counts"][0]["counts"][5]["value"]);+ ASSERT_EQ("Z6 Lite", results["facet_counts"][0]["counts"][6]["value"]);++ //sort facets by brand in desc order+ search_op = coll1->search("*", {}, "", {"brand(sort_by:_alpha:desc)"},+ {}, {2});++ if (!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }++ results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(7, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Xiaomi", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("Vivo", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("Samsung", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("Realme", results["facet_counts"][0]["counts"][3]["value"]);+ ASSERT_EQ("Oneplus", results["facet_counts"][0]["counts"][4]["value"]);+ ASSERT_EQ("Moto", results["facet_counts"][0]["counts"][5]["value"]);+ ASSERT_EQ("Iqoo", results["facet_counts"][0]["counts"][6]["value"]);++ //sort facets by brand in desc order and phone by asc order+ search_op = coll1->search("*", {}, "", {"brand(sort_by:_alpha:desc)",+ "phone(sort_by:_alpha:asc)"},+ {}, {2});++ if (!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }++ results = search_op.get();+ ASSERT_EQ(2, results["facet_counts"].size());++ ASSERT_EQ(7, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Xiaomi", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("Vivo", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("Samsung", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("Realme", results["facet_counts"][0]["counts"][3]["value"]);+ ASSERT_EQ("Oneplus", results["facet_counts"][0]["counts"][4]["value"]);+ ASSERT_EQ("Moto", results["facet_counts"][0]["counts"][5]["value"]);+ ASSERT_EQ("Iqoo", results["facet_counts"][0]["counts"][6]["value"]);++ ASSERT_EQ(7, results["facet_counts"][1]["counts"].size());+ ASSERT_EQ("Fusion Plus", results["facet_counts"][1]["counts"][0]["value"]);+ ASSERT_EQ("GT Master", results["facet_counts"][1]["counts"][1]["value"]);+ ASSERT_EQ("Mi 6", results["facet_counts"][1]["counts"][2]["value"]);+ ASSERT_EQ("Oneplus 11R", results["facet_counts"][1]["counts"][3]["value"]);+ ASSERT_EQ("S22 Ultra", results["facet_counts"][1]["counts"][4]["value"]);+ ASSERT_EQ("T2", results["facet_counts"][1]["counts"][5]["value"]);+ ASSERT_EQ("Z6 Lite", results["facet_counts"][1]["counts"][6]["value"]);+}++TEST_F(CollectionFacetingTest, FacetSortByOtherField) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "receipe", "type": "object", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "receipe": {+ "name": "cheese pizza",+ "calories": 300,+ "origin": "america"+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc2 = R"({+ "receipe": {+ "name": "noodles",+ "calories": 250,+ "origin": "china"+ }+ })"_json;++ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc3 = R"({+ "receipe": {+ "name": "hamburger",+ "calories": 350,+ "origin": "america"+ }+ })"_json;++ add_op = coll1->add(doc3.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc4 = R"({+ "receipe": {+ "name": "schezwan rice",+ "calories": 150,+ "origin": "china"+ }+ })"_json;++ add_op = coll1->add(doc4.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc5 = R"({+ "receipe": {+ "name": "butter chicken",+ "calories": 270,+ "origin": "india"+ }+ })"_json;++ add_op = coll1->add(doc5.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ //search by calories in asc order+ auto search_op = coll1->search("*", {},"",+ {"receipe.name(sort_by:receipe.calories:asc)"},+ {}, {2});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("schezwan rice", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("noodles", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("butter chicken", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("cheese pizza", results["facet_counts"][0]["counts"][3]["value"]);+ ASSERT_EQ("hamburger", results["facet_counts"][0]["counts"][4]["value"]);++ //search by calories in desc order+ search_op = coll1->search("*", {},"",+ {"receipe.name(sort_by:receipe.calories:desc)"},+ {}, {2});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ results = search_op.get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("hamburger", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("cheese pizza", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("butter chicken", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("noodles", results["facet_counts"][0]["counts"][3]["value"]);+ ASSERT_EQ("schezwan rice", results["facet_counts"][0]["counts"][4]["value"]);+}++TEST_F(CollectionFacetingTest, FacetSortByOtherFloatField) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "investment", "type": "object", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ nlohmann::json doc1 = R"({+ "investment": {+ "name": "Term Deposits",+ "interest_rate": 7.1,+ "class": "fixed"+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc2 = R"({+ "investment": {+ "name": "Gold",+ "interest_rate": 5.4,+ "class": "fixed"+ }+ })"_json;++ add_op = coll1->add(doc2.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc3 = R"({+ "investment": {+ "name": "Mutual Funds",+ "interest_rate": 12,+ "class": "Equity"+ }+ })"_json;++ add_op = coll1->add(doc3.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc4 = R"({+ "investment": {+ "name": "Land",+ "interest_rate": 9.1,+ "class": "real estate"+ }+ })"_json;++ add_op = coll1->add(doc4.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ nlohmann::json doc5 = R"({+ "investment": {+ "name": "Bonds",+ "interest_rate": 7.24,+ "class": "g-sec"+ }+ })"_json;++ add_op = coll1->add(doc5.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ //search by calories in asc order+ auto search_op = coll1->search("*", {},"",+ {"investment.name(sort_by:investment.interest_rate:asc)"},+ {}, {2});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ auto results = search_op.get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Gold", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("Term Deposits", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("Bonds", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("Land", results["facet_counts"][0]["counts"][3]["value"]);+ ASSERT_EQ("Mutual Funds", results["facet_counts"][0]["counts"][4]["value"]);++ //search by calories in desc order+ search_op = coll1->search("*", {},"",+ {"investment.name(sort_by:investment.interest_rate:desc)"},+ {}, {2});++ if(!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }+ results = search_op.get();++ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Mutual Funds", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("Land", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("Bonds", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ("Term Deposits", results["facet_counts"][0]["counts"][3]["value"]);+ ASSERT_EQ("Gold", results["facet_counts"][0]["counts"][4]["value"]);+}++TEST_F(CollectionFacetingTest, FacetSortValidation) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "phone", "type": "string", "optional": false, "facet": true },+ {"name": "brand", "type": "string", "optional": false, "facet": true },+ {"name": "rating", "type": "float", "optional": false, "facet": true }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ nlohmann::json doc;++ doc["phone"] = "Oneplus 11R";+ doc["brand"] = "Oneplus";+ doc["rating"] = 4.6;+ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "Fusion Plus";+ doc["brand"] = "Moto";+ doc["rating"] = 4.2;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ doc["phone"] = "S22 Ultra";+ doc["brand"] = "Samsung";+ doc["rating"] = 4.1;+ add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ //try sort on non string field+ auto search_op = coll1->search("*", {}, "", {"rating(sort_by:_alpha:desc)"},+ {}, {2});++ ASSERT_EQ(400, search_op.code());+ ASSERT_EQ("Facet field should be string type to apply alpha sort.", search_op.error());++ //try sort by string field+ search_op = coll1->search("*", {}, "", {"phone(sort_by:brand:desc)"},+ {}, {2});++ ASSERT_EQ(400, search_op.code());+ ASSERT_EQ("Sort field should be non string type to apply sort.", search_op.error());++ //incorrect syntax+ search_op = coll1->search("*", {}, "", {"phone(sort_by:desc)"},+ {}, {2});++ ASSERT_EQ(400, search_op.code());+ ASSERT_EQ("Invalid sort format.", search_op.error());++ search_op = coll1->search("*", {}, "", {"phone(sort:_alpha:desc)"},+ {}, {2});++ ASSERT_EQ(400, search_op.code());+ ASSERT_EQ("Invalid sort format.", search_op.error());++ //invalid param+ search_op = coll1->search("*", {}, "", {"phone(sort_by:_alpha:foo)"},+ {}, {2});++ ASSERT_EQ(400, search_op.code());+ ASSERT_EQ("Invalid sort param.", search_op.error());++ //whitespace is allowed+ search_op = coll1->search("*", {}, "", {"phone( sort_by: _alpha : asc)"},+ {}, {2});++ if (!search_op.ok()) {+ LOG(ERROR) << search_op.error();+ FAIL();+ }++ auto results = search_op.get();+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("Fusion Plus", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("Oneplus 11R", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("S22 Ultra", results["facet_counts"][0]["counts"][2]["value"]);+}++TEST_F(CollectionFacetingTest, FhashInt64MapTest) {- std::vector<int64_t> visitors = {227489798,124098972,180247624};++ std::vector<int64_t> visitors = {227489798, 124098972, 180247624};+ facet_index_t facet_index_v4;+ std::unordered_map<facet_value_id_t, std::vector<uint32_t>, facet_value_id_t::Hash> fvalue_to_seq_ids;+ std::unordered_map<uint32_t, std::vector<facet_value_id_t>> seq_id_to_fvalues;++ facet_index_v4.initialize("visitors");++ //insert timestamps+ int seq_id = 0;- for(auto it = visitors.begin(); it != visitors.end(); ++it) {++ for (auto it = visitors.begin(); it != visitors.end(); ++it) {+ auto val = std::to_string(*it);+ facet_value_id_t facet_value_id(val);+ fvalue_to_seq_ids[facet_value_id].push_back(seq_id);+ seq_id_to_fvalues[seq_id].push_back(facet_value_id);+ ++seq_id;+ }++ facet_index_v4.insert("visitors", fvalue_to_seq_ids, seq_id_to_fvalues);+ ASSERT_EQ(3, facet_index_v4.get_fhash_int64_map("visitors").size());++ facet_index_v4.remove("visitors", 0);+ ASSERT_EQ(2, facet_index_v4.get_fhash_int64_map("visitors").size());++ fvalue_to_seq_ids.clear();+ seq_id_to_fvalues.clear();++ facet_value_id_t facet_value_id("124798721");+ fvalue_to_seq_ids[facet_value_id].push_back(seq_id);+ seq_id_to_fvalues[seq_id].push_back(facet_value_id);++ facet_index_v4.insert("visitors", fvalue_to_seq_ids, seq_id_to_fvalues);+ ASSERT_EQ(3, facet_index_v4.get_fhash_int64_map("visitors").size());- }++}+++ TEST_F(CollectionFacetingTest, RangeFacetTestWithGroupBy) {+ std::vector<field> fields = {field("place", field_types::STRING, false),+ field("state", field_types::STRING, true),+ field("visitors", field_types::INT32, true),+ field("rating", field_types::FLOAT, true),+ field("trackingFrom", field_types::INT32, true),};+ Collection* coll1 = collectionManager.create_collection(+ "coll1", 1, fields, "", 0, "", {}, {}+ ).get();+ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["place"] = "Mysore Palace";+ doc1["state"] = "Karnataka";+ doc1["visitors"] = 235486;+ doc1["rating"] = 4.5;+ doc1["trackingFrom"] = 1900;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["place"] = "Hampi";+ doc2["state"] = "Karnataka";+ doc2["visitors"] = 201022;+ doc2["rating"] = 4.5;+ doc2["trackingFrom"] = 1900;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["place"] = "Mahabalipuram";+ doc3["state"] = "TamilNadu";+ doc3["visitors"] = 174684;+ doc3["rating"] = 3.8;+ doc3["trackingFrom"] = 1900;++ nlohmann::json doc4;+ doc4["id"] = "3";+ doc4["place"] = "Meenakshi Amman Temple";+ doc4["state"] = "TamilNadu";+ doc4["visitors"] = 246676;+ doc4["rating"] = 4.5;+ doc4["trackingFrom"] = 2000;++ nlohmann::json doc5;+ doc5["id"] = "4";+ doc5["place"] = "Staue of Unity";+ doc5["state"] = "Gujarat";+ doc5["visitors"] = 345878;+ doc5["rating"] = 3.8;+ doc5["trackingFrom"] = 2000;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());++ auto result = coll1->search("Karnataka", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true);+ if(!result.ok()) {+ LOG(INFO) << result.error();+ }++ auto results = result.get();++ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_EQ("VeryBusy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());++ //apply group_by+ result = coll1->search("*", {"state"},+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},+ {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 10,+ {}, {}, {"rating"}, 10,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", true);++ if(!result.ok()) {+ LOG(INFO) << result.error();+ }++ results = result.get();++ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_EQ("VeryBusy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_EQ("Busy", results["facet_counts"][0]["counts"][1]["value"].get<std::string>());++ ASSERT_EQ(2, results["grouped_hits"].size());++ ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());+ ASSERT_EQ(3.8, results["grouped_hits"][0]["group_key"][0]);++ ASSERT_EQ(3, results["grouped_hits"][1]["hits"].size());+ ASSERT_EQ(4.5, results["grouped_hits"][1]["group_key"][0]);++ collectionManager.drop_collection("coll1");-}++}diff --cc test/collection_grouping_test.cppindex 1f9843e9,8a4a3330..23815f75--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@@ -606,6 -606,77 +606,79 @@@ TEST_F(CollectionGroupingTest, RepeatedASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());}+ TEST_F(CollectionGroupingTest, ControlMissingValues) {+ std::vector<field> fields = {+ field("brand", field_types::STRING, true, true),+ };++ Collection* coll2 = collectionManager.get_collection("coll2").get();+ if(coll2 == nullptr) {+ coll2 = collectionManager.create_collection("coll2", 1, fields).get();+ }+++ LOG(INFO) << "----------------------";+++ nlohmann::json doc;+ doc["id"] = "0";+ doc["brand"] = "Omega";+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["brand"] = nullptr;+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ doc["id"] = "2";+ doc["brand"] = nullptr;+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ doc["id"] = "3";+ doc["brand"] = "Omega";+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ // disable null value aggregation++ auto res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,+ {true}, 10,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"brand"}, 2,+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,- 100, 0, 0, 30000, 2, true, false).get();++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, false).get();++ ASSERT_EQ(3, res["grouped_hits"].size());+ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());++ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());+ ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());+ ASSERT_EQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());++ ASSERT_EQ(0, res["grouped_hits"][2]["group_key"].size());+ ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());+ ASSERT_EQ("1", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>());++ // with null value aggregation (default)+ res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,+ {true}, 10,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10,+ {}, {}, {"brand"}, 2,+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,- 100, 0, 0, 30000, 2, true, true).get();++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, true).get();++ ASSERT_EQ(2, res["grouped_hits"].size());++ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());++ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());+ ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());+ }+TEST_F(CollectionGroupingTest, SortingOnGroupCount) {std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};diff --cc test/collection_schema_change_test.cppindex c936ce78,7e454860..5030370c--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@@ -1793,3 -1793,46 +1793,45 @@@ TEST_F(CollectionSchemaChangeTest, EmbeASSERT_EQ(0, vec_index.size());ASSERT_EQ(0, vec_index.count("embedding"));}+-+ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterUpdateOldDocs) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "nested", "type": "object"}+ ],+ "enable_nested_fields": true+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ nlohmann::json doc;+ doc["title"] = "hello";+ doc["nested"] = nlohmann::json::object();+ doc["nested"]["hello"] = "world";++ auto add_op = coll->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ nlohmann::json schema_change = R"({+ "fields": [+ {"name": "embedding", "type":"float[]", "embed":{"from": ["title"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ auto schema_change_op = coll->alter(schema_change);+ ASSERT_TRUE(schema_change_op.ok());++ auto search_res = coll->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);++ ASSERT_EQ(1, search_res.get()["found"].get<size_t>());+ ASSERT_EQ(384, search_res.get()["hits"][0]["document"]["embedding"].get<std::vector<float>>().size());+ ASSERT_EQ(1, search_res.get()["hits"][0]["document"]["nested"].size());+ ASSERT_EQ(0, search_res.get()["hits"][0]["document"].count(".flat"));+ ASSERT_EQ(0, search_res.get()["hits"][0]["document"].count("nested.hello"));+ }diff --cc test/collection_specific_more_test.cppindex d6538c86,266eb0a6..c89d761a--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@@ -1856,6 -1846,62 +1856,62 @@@ TEST_F(CollectionSpecificMoreTest, ConsASSERT_EQ("0", res["hits"][1]["document"]["id"].get<std::string>());}+ TEST_F(CollectionSpecificMoreTest, DisableFieldCountForScoring) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "brand", "type": "string"}+ ]+ })"_json;++ Collection *coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["name"] = "Alpha beta gamma";+ doc["brand"] = "Alpha beta gamma";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["name"] = "Alpha beta gamma";+ doc["brand"] = "Theta";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto res_op = coll1->search("beta", {"name", "brand"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,- 100, 0, 0, 30000, 2, true);++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true);+++ auto res = coll1->search("beta", {"name", "brand"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,- 100, 0, 0, 30000, 2, false).get();++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, false).get();++ size_t score1 = std::stoul(res["hits"][0]["text_match_info"]["score"].get<std::string>());+ size_t score2 = std::stoul(res["hits"][1]["text_match_info"]["score"].get<std::string>());+ ASSERT_TRUE(score1 == score2);++ res = coll1->search("beta", {"name", "brand"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,- 100, 0, 0, 30000, 2, true).get();++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true).get();++ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", res["hits"][1]["document"]["id"].get<std::string>());++ score1 = std::stoul(res["hits"][0]["text_match_info"]["score"].get<std::string>());+ score2 = std::stoul(res["hits"][1]["text_match_info"]["score"].get<std::string>());+ ASSERT_TRUE(score1 > score2);+ }+TEST_F(CollectionSpecificMoreTest, NonNestedFieldNameWithDot) {nlohmann::json schema = R"({"name": "coll1",diff --cc test/collection_vector_search_test.cppindex 0c31beae,c3a608ba..34f5a405--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@@ -2038,4 -2019,200 +2038,199 @@@ TEST_F(CollectionVectorTest, TestMultil0, spp::sparse_hash_set<std::string>());ASSERT_TRUE(semantic_results.ok());- }+ }+++ TEST_F(CollectionVectorTest, TestTwoEmbeddingFieldsSamePrefix) {+ nlohmann::json schema = R"({+ "name": "docs",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ },+ {+ "name": "embedding_en",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "title": "john doe"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto semantic_results = coll1->search("john", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>());++ ASSERT_TRUE(semantic_results.ok());+ }++ TEST_F(CollectionVectorTest, TestOneEmbeddingOneKeywordFieldsHaveSamePrefix) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "title_vec",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "title": "john doe"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto keyword_results = coll1->search("john", {"title"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>());++ ASSERT_TRUE(keyword_results.ok());+ }++ TEST_F(CollectionVectorTest, HybridSearchOnlyKeyworMatchDoNotHaveVectorDistance) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "embed": {+ "from": [+ "title"+ ],+ "model_config": {+ "model_name": "ts/e5-small"+ }+ }+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);++ ASSERT_TRUE(collection_create_op.ok());++ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "title": "john doe"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ // hybrid search with empty vector (to pass distance threshold param)+ std::string vec_query = "embedding:([], distance_threshold: 0.05)";++ auto hybrid_results = coll1->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, vec_query);++ ASSERT_TRUE(hybrid_results.ok());++ ASSERT_EQ(1, hybrid_results.get()["hits"].size());+ ASSERT_EQ(0, hybrid_results.get()["hits"][0].count("vector_distance"));+ }+-+ TEST_F(CollectionVectorTest, QueryByNotAutoEmbeddingVectorField) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {+ "name": "title",+ "type": "string"+ },+ {+ "name": "embedding",+ "type": "float[]",+ "num_dim": 384+ }+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(collection_create_op.ok());++ auto coll = collection_create_op.get();++ auto search_res = coll->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,+ fallback,+ 4, {off}, 32767, 32767, 2,+ false, true, "embedding:([0.96826, 0.94, 0.39557, 0.306488])");++ ASSERT_FALSE(search_res.ok());++ ASSERT_EQ("Vector field `embedding` is not an auto-embedding field, do not use `query_by` with it, use `vector_query` instead.", search_res.error());-}++}