Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.26-facets

Kishore Nallance4b8e3e2cfd
diff --cc include/collection.h
index 57107cd2,6e20beb6..920b4597
--- a/include/collection.h
+++ b/include/collection.h
@@@ -492,13 -466,10 +492,15 @@@ public
const size_t facet_sample_percent = 100,
const size_t facet_sample_threshold = 0,
const size_t page_offset = 0,
+ facet_index_type_t facet_index_type = HASH,
const size_t remote_embedding_timeout_ms = 30000,
const size_t remote_embedding_num_tries = 2,
+ const std::string& stopwords_set="",
+ const std::vector<std::string>& facet_return_parent = {},
+ const std::vector<ref_include_fields>& ref_include_fields_vec = {},
- const drop_tokens_mode_t drop_tokens_mode = right_to_left) const;
++ const drop_tokens_mode_t drop_tokens_mode = right_to_left,
+ const bool prioritize_num_matching_fields = true,
+ const bool group_missing_values = true) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --cc include/index.h
index c4e65e16,a987f3f2..a78cf783
--- a/include/index.h
+++ b/include/index.h
@@@ -382,9 -373,8 +388,10 @@@ private
bool estimate_facets, size_t facet_sample_percent,
const std::vector<facet_info_t>& facet_infos,
size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
- const uint32_t* result_ids, size_t results_size) const;
+ const uint32_t* result_ids, size_t results_size,
+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided,
+ facet_index_type_t facet_index_type) const;
bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,
filter_node_t*& filter_tree_root) const;
@@@ -430,36 -421,38 +438,38 @@@
size_t min_len_2typo,
size_t max_candidates) const;
- void search_all_candidates(const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<search_field_t>& the_fields,
- const uint32_t* filter_ids, size_t filter_ids_length,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by>& sort_fields,
- std::vector<tok_candidates>& token_candidates_vec,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const size_t typo_tokens_threshold,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool exhaustive_search,
- const bool prioritize_num_matching_fields,
- const size_t max_candidates,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const;
+ Option<bool> search_all_candidates(const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<search_field_t>& the_fields,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<tok_candidates>& token_candidates_vec,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ const bool exhaustive_search,
++ const bool prioritize_num_matching_fields,
+ const size_t max_candidates,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff, const std::string& collection_name = "") const;
void search_candidates(const uint8_t & field_id,
bool field_is_array,
@@@ -675,7 -705,8 +691,10 @@@ public
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
const std::vector<char>& symbols_to_index,
- const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true);
- const bool do_validation, const size_t remote_embedding_batch_size = 200, const bool generate_embeddings = true,
- const bool use_addition_fields = false, const tsl::htrie_map<char, field>& addition_fields = {});
++ const bool do_validation, const size_t remote_embedding_batch_size = 200,
++ const bool generate_embeddings = true,
++ const bool use_addition_fields = false,
++ const tsl::htrie_map<char, field>& addition_fields = {});
void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);
@@@ -704,21 -752,21 +723,22 @@@
// the following methods are not synchronized because their parent calls are synchronized or they are const/static
- void search_wildcard(filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
- uint32_t filter_ids_length, const size_t concurrency,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const;
+ Option<bool> search_wildcard(filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
++ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ const size_t concurrency,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name = "") const;
void search_infix(const std::string& query, const std::string& field_name, std::vector<uint32_t>& ids,
size_t max_extra_prefix, size_t max_extra_suffix) const;
@@@ -758,103 -797,107 +778,111 @@@
const std::vector<search_field_t>& search_fields, uint32_t*& exclude_token_ids,
size_t& exclude_token_ids_size) const;
- void do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
- const std::vector<enable_t>& infixes,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values, const size_t max_extra_prefix,
- const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,
- const uint32_t *filter_ids, size_t filter_ids_length,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;
-
- void do_synonym_search(const std::vector<search_field_t>& the_fields,
- const text_match_type_t match_type,
- filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
- const token_ordering& token_order,
- const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool prioritize_num_matching_fields,
- const bool exhaustive_search, const size_t concurrency,
- const std::vector<bool>& prefixes,
- size_t min_len_1typo,
- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* actual_topster,
- std::vector<std::vector<token_t>>& q_pos_synonyms,
- int syn_orig_num_tokens,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
- std::set<uint64>& query_hashes,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- tsl::htrie_map<char, token_leaf>& qtoken_set) const;
-
- void do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
- std::vector<query_tokens_t>& field_query_tokens,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- Topster* actual_topster,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- const std::set<uint32_t>& curated_ids,
- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* curated_topster,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- bool is_wildcard_query,
- uint32_t*& filter_ids, uint32_t& filter_ids_length) const;
-
- void fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
- const std::vector<token_t>& query_tokens,
- const std::vector<token_t>& dropped_tokens,
- const text_match_type_t match_type,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const uint32_t* filter_ids, size_t filter_ids_length,
- const std::vector<uint32_t>& curated_ids,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by>& sort_fields,
- const std::vector<uint32_t>& num_typos,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const size_t group_limit, const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool prioritize_num_matching_fields,
- std::set<uint64>& query_hashes,
- const token_ordering token_order,
- const std::vector<bool>& prefixes,
- const size_t typo_tokens_threshold,
- const bool exhaustive_search,
- const size_t max_candidates,
- size_t min_len_1typo,
- size_t min_len_2typo,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const;
+ Option<bool> do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
+ const std::vector<enable_t>& infixes,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
++ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
++ const size_t max_extra_prefix,
+ const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,
+ filter_result_iterator_t* const filter_result_iterator,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::string& collection_name = "") const;
+
+ [[nodiscard]] Option<bool> do_synonym_search(const std::vector<search_field_t>& the_fields,
+ const text_match_type_t match_type,
+ filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
+ const token_ordering& token_order,
+ const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
++ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
++ bool prioritize_exact_match,
+ const bool prioritize_token_position,
++ const bool prioritize_num_matching_fields,
+ const bool exhaustive_search, const size_t concurrency,
+ const std::vector<bool>& prefixes,
+ size_t min_len_1typo,
+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* actual_topster,
+ std::vector<std::vector<token_t>>& q_pos_synonyms,
+ int syn_orig_num_tokens,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ std::set<uint64>& query_hashes,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::string& collection_name = "") const;
+
+ Option<bool> do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
+ std::vector<query_tokens_t>& field_query_tokens,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ Topster* actual_topster,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ filter_result_iterator_t*& filter_result_iterator,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::set<uint32_t>& curated_ids,
+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* curated_topster,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ bool is_wildcard_query, const std::string& collection_name = "") const;
+
+ [[nodiscard]] Option<bool> fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<token_t>& dropped_tokens,
+ const text_match_type_t match_type,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ filter_result_iterator_t* const filter_result_iterator,
+ const std::vector<uint32_t>& curated_ids,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by>& sort_fields,
+ const std::vector<uint32_t>& num_typos,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
++ const bool prioritize_num_matching_fields,
+ std::set<uint64>& query_hashes,
+ const token_ordering token_order,
+ const std::vector<bool>& prefixes,
+ const size_t typo_tokens_threshold,
+ const bool exhaustive_search,
+ const size_t max_candidates,
+ size_t min_len_1typo,
+ size_t min_len_2typo,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name = "") const;
void find_across_fields(const token_t& previous_token,
const std::string& previous_token_str,
@@@ -866,34 -909,36 +894,36 @@@
std::vector<uint32_t>& prev_token_doc_ids,
std::vector<size_t>& top_prefix_field_ids) const;
- void search_across_fields(const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- const std::vector<search_field_t>& the_fields,
- const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<sort_by>& sort_fields,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- bool prioritize_exact_match,
- const bool search_all_candidates,
- const bool prioritize_num_matching_fields,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
- const uint32_t total_cost,
- const int syn_orig_num_tokens,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::vector<uint32_t>& id_buff,
- uint32_t*& all_result_ids,
- size_t& all_result_ids_len) const;
+ Option<bool> search_across_fields(const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ const std::vector<search_field_t>& the_fields,
+ const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<sort_by>& sort_fields,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ bool prioritize_exact_match,
+ const bool search_all_candidates,
++ const bool prioritize_num_matching_fields,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t total_cost,
+ const int syn_orig_num_tokens,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::vector<uint32_t>& id_buff,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const std::string& collection_name = "") const;
void
search_fields(const std::vector<filter>& filters,
@@@ -923,26 -968,23 +953,27 @@@
filter_node_t*& filter_tree_root,
std::vector<const override_t*>& matched_dynamic_overrides) const;
- void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices, uint32_t seq_id,
- size_t filter_index,
- int64_t max_field_match_score,
- int64_t* scores,
- int64_t& match_score_index, float vector_distance = 0) const;
-
- void
- process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
- const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
- const size_t group_limit, const bool group_missing_values,
- const bool filter_curated_hits, const uint32_t* filter_ids,
- uint32_t filter_ids_length, std::set<uint32_t>& curated_ids,
- std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- std::vector<uint32_t>& included_ids_vec,
- std::unordered_set<uint32_t>& excluded_group_ids) const;
+ Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices, uint32_t seq_id,
+ const std::map<basic_string<char>, reference_filter_result_t>& references,
+ size_t filter_index,
+ int64_t max_field_match_score,
+ int64_t* scores,
+ int64_t& match_score_index, float vector_distance = 0,
+ const std::string& collection_name = "") const;
+
+ void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
- const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
- const size_t group_limit, const bool filter_curated_hits,
++ const std::vector<uint32_t>& excluded_ids,
++ const std::vector<std::string>& group_by_fields,
++ const size_t group_limit, const bool group_missing_values, const bool filter_curated_hits,
+ filter_result_iterator_t* const filter_result_iterator,
+ std::set<uint32_t>& curated_ids,
+ std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ std::vector<uint32_t>& included_ids_vec,
+ std::unordered_set<uint32_t>& excluded_group_ids) const;
+
+ int64_t get_doc_val_from_sort_index(sort_index_iterator it, uint32_t doc_seq_id) const;
Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,
std::vector<uint32_t>& outside_seq_ids);
diff --cc include/topster.h
index 355300ea,fc9d3582..a471564c
--- a/include/topster.h
+++ b/include/topster.h
@@@ -15,11 -14,13 +15,11 @@@ struct KV
uint64_t key{};
uint64_t distinct_key{};
int64_t scores[3]{}; // match score + 2 custom attributes
-
+
// only to be used in hybrid search
- float vector_distance = 2.0f;
+ float vector_distance = -1.0f;
int64_t text_match_score = 0;
- reference_filter_result_t* reference_filter_result = nullptr;
-
// to be used only in final aggregation
uint64_t* query_indices = nullptr;
diff --cc src/collection.cpp
index 12f12e90,717f99ac..9eb782c3
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -1384,13 -1113,10 +1388,15 @@@ Option<nlohmann::json> Collection::sear
const size_t facet_sample_percent,
const size_t facet_sample_threshold,
const size_t page_offset,
+ facet_index_type_t facet_index_type,
const size_t remote_embedding_timeout_ms,
const size_t remote_embedding_num_tries,
+ const std::string& stopwords_set,
+ const std::vector<std::string>& facet_return_parent,
+ const std::vector<ref_include_fields>& ref_include_fields_vec,
- const drop_tokens_mode_t drop_tokens_mode) const {
++ const drop_tokens_mode_t drop_tokens_mode,
+ const bool prioritize_num_matching_fields,
+ const bool group_missing_values) const {
std::shared_lock lock(mutex);
@@@ -4110,10 -3762,11 +4123,11 @@@ Option<bool> Collection::batch_alter_da
// Update schema with additions (deletions can only be made later)
std::vector<field> new_fields;
tsl::htrie_map<char, field> schema_additions;
-
std::vector<std::string> nested_field_names;
-
+ bool found_embedding_field = false;
+ std::unique_lock ulock(mutex);
+
for(auto& f: alter_fields) {
if(f.name == ".*") {
fields.push_back(f);
@@@ -4194,9 -3845,25 +4209,26 @@@
index->remove(seq_id, rec.doc, del_fields, true);
}
}
-
+
- Index::batch_memory_index(index, iter_batch, default_sorting_field, schema_additions, embedding_fields,
- fallback_field_type, token_separators, symbols_to_index, true, 200, false);
+ Index::batch_memory_index(index, iter_batch, default_sorting_field, search_schema, embedding_fields,
- fallback_field_type, token_separators, symbols_to_index, true, 200, found_embedding_field, true, schema_additions);
++ fallback_field_type, token_separators, symbols_to_index, true, 200,
++ found_embedding_field, true, schema_additions);
+ if(found_embedding_field) {
+ for(auto& index_record : iter_batch) {
+ if(index_record.indexed.ok()) {
+ remove_flat_fields(index_record.doc);
+ const std::string& serialized_json = index_record.doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);
+ bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);
+
+ if(!write_ok) {
+ LOG(ERROR) << "Inserting doc with new embedding field failed for seq id: " << index_record.seq_id;
+ index_record.index_failure(500, "Could not write to on-disk storage.");
+ } else {
+ index_record.index_success();
+ }
+ }
+ }
+ }
iter_batch.clear();
}
diff --cc src/collection_manager.cpp
index 78c99a2d,657a906a..537bd9f1
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -976,7 -751,7 +977,8 @@@ Option<bool> CollectionManager::do_sear
const char *FACET_SAMPLE_PERCENT = "facet_sample_percent";
const char *FACET_SAMPLE_THRESHOLD = "facet_sample_threshold";
+ const char *DROP_TOKENS_MODE = "drop_tokens_mode";
+ const char *PRIORITIZE_NUM_MATCHING_FIELDS = "prioritize_num_matching_fields";
// enrich params with values from embedded params
for(auto& item: embedded_params.items()) {
@@@ -1098,7 -863,7 +1101,8 @@@
size_t facet_sample_percent = 100;
size_t facet_sample_threshold = 0;
+ std::string drop_tokens_mode_str = "right_to_left";
+ bool prioritize_num_matching_fields = true;
std::unordered_map<std::string, size_t*> unsigned_int_values = {
{MIN_LEN_1TYPO, &min_len_1typo},
@@@ -1347,13 -1104,11 +1353,16 @@@
facet_sample_percent,
facet_sample_threshold,
offset,
+ HASH,
remote_embedding_timeout_ms,
remote_embedding_num_tries,
+ stopwords_set,
+ facet_return_parent,
+ ref_include_fields_vec,
- drop_tokens_mode);
++ drop_tokens_mode,
+ prioritize_num_matching_fields,
+ group_missing_values
+ );
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --cc src/index.cpp
index 3b68cef9,3b3daea6..63d1ced1
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -514,10 -511,9 +514,10 @@@ void Index::validate_and_preprocess(Ind
}
}
-size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,
+size_t Index::batch_memory_index(Index *index,
+ std::vector<index_record>& iter_batch,
const std::string & default_sorting_field,
- const tsl::htrie_map<char, field> & search_schema,
+ const tsl::htrie_map<char, field> & actual_search_schema,
const tsl::htrie_map<char, field> & embedding_fields,
const std::string& fallback_field_type,
const std::vector<char>& token_separators,
@@@ -1260,15 -1171,8 +1263,16 @@@ void Index::do_facets(std::vector<facet
bool estimate_facets, size_t facet_sample_percent,
const std::vector<facet_info_t>& facet_infos,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ const bool group_missing_values,
- const uint32_t* result_ids, size_t results_size) const {
+ const uint32_t* result_ids, size_t results_size,
+ int max_facet_count, bool is_wildcard_query, bool no_filters_provided,
+ facet_index_type_t facet_index_type) const {
+
+ if(results_size == 0) {
+ return ;
+ }
+
+ size_t total_docs = seq_ids->num_ids();
// assumed that facet fields have already been validated upstream
for(size_t findex=0; findex < facets.size(); findex++) {
@@@ -1283,149 -1186,101 +1287,153 @@@
size_t mod_value = 100 / facet_sample_percent;
- facet_map_t::iterator facet_map_it;
- single_val_facet_map_t::iterator single_facet_map_it;
- uint64_t fhash = 0;
- size_t facet_hash_count = 1;
- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
-
- if((field_facet_mapping_it == facet_index_v3.end())
- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
- continue;
+ auto num_facet_values = facet_index_v4->get_facet_count(facet_field.name);
+ if(num_facet_values == 0) {
+ continue;
}
- for(size_t i = 0; i < results_size; i++) {
- // if sampling is enabled, we will skip a portion of the results to speed up things
- if(estimate_facets) {
- if(i % mod_value != 0) {
- continue;
+ bool is_wildcard_no_filter_query = is_wildcard_query && no_filters_provided;
+ bool facet_value_index_exists = facet_index_v4->has_value_index(facet_field.name);
+
+#ifdef TEST_BUILD
+ if(facet_index_type == VALUE) {
+#else
+ if(facet_value_index_exists && use_value_index) {
+#endif
+ // LOG(INFO) << "Using intersection to find facets";
+ a_facet.is_intersected = true;
+
+ std::map<std::string, docid_count_t> facet_results;
+ std::string sort_order = a_facet.is_sort_by_alpha ? a_facet.sort_order : "";
+
+ facet_index_v4->intersect(a_facet, use_facet_query,
+ facet_infos[findex].fvalue_searched_tokens, result_ids,
+ results_size, max_facet_count, facet_results,
+ is_wildcard_no_filter_query, sort_order);
+
+ for(const auto& kv : facet_results) {
+ //range facet processing
+ if(a_facet.is_range_query) {
+ const auto doc_val = kv.first;
+ std::pair<int64_t , std::string> range_pair {};
+ if(a_facet.get_range(std::stoll(doc_val), range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count = kv.second.count;
+ }
+ } else {
+ facet_count_t& facet_count = a_facet.value_result_map[kv.first];
+ facet_count.count = kv.second.count;
+ facet_count.doc_id = kv.second.doc_id;
}
+
+ if(should_compute_stats) {
+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;
+ for(size_t i = 0; i < kv.second.count; ++i) {
+ compute_facet_stats(a_facet, kv.first, facet_field.type);
+ }
+ }
}
-
- uint32_t doc_seq_id = result_ids[i];
- if(facet_field.is_array()) {
- const auto& field_facet_mapping = field_facet_mapping_it->second;
- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
- facet_hash_count = facet_map_it->second.size();
- } else {
- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
- facet_hash_count = 1;
- fhash = single_facet_map_it->second;
+ } else {
+ //LOG(INFO) << "Using hashing to find facets";
+ bool facet_hash_index_exists = facet_index_v4->has_hash_index(facet_field.name);
+ if(!facet_hash_index_exists) {
+ continue;
}
-
- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields,
- doc_seq_id, group_missing_values) : 0;
- if(((i + 1) % 16384) == 0) {
- RETURN_CIRCUIT_BREAKER
- }
+ const auto& fhash_int64_map = facet_index_v4->get_fhash_int64_map(a_facet.field_name);
- std::set<uint32_t> unique_facet_hashes;
+ const auto facet_field_is_array = facet_field.is_array();
- for(size_t j = 0; j < facet_hash_count; j++) {
-
- if(facet_field.is_array()) {
- fhash = facet_map_it->second.hashes[j];
+ const auto& facet_index = facet_index_v4->get_facet_hash_index(facet_field.name);
+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();
+ std::vector<uint32_t> facet_hashes;
+
+ for(size_t i = 0; i < results_size; i++) {
+ // if sampling is enabled, we will skip a portion of the results to speed up things
+ if(estimate_facets) {
+ if(i % mod_value != 0) {
+ continue;
+ }
}
- if(unique_facet_hashes.count(fhash) == 0) {
- unique_facet_hashes.insert(fhash);
- } else {
+ uint32_t doc_seq_id = result_ids[i];
+ facet_index_it.skip_to(doc_seq_id);
+
+ if(!facet_index_it.valid()) {
+ break;
+ }
+
+ if(facet_index_it.id() != doc_seq_id) {
continue;
}
-
- if(should_compute_stats) {
- compute_facet_stats(a_facet, fhash, facet_field.type);
+
+ facet_hashes.clear();
+ posting_list_t::get_offsets(facet_index_it, facet_hashes);
+
- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
++ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id, group_missing_values) : 0;
+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;
+ if(((i + 1) % 16384) == 0) {
+ RETURN_CIRCUIT_BREAKER
}
- if(a_facet.is_range_query) {
- if(sort_index_it != sort_index.end()){
- auto doc_id_val_map = sort_index_it->second;
- auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);
-
- if(doc_seq_id_it != doc_id_val_map->end()){
- int64_t doc_val = doc_seq_id_it->second;
- std::pair<int64_t, std::string> range_pair {};
- if(a_facet.get_range(doc_val, range_pair)) {
- int64_t range_id = range_pair.first;
- facet_count_t& facet_count = a_facet.result_map[range_id];
- facet_count.count += 1;
-
- if(group_limit) {
- a_facet.hash_groups[range_id].emplace(distinct_id);
- }
+
+ std::set<uint32_t> unique_facet_hashes;
+
+ for(size_t j = 0; j < facet_hashes.size(); j++) {
+
+ const auto& fhash = facet_hashes[j];
+
+ if(unique_facet_hashes.count(fhash) == 0) {
+ unique_facet_hashes.insert(fhash);
+ } else {
+ continue;
+ }
+
+ if(should_compute_stats) {
+ int64_t val = fhash;
+ if(facet_field.is_int64()) {
+ if(fhash_int64_map.find(fhash) != fhash_int64_map.end()) {
+ val = fhash_int64_map.at(fhash);
+ } else {
+ val = INT64_MAX;
}
}
+
+ compute_facet_stats(a_facet, val, facet_field.type);
}
- } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
- facet_count_t& facet_count = a_facet.result_map[fhash];
- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
- facet_count.doc_id = doc_seq_id;
- facet_count.array_pos = j;
- if(group_limit) {
- a_facet.hash_groups[fhash].emplace(distinct_id);
- } else {
- facet_count.count += 1;
- }
- if(use_facet_query) {
- a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
+
+ if(a_facet.is_range_query) {
+ int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);
+
+ std::pair<int64_t , std::string> range_pair {};
+ if(a_facet.get_range(doc_val, range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count += 1;
++
++ if(group_limit) {
++ a_facet.hash_groups[range_id].emplace(distinct_id);
++ }
+ }
+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
+ facet_count_t& facet_count = a_facet.result_map[fhash];
+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
+ facet_count.doc_id = doc_seq_id;
+ facet_count.array_pos = j;
+ if(group_limit) {
+ a_facet.hash_groups[fhash].emplace(distinct_id);
+ } else {
+ facet_count.count += 1;
+ }
+ if(use_facet_query) {
+ //LOG (INFO) << "adding hash tokens for hash " << fhash;
+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
+ }
+ if(!a_facet.sort_field.empty()) {
+ sort_index_it = sort_index.find(a_facet.sort_field);
+ facet_count.sort_field_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);
+ //LOG(INFO) << "found sort_field val " << facet_count.sort_field;
+ }
}
}
}
@@@ -1448,36 -1303,38 +1456,38 @@@ void Index::aggregate_topster(Topster*
}
}
-void Index::search_all_candidates(const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<search_field_t>& the_fields,
- const uint32_t* filter_ids, size_t filter_ids_length,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by>& sort_fields,
- std::vector<tok_candidates>& token_candidates_vec,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const size_t typo_tokens_threshold,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool prioritize_num_matching_fields,
- const bool exhaustive_search,
- const size_t max_candidates,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const {
+Option<bool> Index::search_all_candidates(const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<search_field_t>& the_fields,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<tok_candidates>& token_candidates_vec,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
++ const bool prioritize_num_matching_fields,
+ const bool exhaustive_search,
+ const size_t max_candidates,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff, const std::string& collection_name) const {
/*if(!token_candidates_vec.empty()) {
LOG(INFO) << "Prefix candidates size: " << token_candidates_vec.back().candidates.size();
@@@ -1516,27 -1373,18 +1526,28 @@@
//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
- search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields, match_type,
- sort_fields, topster,groups_processed,
- searched_queries, qtoken_set, dropped_tokens, group_limit, group_by_fields,
- group_missing_values,
- prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,
- filter_ids, filter_ids_length, total_cost, syn_orig_num_tokens,
- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
- sort_order, field_values, geopoint_indices,
- id_buff, all_result_ids, all_result_ids_len);
+ auto search_across_fields_op = search_across_fields(query_suggestion, num_typos, prefixes, the_fields,
+ num_search_fields, match_type,
+ sort_fields, topster,groups_processed,
+ searched_queries, qtoken_set, dropped_tokens,
- group_limit, group_by_fields,
++ group_limit, group_by_fields, group_missing_values,
+ prioritize_exact_match, prioritize_token_position,
++ prioritize_num_matching_fields,
+ filter_result_iterator,
+ total_cost, syn_orig_num_tokens,
+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
+ sort_order, field_values, geopoint_indices,
+ id_buff, all_result_ids, all_result_ids_len,
+ collection_name);
+ if (!search_across_fields_op.ok()) {
+ return search_across_fields_op;
+ }
query_hashes.insert(qhash);
+ filter_result_iterator->reset();
}
+
+ return Option<bool>(true);
}
void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
@@@ -1571,110 -1420,792 +1583,110 @@@
populate_sort_mapping(sort_order, geopoint_indices, sort_fields, field_values);
// escape hatch to prevent too much looping
- size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
-
- for (long long n = 0; n < N && n < combination_limit; ++n) {
- RETURN_CIRCUIT_BREAKER
-
- // every element in `query_suggestion` contains a token and its associated hits
- std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());
-
- // actual query suggestion preserves original order of tokens in query
- std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());
- uint64 qhash;
-
- uint32_t token_bits = 0;
- uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,
- query_suggestion, syn_orig_num_tokens, token_bits, qhash);
-
- if(query_hashes.find(qhash) != query_hashes.end()) {
- // skip this query since it has already been processed before
- continue;
- }
-
- query_hashes.insert(qhash);
-
- //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
- //LOG(INFO) << "n: " << n;
-
- /*std::stringstream fullq;
- for(const auto& qleaf : actual_query_suggestion) {
- std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);
- fullq << qtok << " ";
- }
- LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;*/
-
- // Prepare excluded document IDs that we can later remove from the result set
- uint32_t* excluded_result_ids = nullptr;
- size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
- &curated_ids[0], curated_ids.size(), &excluded_result_ids);
-
- std::vector<void*> posting_lists;
-
- for(auto& query_leaf : query_suggestion) {
- posting_lists.push_back(query_leaf->values);
- }
-
- result_iter_state_t iter_state(
- excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length
- );
-
- // We fetch offset positions only for multi token query
- bool fetch_offsets = (query_suggestion.size() > 1);
- bool single_exact_query_token = false;
-
- if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {
- // does this candidate suggestion token match query token exactly?
- single_exact_query_token = true;
- }
-
- if(topster == nullptr) {
- posting_t::block_intersector_t(posting_lists, iter_state)
- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {
- id_buff.push_back(seq_id);
- });
- } else {
- posting_t::block_intersector_t(posting_lists, iter_state)
- .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {
- score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
- total_cost, topster, query_suggestion, groups_processed,
- seq_id, sort_order, field_values, geopoint_indices,
- group_limit, group_by_fields, group_missing_values, token_bits,
- prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);
-
- id_buff.push_back(seq_id);
- });
- }
-
- delete [] excluded_result_ids;
- const size_t num_result_ids = id_buff.size();
-
- if(id_buff.size() > 100000) {
- // prevents too many ORs during exhaustive searching
- std::sort(id_buff.begin(), id_buff.end());
- id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());
-
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],
- id_buff.size(), &new_all_result_ids);
- delete[] *all_result_ids;
- *all_result_ids = new_all_result_ids;
- id_buff.clear();
- }
-
- if(num_result_ids == 0) {
- continue;
- }
-
- field_num_results += num_result_ids;
- searched_queries.push_back(actual_query_suggestion);
- }
-}
-
-void Index::numeric_not_equals_filter(num_tree_t* const num_tree,
- const int64_t value,
- const uint32_t& context_ids_length,
- uint32_t* const& context_ids,
- size_t& ids_len,
- uint32_t*& ids) const {
- uint32_t* to_exclude_ids = nullptr;
- size_t to_exclude_ids_len = 0;
-
- if (context_ids_length != 0) {
- num_tree->contains(EQUALS, value, context_ids_length, context_ids, to_exclude_ids_len, to_exclude_ids);
- } else {
- num_tree->search(EQUALS, value, &to_exclude_ids, to_exclude_ids_len);
- }
-
- auto all_ids = seq_ids->uncompress();
- auto all_ids_size = seq_ids->num_ids();
-
- uint32_t* to_include_ids = nullptr;
- size_t to_include_ids_len = 0;
-
- to_include_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, to_exclude_ids,
- to_exclude_ids_len, &to_include_ids);
-
- delete[] all_ids;
- delete[] to_exclude_ids;
-
- uint32_t* out = nullptr;
- ids_len = ArrayUtils::or_scalar(ids, ids_len, to_include_ids, to_include_ids_len, &out);
-
- delete[] ids;
- delete[] to_include_ids;
-
- ids = out;
-}
-
-bool Index::field_is_indexed(const std::string& field_name) const {
- return search_index.count(field_name) != 0 ||
- numerical_index.count(field_name) != 0 ||
- geopoint_index.count(field_name) != 0;
-}
-
-Option<bool> Index::do_filtering(filter_node_t* const root,
- filter_result_t& result,
- const std::string& collection_name,
- const uint32_t& context_ids_length,
- uint32_t* const& context_ids) const {
- // auto begin = std::chrono::high_resolution_clock::now();
- const filter a_filter = root->filter_exp;
-
- bool is_referenced_filter = !a_filter.referenced_collection_name.empty();
- if (is_referenced_filter) {
- // Apply filter on referenced collection and get the sequence ids of current collection from the filtered documents.
- auto& cm = CollectionManager::get_instance();
- auto collection = cm.get_collection(a_filter.referenced_collection_name);
- if (collection == nullptr) {
- return Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");
- }
-
- filter_result_t reference_filter_result;
- auto reference_filter_op = collection->get_reference_filter_ids(a_filter.field_name,
- reference_filter_result,
- collection_name);
- if (!reference_filter_op.ok()) {
- return Option<bool>(400, "Failed to apply reference filter on `" + a_filter.referenced_collection_name
- + "` collection: " + reference_filter_op.error());
- }
-
- if (context_ids_length != 0) {
- std::vector<uint32_t> include_indexes;
- include_indexes.reserve(std::min(context_ids_length, reference_filter_result.count));
-
- size_t context_index = 0, reference_result_index = 0;
- while (context_index < context_ids_length && reference_result_index < reference_filter_result.count) {
- if (context_ids[context_index] == reference_filter_result.docs[reference_result_index]) {
- include_indexes.push_back(reference_result_index);
- context_index++;
- reference_result_index++;
- } else if (context_ids[context_index] < reference_filter_result.docs[reference_result_index]) {
- context_index++;
- } else {
- reference_result_index++;
- }
- }
-
- result.count = include_indexes.size();
- result.docs = new uint32_t[include_indexes.size()];
- auto& result_references = result.reference_filter_results[a_filter.referenced_collection_name];
- result_references = new reference_filter_result_t[include_indexes.size()];
-
- for (uint32_t i = 0; i < include_indexes.size(); i++) {
- result.docs[i] = reference_filter_result.docs[include_indexes[i]];
- result_references[i] = reference_filter_result.reference_filter_results[a_filter.referenced_collection_name][include_indexes[i]];
- }
-
- return Option(true);
- }
-
- result = std::move(reference_filter_result);
- return Option(true);
- }
-
- if (a_filter.field_name == "id") {
- // we handle `ids` separately
- std::vector<uint32> result_ids;
- for (const auto& id_str : a_filter.values) {
- result_ids.push_back(std::stoul(id_str));
- }
-
- std::sort(result_ids.begin(), result_ids.end());
-
- auto result_array = new uint32[result_ids.size()];
- std::copy(result_ids.begin(), result_ids.end(), result_array);
-
- if (context_ids_length != 0) {
- uint32_t* out = nullptr;
- result.count = ArrayUtils::and_scalar(context_ids, context_ids_length,
- result_array, result_ids.size(), &out);
-
- delete[] result_array;
-
- result.docs = out;
- return Option(true);
- }
-
- result.docs = result_array;
- result.count = result_ids.size();
- return Option(true);
- }
-
- if (!field_is_indexed(a_filter.field_name)) {
- return Option(true);
- }
-
- field f = search_schema.at(a_filter.field_name);
-
- uint32_t* result_ids = nullptr;
- size_t result_ids_len = 0;
-
- if (f.is_integer()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
-
- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
- const std::string& filter_value = a_filter.values[fi];
- int64_t value = (int64_t)std::stol(filter_value);
-
- if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
- const std::string& next_filter_value = a_filter.values[fi + 1];
- auto const range_end_value = (int64_t)std::stol(next_filter_value);
-
- if (context_ids_length != 0) {
- num_tree->range_inclusive_contains(value, range_end_value, context_ids_length, context_ids,
- result_ids_len, result_ids);
- } else {
- num_tree->range_inclusive_search(value, range_end_value, &result_ids, result_ids_len);
- }
-
- fi++;
- } else if (a_filter.comparators[fi] == NOT_EQUALS) {
- numeric_not_equals_filter(num_tree, value, context_ids_length, context_ids, result_ids_len, result_ids);
- } else {
- if (context_ids_length != 0) {
- num_tree->contains(a_filter.comparators[fi], value,
- context_ids_length, context_ids, result_ids_len, result_ids);
- } else {
- num_tree->search(a_filter.comparators[fi], value, &result_ids, result_ids_len);
- }
- }
- }
- } else if (f.is_float()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
-
- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
- const std::string& filter_value = a_filter.values[fi];
- float value = (float)std::atof(filter_value.c_str());
- int64_t float_int64 = float_to_int64_t(value);
-
- if(a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
- const std::string& next_filter_value = a_filter.values[fi+1];
- int64_t range_end_value = float_to_int64_t((float) std::atof(next_filter_value.c_str()));
-
- if (context_ids_length != 0) {
- num_tree->range_inclusive_contains(float_int64, range_end_value, context_ids_length, context_ids,
- result_ids_len, result_ids);
- } else {
- num_tree->range_inclusive_search(float_int64, range_end_value, &result_ids, result_ids_len);
- }
-
- fi++;
- } else if (a_filter.comparators[fi] == NOT_EQUALS) {
- numeric_not_equals_filter(num_tree, float_int64,
- context_ids_length, context_ids, result_ids_len, result_ids);
- } else {
- if (context_ids_length != 0) {
- num_tree->contains(a_filter.comparators[fi], float_int64,
- context_ids_length, context_ids, result_ids_len, result_ids);
- } else {
- num_tree->search(a_filter.comparators[fi], float_int64, &result_ids, result_ids_len);
- }
- }
- }
- } else if (f.is_bool()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
-
- size_t value_index = 0;
- for (const std::string& filter_value : a_filter.values) {
- int64_t bool_int64 = (filter_value == "1") ? 1 : 0;
- if (a_filter.comparators[value_index] == NOT_EQUALS) {
- numeric_not_equals_filter(num_tree, bool_int64,
- context_ids_length, context_ids, result_ids_len, result_ids);
- } else {
- if (context_ids_length != 0) {
- num_tree->contains(a_filter.comparators[value_index], bool_int64,
- context_ids_length, context_ids, result_ids_len, result_ids);
- } else {
- num_tree->search(a_filter.comparators[value_index], bool_int64, &result_ids, result_ids_len);
- }
- }
-
- value_index++;
- }
- } else if (f.is_geopoint()) {
- for (const std::string& filter_value : a_filter.values) {
- std::vector<uint32_t> geo_result_ids;
-
- std::vector<std::string> filter_value_parts;
- StringUtils::split(filter_value, filter_value_parts, ","); // x, y, 2, km (or) list of points
-
- bool is_polygon = StringUtils::is_float(filter_value_parts.back());
- S2Region* query_region;
-
- if (is_polygon) {
- const int num_verts = int(filter_value_parts.size()) / 2;
- std::vector<S2Point> vertices;
- double sum = 0.0;
-
- for (size_t point_index = 0; point_index < size_t(num_verts);
- point_index++) {
- double lat = std::stod(filter_value_parts[point_index * 2]);
- double lon = std::stod(filter_value_parts[point_index * 2 + 1]);
- S2Point vertex = S2LatLng::FromDegrees(lat, lon).ToPoint();
- vertices.emplace_back(vertex);
- }
-
- auto loop = new S2Loop(vertices, S2Debug::DISABLE);
- loop->Normalize(); // if loop is not CCW but CW, change to CCW.
-
- S2Error error;
- if (loop->FindValidationError(&error)) {
- LOG(ERROR) << "Query vertex is bad, skipping. Error: " << error;
- delete loop;
- continue;
- } else {
- query_region = loop;
- }
- } else {
- double radius = std::stof(filter_value_parts[2]);
- const auto& unit = filter_value_parts[3];
-
- if (unit == "km") {
- radius *= 1000;
- } else {
- // assume "mi" (validated upstream)
- radius *= 1609.34;
- }
-
- S1Angle query_radius = S1Angle::Radians(S2Earth::MetersToRadians(radius));
- double query_lat = std::stod(filter_value_parts[0]);
- double query_lng = std::stod(filter_value_parts[1]);
- S2Point center = S2LatLng::FromDegrees(query_lat, query_lng).ToPoint();
- query_region = new S2Cap(center, query_radius);
- }
-
- S2RegionTermIndexer::Options options;
- options.set_index_contains_points_only(true);
- S2RegionTermIndexer indexer(options);
-
- for (const auto& term : indexer.GetQueryTerms(*query_region, "")) {
- auto geo_index = geopoint_index.at(a_filter.field_name);
- const auto& ids_it = geo_index->find(term);
- if(ids_it != geo_index->end()) {
- geo_result_ids.insert(geo_result_ids.end(), ids_it->second.begin(), ids_it->second.end());
- }
- }
-
- gfx::timsort(geo_result_ids.begin(), geo_result_ids.end());
- geo_result_ids.erase(std::unique( geo_result_ids.begin(), geo_result_ids.end() ), geo_result_ids.end());
-
- // `geo_result_ids` will contain all IDs that are within approximately within query radius
- // we still need to do another round of exact filtering on them
-
- if (context_ids_length != 0) {
- uint32_t *out = nullptr;
- uint32_t count = ArrayUtils::and_scalar(context_ids, context_ids_length,
- &geo_result_ids[0], geo_result_ids.size(), &out);
-
- geo_result_ids = std::vector<uint32_t>(out, out + count);
- }
-
- std::vector<uint32_t> exact_geo_result_ids;
-
- if (f.is_single_geopoint()) {
- spp::sparse_hash_map<uint32_t, int64_t>* sort_field_index = sort_index.at(f.name);
-
- for (auto result_id : geo_result_ids) {
- // no need to check for existence of `result_id` because of indexer based pre-filtering above
- int64_t lat_lng = sort_field_index->at(result_id);
- S2LatLng s2_lat_lng;
- GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);
- if (query_region->Contains(s2_lat_lng.ToPoint())) {
- exact_geo_result_ids.push_back(result_id);
- }
- }
- } else {
- spp::sparse_hash_map<uint32_t, int64_t*>* geo_field_index = geo_array_index.at(f.name);
-
- for (auto result_id : geo_result_ids) {
- int64_t* lat_lngs = geo_field_index->at(result_id);
-
- bool point_found = false;
-
- // any one point should exist
- for (size_t li = 0; li < lat_lngs[0]; li++) {
- int64_t lat_lng = lat_lngs[li + 1];
- S2LatLng s2_lat_lng;
- GeoPoint::unpack_lat_lng(lat_lng, s2_lat_lng);
- if (query_region->Contains(s2_lat_lng.ToPoint())) {
- point_found = true;
- break;
- }
- }
-
- if (point_found) {
- exact_geo_result_ids.push_back(result_id);
- }
- }
- }
-
- uint32_t* out = nullptr;
- result_ids_len = ArrayUtils::or_scalar(&exact_geo_result_ids[0], exact_geo_result_ids.size(),
- result_ids, result_ids_len, &out);
-
- delete[] result_ids;
- result_ids = out;
-
- delete query_region;
- }
- } else if (f.is_string()) {
- art_tree* t = search_index.at(a_filter.field_name);
-
- uint32_t* or_ids = nullptr;
- size_t or_ids_size = 0;
-
- // aggregates IDs across array of filter values and reduces excessive ORing
- std::vector<uint32_t> f_id_buff;
-
- for (const std::string& filter_value : a_filter.values) {
- std::vector<void*> posting_lists;
-
- // there could be multiple tokens in a filter value, which we have to treat as ANDs
- // e.g. country: South Africa
- Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);
-
- std::string str_token;
- size_t token_index = 0;
- std::vector<std::string> str_tokens;
-
- while (tokenizer.next(str_token, token_index)) {
- if (str_token.size() > 100) {
- str_token.erase(100);
- }
- str_tokens.push_back(str_token);
-
- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
- str_token.length()+1);
- if (leaf == nullptr) {
- continue;
- }
-
- posting_lists.push_back(leaf->values);
- }
-
- if (posting_lists.size() != str_tokens.size()) {
- continue;
- }
-
- if(a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {
- // needs intersection + exact matching (unlike CONTAINS)
- std::vector<uint32_t> result_id_vec;
- posting_t::intersect(posting_lists, result_id_vec, context_ids_length, context_ids);
-
- if (result_id_vec.empty()) {
- continue;
- }
-
- // need to do exact match
- uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];
- size_t exact_str_ids_size = 0;
- std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);
-
- posting_t::get_exact_matches(posting_lists, f.is_array(), result_id_vec.data(), result_id_vec.size(),
- exact_str_ids, exact_str_ids_size);
-
- if (exact_str_ids_size == 0) {
- continue;
- }
-
- for (size_t ei = 0; ei < exact_str_ids_size; ei++) {
- f_id_buff.push_back(exact_str_ids[ei]);
- }
- } else {
- // CONTAINS
- size_t before_size = f_id_buff.size();
- posting_t::intersect(posting_lists, f_id_buff, context_ids_length, context_ids);
- if (f_id_buff.size() == before_size) {
- continue;
- }
- }
-
- if (f_id_buff.size() > 100000 || a_filter.values.size() == 1) {
- gfx::timsort(f_id_buff.begin(), f_id_buff.end());
- f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
-
- uint32_t* out = nullptr;
- or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
- delete[] or_ids;
- or_ids = out;
- std::vector<uint32_t>().swap(f_id_buff); // clears out memory
- }
- }
-
- if (!f_id_buff.empty()) {
- gfx::timsort(f_id_buff.begin(), f_id_buff.end());
- f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
-
- uint32_t* out = nullptr;
- or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
- delete[] or_ids;
- or_ids = out;
- std::vector<uint32_t>().swap(f_id_buff); // clears out memory
- }
-
- result_ids = or_ids;
- result_ids_len = or_ids_size;
- }
-
- if (a_filter.apply_not_equals) {
- auto all_ids = seq_ids->uncompress();
- auto all_ids_size = seq_ids->num_ids();
-
- uint32_t* to_include_ids = nullptr;
- size_t to_include_ids_len = 0;
-
- to_include_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, result_ids,
- result_ids_len, &to_include_ids);
-
- delete[] all_ids;
- delete[] result_ids;
-
- result_ids = to_include_ids;
- result_ids_len = to_include_ids_len;
-
- if (context_ids_length != 0) {
- uint32_t *out = nullptr;
- result.count = ArrayUtils::and_scalar(context_ids, context_ids_length,
- result_ids, result_ids_len, &out);
-
- delete[] result_ids;
-
- result.docs = out;
- return Option(true);
- }
- }
-
- result.docs = result_ids;
- result.count = result_ids_len;
-
- return Option(true);
- /*long long int timeMillis =
- std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now()
- - begin).count();
-
- LOG(INFO) << "Time taken for filtering: " << timeMillis << "ms";*/
-}
-
-void Index::aproximate_numerical_match(num_tree_t* const num_tree,
- const NUM_COMPARATOR& comparator,
- const int64_t& value,
- const int64_t& range_end_value,
- uint32_t& filter_ids_length) const {
- if (comparator == RANGE_INCLUSIVE) {
- num_tree->approx_range_inclusive_search_count(value, range_end_value, filter_ids_length);
- return;
- }
-
- if (comparator == NOT_EQUALS) {
- uint32_t to_exclude_ids_len = 0;
- num_tree->approx_search_count(EQUALS, value, to_exclude_ids_len);
-
- auto all_ids_size = seq_ids->num_ids();
- filter_ids_length += (all_ids_size - to_exclude_ids_len);
- return;
- }
-
- num_tree->approx_search_count(comparator, value, filter_ids_length);
-}
-
-Option<bool> Index::_approximate_filter_ids(const filter& a_filter,
- uint32_t& filter_ids_length,
- const std::string& collection_name) const {
- if (!a_filter.referenced_collection_name.empty()) {
- auto& cm = CollectionManager::get_instance();
- auto collection = cm.get_collection(a_filter.referenced_collection_name);
- if (collection == nullptr) {
- return Option<bool>(400, "Referenced collection `" + a_filter.referenced_collection_name + "` not found.");
- }
-
- return collection->get_approximate_reference_filter_ids(a_filter.field_name, filter_ids_length);
- }
-
- if (a_filter.field_name == "id") {
- filter_ids_length = a_filter.values.size();
- return Option(true);
- }
-
- if (!field_is_indexed(a_filter.field_name)) {
- return Option(true);
- }
-
- field f = search_schema.at(a_filter.field_name);
-
- if (f.is_integer()) {
- auto num_tree = numerical_index.at(f.name);
-
- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
- const std::string& filter_value = a_filter.values[fi];
- auto const value = (int64_t)std::stol(filter_value);
-
- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
- const std::string& next_filter_value = a_filter.values[fi + 1];
- auto const range_end_value = (int64_t)std::stol(next_filter_value);
-
- aproximate_numerical_match(num_tree, a_filter.comparators[fi], value, range_end_value,
- filter_ids_length);
- fi++;
- } else {
- aproximate_numerical_match(num_tree, a_filter.comparators[fi], value, 0, filter_ids_length);
- }
- }
- } else if (f.is_float()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
+ size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
- for (size_t fi = 0; fi < a_filter.values.size(); fi++) {
- const std::string& filter_value = a_filter.values[fi];
- float value = (float)std::atof(filter_value.c_str());
- int64_t float_int64 = float_to_int64_t(value);
+ for (long long n = 0; n < N && n < combination_limit; ++n) {
+ RETURN_CIRCUIT_BREAKER
- if (a_filter.comparators[fi] == RANGE_INCLUSIVE && fi+1 < a_filter.values.size()) {
- const std::string& next_filter_value = a_filter.values[fi + 1];
- auto const range_end_value = float_to_int64_t((float) std::atof(next_filter_value.c_str()));
+ // every element in `query_suggestion` contains a token and its associated hits
+ std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());
- aproximate_numerical_match(num_tree, a_filter.comparators[fi], float_int64, range_end_value,
- filter_ids_length);
- fi++;
- } else {
- aproximate_numerical_match(num_tree, a_filter.comparators[fi], float_int64, 0, filter_ids_length);
- }
- }
- } else if (f.is_bool()) {
- auto num_tree = numerical_index.at(a_filter.field_name);
+ // actual query suggestion preserves original order of tokens in query
+ std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());
+ uint64 qhash;
- size_t value_index = 0;
- for (const std::string& filter_value : a_filter.values) {
- int64_t bool_int64 = (filter_value == "1") ? 1 : 0;
+ uint32_t token_bits = 0;
+ uint32_t total_cost = next_suggestion(token_candidates_vec, n, actual_query_suggestion,
+ query_suggestion, syn_orig_num_tokens, token_bits, qhash);
- aproximate_numerical_match(num_tree, a_filter.comparators[value_index], bool_int64, 0, filter_ids_length);
- value_index++;
+ if(query_hashes.find(qhash) != query_hashes.end()) {
+ // skip this query since it has already been processed before
+ continue;
}
- } else if (f.is_geopoint()) {
- filter_ids_length = 100;
- } else if (f.is_string()) {
- art_tree* t = search_index.at(a_filter.field_name);
-
- for (const std::string& filter_value : a_filter.values) {
- Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);
- std::string str_token;
- size_t token_index = 0;
+ query_hashes.insert(qhash);
- while (tokenizer.next(str_token, token_index)) {
- auto const leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
- str_token.length()+1);
- if (leaf == nullptr) {
- continue;
- }
+ //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
+ //LOG(INFO) << "n: " << n;
- filter_ids_length += posting_t::num_ids(leaf->values);
- }
+ /*std::stringstream fullq;
+ for(const auto& qleaf : actual_query_suggestion) {
+ std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);
+ fullq << qtok << " ";
}
- }
-
- if (a_filter.apply_not_equals) {
- auto all_ids_size = seq_ids->num_ids();
- filter_ids_length = (all_ids_size - filter_ids_length);
- }
+ LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;*/
- return Option(true);
-}
+ // Prepare excluded document IDs that we can later remove from the result set
+ uint32_t* excluded_result_ids = nullptr;
+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);
-Option<bool> Index::rearrange_filter_tree(filter_node_t* const root,
- uint32_t& approx_filter_ids_length,
- const std::string& collection_name) const {
- if (root == nullptr) {
- return Option(true);
- }
+ std::vector<void*> posting_lists;
- if (root->isOperator) {
- uint32_t l_filter_ids_length = 0;
- if (root->left != nullptr) {
- auto rearrange_op = rearrange_filter_tree(root->left, l_filter_ids_length, collection_name);
- if (!rearrange_op.ok()) {
- return rearrange_op;
- }
+ for(auto& query_leaf : query_suggestion) {
+ posting_lists.push_back(query_leaf->values);
}
- uint32_t r_filter_ids_length = 0;
- if (root->right != nullptr) {
- auto rearrange_op = rearrange_filter_tree(root->right, r_filter_ids_length, collection_name);
- if (!rearrange_op.ok()) {
- return rearrange_op;
- }
- }
+ result_iter_state_t iter_state(
+ excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length
+ );
- if (root->filter_operator == AND) {
- approx_filter_ids_length = std::min(l_filter_ids_length, r_filter_ids_length);
- } else {
- approx_filter_ids_length = l_filter_ids_length + r_filter_ids_length;
- }
+ // We fetch offset positions only for multi token query
+ bool single_exact_query_token = false;
- if (l_filter_ids_length > r_filter_ids_length) {
- std::swap(root->left, root->right);
+ if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {
+ // does this candidate suggestion token match query token exactly?
+ single_exact_query_token = true;
}
- return Option(true);
- }
+ if(topster == nullptr) {
+ posting_t::block_intersector_t(posting_lists, iter_state)
+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {
+ id_buff.push_back(seq_id);
+ });
+ } else {
+ posting_t::block_intersector_t(posting_lists, iter_state)
+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {
+ score_results(sort_fields, searched_queries.size(), field_id, field_is_array,
+ total_cost, topster, query_suggestion, groups_processed,
+ seq_id, sort_order, field_values, geopoint_indices,
- group_limit, group_by_fields, token_bits,
++ group_limit, group_by_fields, group_missing_values, token_bits,
+ prioritize_exact_match, single_exact_query_token, syn_orig_num_tokens, its);
- _approximate_filter_ids(root->filter_exp, approx_filter_ids_length, collection_name);
- return Option(true);
-}
+ id_buff.push_back(seq_id);
+ });
+ }
-Option<bool> Index::recursive_filter(filter_node_t* const root,
- filter_result_t& result,
- const std::string& collection_name,
- const uint32_t& context_ids_length,
- uint32_t* const& context_ids) const {
- if (root == nullptr) {
- return Option(true);
- }
+ delete [] excluded_result_ids;
+ const size_t num_result_ids = id_buff.size();
- if (root->isOperator) {
- filter_result_t l_result;
- if (root->left != nullptr) {
- auto filter_op = recursive_filter(root->left, l_result , collection_name, context_ids_length, context_ids);
- if (!filter_op.ok()) {
- return filter_op;
- }
- }
+ if(id_buff.size() > 100000) {
+ // prevents too many ORs during exhaustive searching
+ std::sort(id_buff.begin(), id_buff.end());
+ id_buff.erase(std::unique( id_buff.begin(), id_buff.end() ), id_buff.end());
- filter_result_t r_result;
- if (root->right != nullptr) {
- auto filter_op = recursive_filter(root->right, r_result , collection_name, context_ids_length, context_ids);
- if (!filter_op.ok()) {
- return filter_op;
- }
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &id_buff[0],
+ id_buff.size(), &new_all_result_ids);
+ delete[] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+ id_buff.clear();
}
- if (root->filter_operator == AND) {
- filter_result_t::and_filter_results(l_result, r_result, result);
- } else {
- filter_result_t::or_filter_results(l_result, r_result, result);
+ if(num_result_ids == 0) {
+ continue;
}
- return Option(true);
+ field_num_results += num_result_ids;
+ searched_queries.push_back(actual_query_suggestion);
}
+}
- return do_filtering(root, result, collection_name, context_ids_length, context_ids);
+bool Index::field_is_indexed(const std::string& field_name) const {
+ return search_index.count(field_name) != 0 ||
+ numerical_index.count(field_name) != 0 ||
+ range_index.count(field_name) != 0 ||
+ geo_range_index.count(field_name) != 0;
}
Option<bool> Index::do_filtering_with_lock(filter_node_t* const filter_tree_root,
@@@ -2280,10 -2783,10 +2296,11 @@@ Option<bool> Index::search(std::vector<
std::vector<uint32_t> included_ids_vec;
std::unordered_set<uint32_t> excluded_group_ids;
- process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, filter_curated_hits,
- process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit, group_missing_values,
- filter_curated_hits,
- filter_result.docs, filter_result.count, curated_ids, included_ids_map,
++ process_curated_ids(included_ids, excluded_ids, group_by_fields, group_limit,
++ group_missing_values, filter_curated_hits,
+ filter_result_iterator, curated_ids, included_ids_map,
included_ids_vec, excluded_group_ids);
+ filter_result_iterator->reset();
std::vector<uint32_t> curated_ids_sorted(curated_ids.begin(), curated_ids.end());
std::sort(curated_ids_sorted.begin(), curated_ids_sorted.end());
@@@ -2318,24 -2819,14 +2335,25 @@@
// handle phrase searches
if (!field_query_tokens[0].q_phrases.empty()) {
- do_phrase_search(num_search_fields, the_fields, field_query_tokens,
- sort_fields_std, searched_queries, group_limit, group_by_fields, group_missing_values,
- topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,
- all_result_ids, all_result_ids_len, groups_processed, curated_ids,
- excluded_result_ids, excluded_result_ids_size, excluded_group_ids, curated_topster,
- included_ids_map, is_wildcard_query,
- filter_result.docs, filter_result.count);
- if (filter_result.count == 0) {
+ auto do_phrase_search_op = do_phrase_search(num_search_fields, the_fields, field_query_tokens,
+ sort_fields_std, searched_queries, group_limit, group_by_fields,
++ group_missing_values,
+ topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,
+ filter_result_iterator, all_result_ids, all_result_ids_len,
+ groups_processed, curated_ids,
+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
+ curated_topster, included_ids_map, is_wildcard_query,
+ collection_name);
+
+ filter_iterator_guard.release();
+ filter_iterator_guard.reset(filter_result_iterator);
+
+ if (!do_phrase_search_op.ok()) {
+ delete [] all_result_ids;
+ return do_phrase_search_op;
+ }
+
+ if (filter_result_iterator->approx_filter_ids_length == 0) {
goto process_search_results;
}
}
@@@ -2352,10 -2846,10 +2370,10 @@@
uint32_t seq_id = it.id();
uint64_t distinct_id = seq_id;
if (group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
- continue;
- }
+ continue;
+ }
}
int64_t scores[3] = {0};
@@@ -2472,14 -2946,14 +2490,14 @@@
uint64_t distinct_id = seq_id;
if (group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
- continue;
- }
+ continue;
+ }
}
- auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :
- dist_label.first;
+ auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_result.first) :
+ dist_result.first;
if(vec_dist_score > vector_query.distance_threshold) {
continue;
@@@ -2512,29 -2981,14 +2530,31 @@@
all_result_ids_len = nearest_ids.size();
}
} else {
- search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,
- curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
- group_missing_values, curated_ids, curated_ids_sorted,
- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
- all_result_ids, all_result_ids_len,
- filter_result.docs, filter_result.count, concurrency,
- sort_order, field_values, geopoint_indices);
+ // if filters were not provided, use the seq_ids index to generate the list of all document ids
+ if (no_filters_provided) {
+ filter_result_iterator = new filter_result_iterator_t(seq_ids->uncompress(), seq_ids->num_ids());
+ filter_iterator_guard.reset(filter_result_iterator);
+ }
+
+ auto search_wildcard_op = search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,
+ curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
++ group_missing_values,
+ curated_ids, curated_ids_sorted,
+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
+ all_result_ids, all_result_ids_len,
+ filter_result_iterator, concurrency,
+ sort_order, field_values, geopoint_indices, collection_name);
+ if (!search_wildcard_op.ok()) {
+ return search_wildcard_op;
+ }
++
+ filter_result_iterator->reset();
}
+
+ uint32_t _all_result_ids_len = all_result_ids_len;
+ curate_filtered_ids(curated_ids, excluded_result_ids,
+ excluded_result_ids_size, all_result_ids, _all_result_ids_len, curated_ids_sorted);
+ all_result_ids_len = _all_result_ids_len;
} else {
// Non-wildcard
// In multi-field searches, a record can be matched across different fields, so we use this for aggregation
@@@ -2573,21 -3027,16 +2593,22 @@@
}
}
- fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type, excluded_result_ids,
- excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,
- excluded_group_ids,
- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields,
- group_missing_values, prioritize_exact_match,
- prioritize_token_position, prioritize_num_matching_fields, query_hashes, token_order, prefixes,
- typo_tokens_threshold, exhaustive_search,
- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,
- field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type,
+ excluded_result_ids, excluded_result_ids_size,
+ filter_result_iterator, curated_ids_sorted,
+ excluded_group_ids, sort_fields_std, num_typos,
+ searched_queries, qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len,
- group_limit, group_by_fields, prioritize_exact_match,
- prioritize_token_position, query_hashes, token_order, prefixes,
++ group_limit, group_by_fields, group_missing_values, prioritize_exact_match,
++ prioritize_token_position, prioritize_num_matching_fields,
++ query_hashes, token_order, prefixes,
+ typo_tokens_threshold, exhaustive_search,
+ max_candidates, min_len_1typo, min_len_2typo,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ collection_name);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
// try split/joining tokens if no results are found
if(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {
@@@ -2617,40 -3066,33 +2638,43 @@@
space_resolved_queries[0][j].size(), 0);
}
- fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,
- excluded_result_ids_size, filter_result.docs, filter_result.count, curated_ids_sorted,
- excluded_group_ids,
- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields,
- group_missing_values, prioritize_exact_match,
- prioritize_token_position, prioritize_num_matching_fields,
- query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,
- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,
- field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,
+ excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,
+ excluded_group_ids,
+ sort_fields_std, num_typos, searched_queries,
+ qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len,
- group_limit, group_by_fields, prioritize_exact_match,
- prioritize_token_position, query_hashes, token_order,
++ group_limit, group_by_fields, group_missing_values,
++ prioritize_exact_match, prioritize_token_position,
++ prioritize_num_matching_fields,
++ query_hashes, token_order,
+ prefixes, typo_tokens_threshold, exhaustive_search,
+ max_candidates, min_len_1typo, min_len_2typo,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ collection_name);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
}
}
// do synonym based searches
- do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map, sort_fields_std,
- curated_topster, token_order,
- 0, group_limit, group_by_fields, group_missing_values,
- prioritize_exact_match, prioritize_token_position,
- prioritize_num_matching_fields,
- exhaustive_search, concurrency, prefixes,
- min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
- topster, q_pos_synonyms, syn_orig_num_tokens,
- groups_processed, searched_queries, all_result_ids, all_result_ids_len,
- filter_result.docs, filter_result.count, query_hashes,
- sort_order, field_values, geopoint_indices,
- qtoken_set);
+ auto do_synonym_search_op = do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map,
+ sort_fields_std, curated_topster, token_order, 0, group_limit,
- group_by_fields, prioritize_exact_match, prioritize_token_position,
- exhaustive_search, concurrency, prefixes,
++ group_by_fields, group_missing_values, prioritize_exact_match, prioritize_token_position,
++ prioritize_num_matching_fields, exhaustive_search, concurrency, prefixes,
+ min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
+ topster, q_pos_synonyms, syn_orig_num_tokens,
+ groups_processed, searched_queries, all_result_ids, all_result_ids_len,
+ filter_result_iterator, query_hashes,
+ sort_order, field_values, geopoint_indices,
+ qtoken_set, collection_name);
+ if (!do_synonym_search_op.ok()) {
+ return do_synonym_search_op;
+ }
++
+ filter_result_iterator->reset();
// gather up both original query and synonym queries and do drop tokens
@@@ -2705,22 -3139,17 +2729,23 @@@
drop_token_prefixes.push_back(p && prefix_search);
}
- fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,
- excluded_result_ids, excluded_result_ids_size,
- filter_result.docs, filter_result.count,
- curated_ids_sorted, excluded_group_ids, sort_fields_std, num_typos, searched_queries,
- qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields,
- group_missing_values,
- prioritize_exact_match, prioritize_token_position, prioritize_num_matching_fields,
- query_hashes, token_order, prefixes, typo_tokens_threshold,
- exhaustive_search, max_candidates, min_len_1typo,
- min_len_2typo, -1, sort_order, field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,
+ excluded_result_ids, excluded_result_ids_size,
+ filter_result_iterator,
+ curated_ids_sorted, excluded_group_ids,
+ sort_fields_std, num_typos, searched_queries,
- qtoken_set, topster, groups_processed,
++ qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len,
- group_limit, group_by_fields,
- prioritize_exact_match, prioritize_token_position, query_hashes,
++ group_limit, group_by_fields, group_missing_values,
++ prioritize_exact_match, prioritize_token_position,
++ prioritize_num_matching_fields, query_hashes,
+ token_order, prefixes, typo_tokens_threshold,
+ exhaustive_search, max_candidates, min_len_1typo,
+ min_len_2typo, -1, sort_order, field_values, geopoint_indices,
+ collection_name);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
} else {
break;
@@@ -2729,20 -3158,13 +2754,21 @@@
}
}
- do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std, searched_queries,
- group_limit, group_by_fields, group_missing_values,
- max_extra_prefix, max_extra_suffix,
- field_query_tokens[0].q_include_tokens,
- topster, filter_result.docs, filter_result.count,
- sort_order, field_values, geopoint_indices,
- curated_ids_sorted, excluded_group_ids, all_result_ids, all_result_ids_len, groups_processed);
+ auto do_infix_search_op = do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std,
+ searched_queries,
- group_limit, group_by_fields,
++ group_limit, group_by_fields, group_missing_values,
+ max_extra_prefix, max_extra_suffix,
+ field_query_tokens[0].q_include_tokens,
+ topster, filter_result_iterator,
+ sort_order, field_values, geopoint_indices,
+ curated_ids_sorted, excluded_group_ids,
+ all_result_ids, all_result_ids_len, groups_processed,
+ collection_name);
+ if (!do_infix_search_op.ok()) {
+ return do_infix_search_op;
+ }
++
+ filter_result_iterator->reset();
if(!vector_query.field_name.empty()) {
// check at least one of sort fields is text match
@@@ -2811,14 -3245,22 +2852,27 @@@
for(size_t res_index = 0; res_index < vec_results.size(); res_index++) {
auto& vec_result = vec_results[res_index];
auto seq_id = vec_result.first;
+
+ filter_result_iterator->skip_to(seq_id);
+ auto references = std::move(filter_result_iterator->reference);
+ filter_result_iterator->reset();
- auto result_it = topster->kv_map.find(seq_id);
-
- if(result_it != topster->kv_map.end()) {
- if(result_it->second->match_score_index < 0 || result_it->second->match_score_index > 2) {
++
+ KV* found_kv = nullptr;
+ if(group_limit != 0) {
+ for(auto& kv : kvs) {
+ if(kv->key == seq_id) {
+ found_kv = kv;
+ break;
+ }
+ }
+ } else {
+ auto result_it = topster->kv_map.find(seq_id);
+ if(result_it != topster->kv_map.end()) {
+ found_kv = result_it->second;
+ }
+ }
+ if(found_kv) {
+ if(found_kv->match_score_index < 0 || found_kv->match_score_index > 2) {
continue;
}
@@@ -2833,20 -3274,14 +2886,20 @@@
((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT));
int64_t match_score_index = -1;
int64_t scores[3] = {0};
-
- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, seq_id, 0,
- match_score, scores, match_score_index, vec_result.second);
+
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
+ geopoint_indices, seq_id, references, 0,
+ match_score, scores, match_score_index,
+ vec_result.second, collection_name);
+ if (!compute_sort_scores_op.ok()) {
+ return compute_sort_scores_op;
+ }
for(int i = 0; i < 3; i++) {
- kv->scores[i] = scores[i];
+ found_kv->scores[i] = scores[i];
}
+
- kv->match_score_index = match_score_index;
+ found_kv->match_score_index = match_score_index;
} else {
// Result has been found only in vector search: we have to add it to both KV and result_ids
@@@ -2870,20 -3298,16 +2923,24 @@@
continue;
}
}
-
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
kv.text_match_score = 0;
kv.vector_distance = vec_result.second;
+
+ if (filter_result_iterator->is_valid &&
+ !filter_result_iterator->reference.empty()) {
+ // The doc_id must be valid otherwise it would've been filtered out upstream.
+ filter_result_iterator->skip_to(seq_id);
+ kv.reference_filter_results = std::move(filter_result_iterator->reference);
+ filter_result_iterator->reset();
+ }
+
- topster->add(&kv);
+ auto ret = topster->add(&kv);
vec_search_ids.push_back(seq_id);
+
+ if(group_limit != 0 && ret < 2) {
+ groups_processed[distinct_id]++;
+ }
}
}
@@@ -2962,21 -3375,18 +3019,23 @@@
num_queued++;
thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,
- batch_result_ids, batch_res_len, &facet_infos,
- estimate_facets, facet_sample_percent, group_missing_values,
+ batch_result_ids, batch_res_len, &facet_infos, max_facet_values,
- is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,
++ is_wildcard_query, no_filters_provided, estimate_facets, facet_sample_percent,
++ group_missing_values,
&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
- &num_processed, &m_process, &cv_process]() {
+ &num_processed, &m_process, &cv_process, facet_index_type]() {
search_begin_us = parent_search_begin;
search_stop_us = parent_search_stop_ms;
search_cutoff = parent_search_cutoff;
auto fq = facet_query;
+ std::vector<std::pair<std::string, uint32_t>> found_docs;
do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,
- facet_infos, group_limit, group_by_fields,
+ facet_infos, group_limit, group_by_fields, group_missing_values,
- batch_result_ids, batch_res_len);
+ batch_result_ids, batch_res_len, max_facet_values,
+ is_wildcard_query, no_filters_provided,
+ facet_index_type);
++
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
parent_search_cutoff = parent_search_cutoff || search_cutoff;
@@@ -3079,16 -3457,13 +3138,16 @@@
std::chrono::high_resolution_clock::now() - beginF).count();
LOG(INFO) << "Time for faceting: " << timeMillisF;*/
}
-
+ std::vector<std::pair<std::string, uint32_t>> found_docs;
std::vector<facet_info_t> facet_infos(facets.size());
compute_facet_infos(facets, facet_query, facet_query_num_typos,
- &included_ids_vec[0], included_ids_vec.size(), group_by_fields, max_candidates, facet_infos);
+ &included_ids_vec[0], included_ids_vec.size(), group_by_fields,
+ group_limit, is_wildcard_no_filter_query,
+ max_candidates, facet_infos, facet_index_type);
do_facets(facets, facet_query, estimate_facets, facet_sample_percent,
- facet_infos, group_limit, group_by_fields, &included_ids_vec[0],
- facet_infos, group_limit, group_by_fields, group_missing_values,
- &included_ids_vec[0], included_ids_vec.size());
++ facet_infos, group_limit, group_by_fields, group_missing_values, &included_ids_vec[0],
+ included_ids_vec.size(), max_facet_values, is_wildcard_query, no_filters_provided,
+ facet_index_type);
all_result_ids_len += curated_topster->size;
@@@ -3103,8 -3478,9 +3162,9 @@@
void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids,
-- const std::vector<std::string>& group_by_fields, const size_t group_limit,
++ const std::vector<std::string>& group_by_fields, const size_t group_limit,
+ const bool group_missing_values,
- const bool filter_curated_hits, const uint32_t* filter_ids, uint32_t filter_ids_length,
+ const bool filter_curated_hits, filter_result_iterator_t* const filter_result_iterator,
std::set<uint32_t>& curated_ids,
std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
std::vector<uint32_t>& included_ids_vec,
@@@ -3189,37 -3567,38 +3249,39 @@@
}
}
-void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
- const std::vector<token_t>& query_tokens,
- const std::vector<token_t>& dropped_tokens,
- const text_match_type_t match_type,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const uint32_t* filter_ids, size_t filter_ids_length,
- const std::vector<uint32_t>& curated_ids,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by> & sort_fields,
- const std::vector<uint32_t>& num_typos,
- std::vector<std::vector<art_leaf*>> & searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t & all_result_ids_len,
- const size_t group_limit, const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool prioritize_num_matching_fields,
- std::set<uint64>& query_hashes,
- const token_ordering token_order,
- const std::vector<bool>& prefixes,
- const size_t typo_tokens_threshold,
- const bool exhaustive_search,
- const size_t max_candidates,
- size_t min_len_1typo,
- size_t min_len_2typo,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const {
+Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<token_t>& dropped_tokens,
+ const text_match_type_t match_type,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ filter_result_iterator_t* const filter_result_iterator,
+ const std::vector<uint32_t>& curated_ids,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by> & sort_fields,
+ const std::vector<uint32_t>& num_typos,
+ std::vector<std::vector<art_leaf*>> & searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t & all_result_ids_len,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
++ const bool prioritize_num_matching_fields,
+ std::set<uint64>& query_hashes,
+ const token_ordering token_order,
+ const std::vector<bool>& prefixes,
+ const size_t typo_tokens_threshold,
+ const bool exhaustive_search,
+ const size_t max_candidates,
+ size_t min_len_1typo,
+ size_t min_len_2typo,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name) const {
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
@@@ -3449,21 -3825,17 +3511,21 @@@
if(token_candidates_vec.size() == query_tokens.size()) {
std::vector<uint32_t> id_buff;
-
- search_all_candidates(num_search_fields, match_type, the_fields, filter_ids, filter_ids_length,
- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
- sort_fields, token_candidates_vec, searched_queries, qtoken_set,
- dropped_tokens, topster,
- groups_processed, all_result_ids, all_result_ids_len,
- typo_tokens_threshold, group_limit, group_by_fields,
- group_missing_values, query_tokens,
- num_typos, prefixes, prioritize_exact_match, prioritize_token_position,
- prioritize_num_matching_fields, exhaustive_search, max_candidates,
- syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
- query_hashes, id_buff);
+ auto search_all_candidates_op = search_all_candidates(num_search_fields, match_type, the_fields,
+ filter_result_iterator,
+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
+ sort_fields, token_candidates_vec, searched_queries, qtoken_set,
+ dropped_tokens, topster,
+ groups_processed, all_result_ids, all_result_ids_len,
- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
++ typo_tokens_threshold, group_limit, group_by_fields,
++ group_missing_values, query_tokens,
+ num_typos, prefixes, prioritize_exact_match, prioritize_token_position,
- exhaustive_search, max_candidates,
++ prioritize_num_matching_fields, exhaustive_search, max_candidates,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ query_hashes, id_buff, collection_name);
+ if (!search_all_candidates_op.ok()) {
+ return search_all_candidates_op;
+ }
if(id_buff.size() > 1) {
gfx::timsort(id_buff.begin(), id_buff.end());
@@@ -3604,32 -3972,33 +3666,34 @@@ void Index::find_across_fields(const to
}
}
-void Index::search_across_fields(const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- const std::vector<search_field_t>& the_fields,
- const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<sort_by>& sort_fields,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool prioritize_num_matching_fields,
- const uint32_t* filter_ids, uint32_t filter_ids_length,
- const uint32_t total_cost, const int syn_orig_num_tokens,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::vector<uint32_t>& id_buff,
- uint32_t*& all_result_ids, size_t& all_result_ids_len) const {
+Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ const std::vector<search_field_t>& the_fields,
+ const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<sort_by>& sort_fields,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ const bool prioritize_exact_match,
+ const bool prioritize_token_position,
++ const bool prioritize_num_matching_fields,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t total_cost, const int syn_orig_num_tokens,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::vector<uint32_t>& id_buff,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const std::string& collection_name) const {
std::vector<art_leaf*> query_suggestion;
@@@ -3831,10 -4197,10 +3895,10 @@@
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
- return;
- }
+ return;
+ }
}
int64_t scores[3] = {0};
@@@ -4454,34 -4520,34 +4522,35 @@@ Option<bool> Index::compute_sort_scores
scores[2] = -scores[2];
}
}
-}
-void Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
- std::vector<query_tokens_t>& field_query_tokens,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- Topster* actual_topster,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- const std::set<uint32_t>& curated_ids,
- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* curated_topster,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- bool is_wildcard_query,
- uint32_t*& filter_ids, uint32_t& filter_ids_length) const {
+ return Option<bool>(true);
+}
+Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
+ std::vector<query_tokens_t>& field_query_tokens,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
+ Topster* actual_topster,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ filter_result_iterator_t*& filter_result_iterator,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::set<uint32_t>& curated_ids,
+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* curated_topster,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ bool is_wildcard_query, const std::string& collection_name) const {
+
+ uint32_t* phrase_result_ids = nullptr;
+ uint32_t phrase_result_count = 0;
std::map<uint32_t, size_t> phrase_match_id_scores;
- uint32_t* phrase_match_ids = nullptr;
- size_t phrase_match_ids_size = 0;
-
for(size_t i = 0; i < num_search_fields; i++) {
const std::string& field_name = search_fields[i].name;
const size_t field_weight = search_fields[i].weight;
@@@ -4622,79 -4695,75 +4691,87 @@@
BREAK_CIRCUIT_BREAKER
}
}
+ filter_result_iterator->reset();
searched_queries.push_back({});
+ return Option<bool>(true);
}
-void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
- const text_match_type_t match_type,
- filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
- const token_ordering& token_order,
- const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool prioritize_num_matching_fields,
- const bool exhaustive_search, const size_t concurrency,
- const std::vector<bool>& prefixes,
- size_t min_len_1typo,
- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* actual_topster,
- std::vector<std::vector<token_t>>& q_pos_synonyms,
- int syn_orig_num_tokens,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const uint32_t* filter_ids, const uint32_t filter_ids_length,
- std::set<uint64>& query_hashes,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- tsl::htrie_map<char, token_leaf>& qtoken_set) const {
+Option<bool> Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
+ const text_match_type_t match_type,
+ filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
+ const token_ordering& token_order,
+ const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
++ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
++ bool prioritize_exact_match,
+ const bool prioritize_token_position,
++ const bool prioritize_num_matching_fields,
+ const bool exhaustive_search, const size_t concurrency,
+ const std::vector<bool>& prefixes,
+ size_t min_len_1typo,
+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* actual_topster,
+ std::vector<std::vector<token_t>>& q_pos_synonyms,
+ int syn_orig_num_tokens,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ std::set<uint64>& query_hashes,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::string& collection_name) const {
for (const auto& syn_tokens : q_pos_synonyms) {
query_hashes.clear();
- fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,
- exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted, excluded_group_ids,
- sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields,
- group_missing_values, prioritize_exact_match,
- prioritize_token_position, prioritize_num_matching_fields,
- query_hashes, token_order, prefixes, typo_tokens_threshold,
- exhaustive_search, max_candidates, min_len_1typo,
- min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,
+ exclude_token_ids_size, filter_result_iterator,
+ curated_ids_sorted, excluded_group_ids, sort_fields_std, {0},
+ searched_queries, qtoken_set, actual_topster, groups_processed,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,
- prioritize_exact_match, prioritize_token_position, query_hashes,
++ group_missing_values,
++ prioritize_exact_match, prioritize_token_position,
++ prioritize_num_matching_fields,
++ query_hashes,
+ token_order, prefixes, typo_tokens_threshold, exhaustive_search,
+ max_candidates, min_len_1typo, min_len_2typo,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ collection_name);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
}
collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
+ return Option<bool>(true);
}
-void Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
- const std::vector<enable_t>& infixes,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const size_t max_extra_prefix,
- const size_t max_extra_suffix,
- const std::vector<token_t>& query_tokens, Topster* actual_topster,
- const uint32_t *filter_ids, size_t filter_ids_length,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {
+Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
+ const std::vector<enable_t>& infixes,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
++ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
++ const size_t max_extra_prefix,
+ const size_t max_extra_suffix,
+ const std::vector<token_t>& query_tokens, Topster* actual_topster,
+ filter_result_iterator_t* const filter_result_iterator,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::string& collection_name) const {
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
auto& field_name = the_fields[field_id].name;
@@@ -4759,15 -4819,14 +4836,15 @@@
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
- continue;
- }
+ continue;
+ }
}
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
int ret = actual_topster->add(&kv);
+
if(group_limit != 0 && ret < 2) {
groups_processed[distinct_id]++;
}
@@@ -5047,27 -5078,24 +5124,29 @@@ void Index::curate_filtered_ids(const s
}
}
-void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool group_missing_values,
- const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len, const uint32_t* filter_ids,
- uint32_t filter_ids_length, const size_t concurrency,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const {
+Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
++ const std::vector<std::string>& group_by_fields,
++ const bool group_missing_values,
++ const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ const size_t concurrency,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::string& collection_name) const {
+
+ filter_result_iterator->compute_result();
+ auto const& approx_filter_ids_length = filter_result_iterator->approx_filter_ids_length;
uint32_t token_bits = 0;
- const bool check_for_circuit_break = (filter_ids_length > 1000000);
+ const bool check_for_circuit_break = (approx_filter_ids_length > 1000000);
//auto beginF = std::chrono::high_resolution_clock::now();
@@@ -5101,16 -5131,15 +5180,17 @@@
searched_queries.push_back({});
topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);
+ auto& compute_sort_score_status = compute_sort_score_statuses[thread_id] = nullptr;
thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
- thread_id, &sort_fields, &searched_queries,
- &group_limit, &group_by_fields, group_missing_values,
- &topsters, &tgroups_processed, &excluded_group_ids,
- &sort_order, field_values, &geopoint_indices, &plists,
- check_for_circuit_break,
- batch_result_ids, batch_res_len,
- &num_processed, &m_process, &cv_process]() {
+ thread_id, &sort_fields, &searched_queries,
- &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,
++ &group_limit, &group_by_fields, group_missing_values,
++ &topsters, &tgroups_processed, &excluded_group_ids,
+ &sort_order, field_values, &geopoint_indices, &plists,
+ check_for_circuit_break,
+ batch_result,
+ &num_processed, &m_process, &cv_process, &compute_sort_score_status, collection_name]() {
+ std::unique_ptr<filter_result_t> batch_result_guard(batch_result);
search_begin_us = parent_search_begin;
search_stop_us = parent_search_stop_ms;
@@@ -5143,15 -5162,15 +5223,15 @@@
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id);
+ distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
if(excluded_group_ids.count(distinct_id) != 0) {
- continue;
- }
+ continue;
+ }
}
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- int ret = topsters[thread_id]->add(&kv);
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
+ int ret = topsters[thread_id]->add(&kv);
if(group_limit != 0 && ret < 2) {
tgroups_processed[thread_id][distinct_id]++;
}
@@@ -5802,37 -5783,48 +5885,38 @@@ void Index::score_results(const std::ve
// pre-filter group_by_fields such that we can avoid the find() check
uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
- const uint32_t seq_id) const {
- const uint32_t seq_id, const bool group_missing_values) const {
- uint64_t distinct_id = 1;
- bool found_group_value = false;
++ const uint32_t seq_id, const bool group_missing_values) const {
+ uint32_t distinct_id = 1; // some constant initial value
// calculate hash from group_by_fields
- for(const auto& field: group_by_fields) {
- const auto& field_facet_mapping_it = facet_index_v3.find(field);
- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);
- if((field_facet_mapping_it == facet_index_v3.end())
- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
+ for(const auto& field_name: group_by_fields) {
+ if(!facet_index_v4->has_hash_index(field_name)) {
continue;
}
- if(search_schema.at(field).is_array()) {
- const auto& field_facet_mapping = field_facet_mapping_it->second;
- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
-
- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
- }
+ posting_list_t* facet_hash_index = facet_index_v4->get_facet_hash_index(field_name);
+ std::vector<uint32_t> facet_hashes;
+ posting_list_t::iterator_t facet_index_it = facet_hash_index->new_iterator();
+ facet_index_it.skip_to(seq_id);
- if(facet_index_it.valid()) {
- const auto& facet_hashes = facet_hashes_it->second;
-
- for(size_t i = 0; i < facet_hashes.size(); i++) {
- distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);
- found_group_value = true;
- }
- } else {
- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
++ if(facet_index_it.valid() && facet_index_it.id() == seq_id) {
+ posting_list_t::get_offsets(facet_index_it, facet_hashes);
- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
+ if(search_schema.at(field_name).is_array()) {
+ //LOG(INFO) << "combining hashes for facet array ";
+ for(size_t i = 0; i < facet_hashes.size(); i++) {
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes[i]);
+ }
+ } else {
+ const auto& facet_hash = facet_hashes[0];
+ //LOG(INFO) << "combining hashes for facet ";
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
}
-
- const auto& facet_hash = facet_hashes_it->second;
- distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
- found_group_value = true;
}
}
- //LOG(INFO) << "distinct id " << distinct_id;
- return distinct_id;
+
- return (!found_group_value && !group_missing_values) ? seq_id : distinct_id;
++ //LOG(INFO) << "seq_id: " << seq_id << ", distinct_id: " << distinct_id;
++ return (distinct_id == 1 && !group_missing_values) ? seq_id : distinct_id;
}
inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token_candidates_vec,
diff --cc test/collection_faceting_test.cpp
index 4ff66550,5e9c1e9d..3d95a45f
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@@ -1698,744 -1690,110 +1698,852 @@@ TEST_F(CollectionFacetingTest, FacetOnA
}
}
+TEST_F(CollectionFacetingTest, FloatFieldValueTruncation) {
+ std::vector<field> fields = {
+ field("tags", field_types::STRING_ARRAY, true),
+ field("points", field_types::FLOAT, true),
+ };
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["tags"] = {"gamma"};
+ doc["points"] = 300;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("*", {},
+ "", {"points"}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+
+ ASSERT_EQ("300", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
+}
+
+TEST_F(CollectionFacetingTest, FacetingReturnParent) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value.color", "type": "string", "optional": false, "facet": true },
+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "value": {
+ "color": "red",
+ "r": 255,
+ "g": 0,
+ "b": 0
+ }
+ })"_json;
+
+ nlohmann::json doc2 = R"({
+ "value": {
+ "color": "blue",
+ "r": 0,
+ "g": 0,
+ "b": 255
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_op = coll1->search("*", {},"", {"value.color"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"value.color"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["parent"].dump());
+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["parent"].dump());
+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);
+
+ //not passing facet_fields in facet_return_parent list will only return facet value, not immediate parent for those field
+ search_op = coll1->search("*", {},"", {"value.color"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);
+
+ search_op = coll1->search("*", {},"", {"value.color", "value.r"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"value.r"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ results = search_op.get();
+ ASSERT_EQ(2, results["facet_counts"].size());
+
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);
+
+ ASSERT_EQ(2, results["facet_counts"][1]["counts"].size());
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][1]["counts"][0]["parent"].dump());
+ ASSERT_EQ("0", results["facet_counts"][1]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][1]["counts"][1]["parent"].dump());
+ ASSERT_EQ("255", results["facet_counts"][1]["counts"][1]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetingReturnParentDeepNested) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "product.specification.detail.width", "type": "int32", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "product" : {
+ "specification": {
+ "detail" : {
+ "width": 25
+ }
+ }
+ }
+ })"_json;
+
+ nlohmann::json doc2 = R"({
+ "product" : {
+ "specification": {
+ "detail" : {
+ "width": 30
+ }
+ }
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_op = coll1->search("*", {},"", {"product.specification.detail.width"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"product.specification.detail.width"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":30}}}", results["facet_counts"][0]["counts"][0]["parent"].dump());
+ ASSERT_EQ("30", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("{\"specification\":{\"detail\":{\"width\":25}}}", results["facet_counts"][0]["counts"][1]["parent"].dump());
+ ASSERT_EQ("25", results["facet_counts"][0]["counts"][1]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetingReturnParentObject) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value", "type": "object", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "value": {
+ "color": "red",
+ "r": 255,
+ "g": 0,
+ "b": 0
+ }
+ })"_json;
+
+ nlohmann::json doc2 = R"({
+ "value": {
+ "color": "blue",
+ "r": 0,
+ "g": 0,
+ "b": 255
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto search_op = coll1->search("*", {},"", {"value.color"},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),10, "",
+ 30, 4, "",
+ Index::TYPO_TOKENS_THRESHOLD, "", "",{},
+ 3, "<mark>", "</mark>", {},
+ UINT32_MAX, true, false, true,
+ "", false, 6000*1000, 4, 7,
+ fallback, 4, {off}, INT16_MAX, INT16_MAX,
+ 2, 2, false, "",
+ true, 0, max_score, 100,
+ 0, 0, HASH, 30000,
+ 2, "", {"value.color"});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("{\"b\":0,\"color\":\"red\",\"g\":0,\"r\":255}", results["facet_counts"][0]["counts"][0]["parent"].dump());
+ ASSERT_EQ("red", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("{\"b\":255,\"color\":\"blue\",\"g\":0,\"r\":0}", results["facet_counts"][0]["counts"][1]["parent"].dump());
+ ASSERT_EQ("blue", results["facet_counts"][0]["counts"][1]["value"]);
+}
+
+
+TEST_F(CollectionFacetingTest, FacetSortByAlpha) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "phone", "type": "string", "optional": false, "facet": true },
+ {"name": "brand", "type": "string", "optional": false, "facet": true },
+ {"name": "rating", "type": "float", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll1 = op.get();
+
+ nlohmann::json doc;
+
+ doc["phone"] = "Oneplus 11R";
+ doc["brand"] = "Oneplus";
+ doc["rating"] = 4.6;
+ auto add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "Fusion Plus";
+ doc["brand"] = "Moto";
+ doc["rating"] = 4.2;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "S22 Ultra";
+ doc["brand"] = "Samsung";
+ doc["rating"] = 4.1;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "GT Master";
+ doc["brand"] = "Realme";
+ doc["rating"] = 4.4;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "T2";
+ doc["brand"] = "Vivo";
+ doc["rating"] = 4.0;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "Mi 6";
+ doc["brand"] = "Xiaomi";
+ doc["rating"] = 3.9;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "Z6 Lite";
+ doc["brand"] = "Iqoo";
+ doc["rating"] = 4.3;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ //sort facets by phone in asc order
+ auto search_op = coll1->search("*", {}, "", {"phone(sort_by:_alpha:asc)"},
+ {}, {2});
+
+ if (!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(7, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Fusion Plus", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("GT Master", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("Mi 6", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("Oneplus 11R", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_EQ("S22 Ultra", results["facet_counts"][0]["counts"][4]["value"]);
+ ASSERT_EQ("T2", results["facet_counts"][0]["counts"][5]["value"]);
+ ASSERT_EQ("Z6 Lite", results["facet_counts"][0]["counts"][6]["value"]);
+
+ //sort facets by brand in desc order
+ search_op = coll1->search("*", {}, "", {"brand(sort_by:_alpha:desc)"},
+ {}, {2});
+
+ if (!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+
+ results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(7, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Xiaomi", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("Vivo", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("Samsung", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("Realme", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_EQ("Oneplus", results["facet_counts"][0]["counts"][4]["value"]);
+ ASSERT_EQ("Moto", results["facet_counts"][0]["counts"][5]["value"]);
+ ASSERT_EQ("Iqoo", results["facet_counts"][0]["counts"][6]["value"]);
+
+ //sort facets by brand in desc order and phone by asc order
+ search_op = coll1->search("*", {}, "", {"brand(sort_by:_alpha:desc)",
+ "phone(sort_by:_alpha:asc)"},
+ {}, {2});
+
+ if (!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+
+ results = search_op.get();
+ ASSERT_EQ(2, results["facet_counts"].size());
+
+ ASSERT_EQ(7, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Xiaomi", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("Vivo", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("Samsung", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("Realme", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_EQ("Oneplus", results["facet_counts"][0]["counts"][4]["value"]);
+ ASSERT_EQ("Moto", results["facet_counts"][0]["counts"][5]["value"]);
+ ASSERT_EQ("Iqoo", results["facet_counts"][0]["counts"][6]["value"]);
+
+ ASSERT_EQ(7, results["facet_counts"][1]["counts"].size());
+ ASSERT_EQ("Fusion Plus", results["facet_counts"][1]["counts"][0]["value"]);
+ ASSERT_EQ("GT Master", results["facet_counts"][1]["counts"][1]["value"]);
+ ASSERT_EQ("Mi 6", results["facet_counts"][1]["counts"][2]["value"]);
+ ASSERT_EQ("Oneplus 11R", results["facet_counts"][1]["counts"][3]["value"]);
+ ASSERT_EQ("S22 Ultra", results["facet_counts"][1]["counts"][4]["value"]);
+ ASSERT_EQ("T2", results["facet_counts"][1]["counts"][5]["value"]);
+ ASSERT_EQ("Z6 Lite", results["facet_counts"][1]["counts"][6]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetSortByOtherField) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "receipe", "type": "object", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "receipe": {
+ "name": "cheese pizza",
+ "calories": 300,
+ "origin": "america"
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc2 = R"({
+ "receipe": {
+ "name": "noodles",
+ "calories": 250,
+ "origin": "china"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc3 = R"({
+ "receipe": {
+ "name": "hamburger",
+ "calories": 350,
+ "origin": "america"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc3.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc4 = R"({
+ "receipe": {
+ "name": "schezwan rice",
+ "calories": 150,
+ "origin": "china"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc4.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc5 = R"({
+ "receipe": {
+ "name": "butter chicken",
+ "calories": 270,
+ "origin": "india"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc5.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ //search by calories in asc order
+ auto search_op = coll1->search("*", {},"",
+ {"receipe.name(sort_by:receipe.calories:asc)"},
+ {}, {2});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("schezwan rice", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("noodles", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("butter chicken", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("cheese pizza", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_EQ("hamburger", results["facet_counts"][0]["counts"][4]["value"]);
+
+ //search by calories in desc order
+ search_op = coll1->search("*", {},"",
+ {"receipe.name(sort_by:receipe.calories:desc)"},
+ {}, {2});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ results = search_op.get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("hamburger", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("cheese pizza", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("butter chicken", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("noodles", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_EQ("schezwan rice", results["facet_counts"][0]["counts"][4]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetSortByOtherFloatField) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "investment", "type": "object", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ nlohmann::json doc1 = R"({
+ "investment": {
+ "name": "Term Deposits",
+ "interest_rate": 7.1,
+ "class": "fixed"
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc2 = R"({
+ "investment": {
+ "name": "Gold",
+ "interest_rate": 5.4,
+ "class": "fixed"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc2.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc3 = R"({
+ "investment": {
+ "name": "Mutual Funds",
+ "interest_rate": 12,
+ "class": "Equity"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc3.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc4 = R"({
+ "investment": {
+ "name": "Land",
+ "interest_rate": 9.1,
+ "class": "real estate"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc4.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json doc5 = R"({
+ "investment": {
+ "name": "Bonds",
+ "interest_rate": 7.24,
+ "class": "g-sec"
+ }
+ })"_json;
+
+ add_op = coll1->add(doc5.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ //search by calories in asc order
+ auto search_op = coll1->search("*", {},"",
+ {"investment.name(sort_by:investment.interest_rate:asc)"},
+ {}, {2});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ auto results = search_op.get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Gold", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("Term Deposits", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("Bonds", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("Land", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_EQ("Mutual Funds", results["facet_counts"][0]["counts"][4]["value"]);
+
+ //search by calories in desc order
+ search_op = coll1->search("*", {},"",
+ {"investment.name(sort_by:investment.interest_rate:desc)"},
+ {}, {2});
+
+ if(!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+ results = search_op.get();
+
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(5, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Mutual Funds", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("Land", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("Bonds", results["facet_counts"][0]["counts"][2]["value"]);
+ ASSERT_EQ("Term Deposits", results["facet_counts"][0]["counts"][3]["value"]);
+ ASSERT_EQ("Gold", results["facet_counts"][0]["counts"][4]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FacetSortValidation) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "phone", "type": "string", "optional": false, "facet": true },
+ {"name": "brand", "type": "string", "optional": false, "facet": true },
+ {"name": "rating", "type": "float", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll1 = op.get();
+
+ nlohmann::json doc;
+
+ doc["phone"] = "Oneplus 11R";
+ doc["brand"] = "Oneplus";
+ doc["rating"] = 4.6;
+ auto add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "Fusion Plus";
+ doc["brand"] = "Moto";
+ doc["rating"] = 4.2;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ doc["phone"] = "S22 Ultra";
+ doc["brand"] = "Samsung";
+ doc["rating"] = 4.1;
+ add_op = coll1->add(doc.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ //try sort on non string field
+ auto search_op = coll1->search("*", {}, "", {"rating(sort_by:_alpha:desc)"},
+ {}, {2});
+
+ ASSERT_EQ(400, search_op.code());
+ ASSERT_EQ("Facet field should be string type to apply alpha sort.", search_op.error());
+
+ //try sort by string field
+ search_op = coll1->search("*", {}, "", {"phone(sort_by:brand:desc)"},
+ {}, {2});
+
+ ASSERT_EQ(400, search_op.code());
+ ASSERT_EQ("Sort field should be non string type to apply sort.", search_op.error());
+
+ //incorrect syntax
+ search_op = coll1->search("*", {}, "", {"phone(sort_by:desc)"},
+ {}, {2});
+
+ ASSERT_EQ(400, search_op.code());
+ ASSERT_EQ("Invalid sort format.", search_op.error());
+
+ search_op = coll1->search("*", {}, "", {"phone(sort:_alpha:desc)"},
+ {}, {2});
+
+ ASSERT_EQ(400, search_op.code());
+ ASSERT_EQ("Invalid sort format.", search_op.error());
+
+ //invalid param
+ search_op = coll1->search("*", {}, "", {"phone(sort_by:_alpha:foo)"},
+ {}, {2});
+
+ ASSERT_EQ(400, search_op.code());
+ ASSERT_EQ("Invalid sort param.", search_op.error());
+
+ //whitespace is allowed
+ search_op = coll1->search("*", {}, "", {"phone( sort_by: _alpha : asc)"},
+ {}, {2});
+
+ if (!search_op.ok()) {
+ LOG(ERROR) << search_op.error();
+ FAIL();
+ }
+
+ auto results = search_op.get();
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("Fusion Plus", results["facet_counts"][0]["counts"][0]["value"]);
+ ASSERT_EQ("Oneplus 11R", results["facet_counts"][0]["counts"][1]["value"]);
+ ASSERT_EQ("S22 Ultra", results["facet_counts"][0]["counts"][2]["value"]);
+}
+
+TEST_F(CollectionFacetingTest, FhashInt64MapTest) {
- std::vector<int64_t> visitors = {227489798,124098972,180247624};
++ std::vector<int64_t> visitors = {227489798, 124098972, 180247624};
+ facet_index_t facet_index_v4;
+ std::unordered_map<facet_value_id_t, std::vector<uint32_t>, facet_value_id_t::Hash> fvalue_to_seq_ids;
+ std::unordered_map<uint32_t, std::vector<facet_value_id_t>> seq_id_to_fvalues;
+
+ facet_index_v4.initialize("visitors");
+
+ //insert timestamps
+ int seq_id = 0;
- for(auto it = visitors.begin(); it != visitors.end(); ++it) {
++ for (auto it = visitors.begin(); it != visitors.end(); ++it) {
+ auto val = std::to_string(*it);
+ facet_value_id_t facet_value_id(val);
+ fvalue_to_seq_ids[facet_value_id].push_back(seq_id);
+ seq_id_to_fvalues[seq_id].push_back(facet_value_id);
+ ++seq_id;
+ }
+
+ facet_index_v4.insert("visitors", fvalue_to_seq_ids, seq_id_to_fvalues);
+ ASSERT_EQ(3, facet_index_v4.get_fhash_int64_map("visitors").size());
+
+ facet_index_v4.remove("visitors", 0);
+ ASSERT_EQ(2, facet_index_v4.get_fhash_int64_map("visitors").size());
+
+ fvalue_to_seq_ids.clear();
+ seq_id_to_fvalues.clear();
+
+ facet_value_id_t facet_value_id("124798721");
+ fvalue_to_seq_ids[facet_value_id].push_back(seq_id);
+ seq_id_to_fvalues[seq_id].push_back(facet_value_id);
+
+ facet_index_v4.insert("visitors", fvalue_to_seq_ids, seq_id_to_fvalues);
+ ASSERT_EQ(3, facet_index_v4.get_fhash_int64_map("visitors").size());
- }
++}
++
+ TEST_F(CollectionFacetingTest, RangeFacetTestWithGroupBy) {
+ std::vector<field> fields = {field("place", field_types::STRING, false),
+ field("state", field_types::STRING, true),
+ field("visitors", field_types::INT32, true),
+ field("rating", field_types::FLOAT, true),
+ field("trackingFrom", field_types::INT32, true),};
+ Collection* coll1 = collectionManager.create_collection(
+ "coll1", 1, fields, "", 0, "", {}, {}
+ ).get();
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["place"] = "Mysore Palace";
+ doc1["state"] = "Karnataka";
+ doc1["visitors"] = 235486;
+ doc1["rating"] = 4.5;
+ doc1["trackingFrom"] = 1900;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["place"] = "Hampi";
+ doc2["state"] = "Karnataka";
+ doc2["visitors"] = 201022;
+ doc2["rating"] = 4.5;
+ doc2["trackingFrom"] = 1900;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["place"] = "Mahabalipuram";
+ doc3["state"] = "TamilNadu";
+ doc3["visitors"] = 174684;
+ doc3["rating"] = 3.8;
+ doc3["trackingFrom"] = 1900;
+
+ nlohmann::json doc4;
+ doc4["id"] = "3";
+ doc4["place"] = "Meenakshi Amman Temple";
+ doc4["state"] = "TamilNadu";
+ doc4["visitors"] = 246676;
+ doc4["rating"] = 4.5;
+ doc4["trackingFrom"] = 2000;
+
+ nlohmann::json doc5;
+ doc5["id"] = "4";
+ doc5["place"] = "Staue of Unity";
+ doc5["state"] = "Gujarat";
+ doc5["visitors"] = 345878;
+ doc5["rating"] = 3.8;
+ doc5["trackingFrom"] = 2000;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc5.dump()).ok());
+
+ auto result = coll1->search("Karnataka", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true);
+ if(!result.ok()) {
+ LOG(INFO) << result.error();
+ }
+
+ auto results = result.get();
+
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_EQ("VeryBusy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
+
+ //apply group_by
+ result = coll1->search("*", {"state"},
+ "", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},
+ {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 10,
+ {}, {}, {"rating"}, 10,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", true);
+
+ if(!result.ok()) {
+ LOG(INFO) << result.error();
+ }
+
+ results = result.get();
+
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_EQ("VeryBusy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
+ ASSERT_EQ("Busy", results["facet_counts"][0]["counts"][1]["value"].get<std::string>());
+
+ ASSERT_EQ(2, results["grouped_hits"].size());
+
+ ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());
+ ASSERT_EQ(3.8, results["grouped_hits"][0]["group_key"][0]);
+
+ ASSERT_EQ(3, results["grouped_hits"][1]["hits"].size());
+ ASSERT_EQ(4.5, results["grouped_hits"][1]["group_key"][0]);
+
+ collectionManager.drop_collection("coll1");
-}
++}
diff --cc test/collection_grouping_test.cpp
index 1f9843e9,8a4a3330..23815f75
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@@ -606,6 -606,77 +606,79 @@@ TEST_F(CollectionGroupingTest, Repeated
ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());
}
+ TEST_F(CollectionGroupingTest, ControlMissingValues) {
+ std::vector<field> fields = {
+ field("brand", field_types::STRING, true, true),
+ };
+
+ Collection* coll2 = collectionManager.get_collection("coll2").get();
+ if(coll2 == nullptr) {
+ coll2 = collectionManager.create_collection("coll2", 1, fields).get();
+ }
+
++ LOG(INFO) << "----------------------";
++
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["brand"] = "Omega";
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["brand"] = nullptr;
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "2";
+ doc["brand"] = nullptr;
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ doc["id"] = "3";
+ doc["brand"] = "Omega";
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ // disable null value aggregation
+
+ auto res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, 30000, 2, true, false).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, false).get();
+
+ ASSERT_EQ(3, res["grouped_hits"].size());
+ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());
+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
+ ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
+ ASSERT_EQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ(0, res["grouped_hits"][2]["group_key"].size());
+ ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());
+ ASSERT_EQ("1", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>());
+
+ // with null value aggregation (default)
+ res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10,
+ {}, {}, {"brand"}, 2,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, 30000, 2, true, true).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true, true).get();
+
+ ASSERT_EQ(2, res["grouped_hits"].size());
+
+ ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());
+ ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
+
+ ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
+ ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());
+ }
+
TEST_F(CollectionGroupingTest, SortingOnGroupCount) {
std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};
diff --cc test/collection_schema_change_test.cpp
index c936ce78,7e454860..5030370c
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@@ -1793,3 -1793,46 +1793,45 @@@ TEST_F(CollectionSchemaChangeTest, Embe
ASSERT_EQ(0, vec_index.size());
ASSERT_EQ(0, vec_index.count("embedding"));
}
+
-
+ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterUpdateOldDocs) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "nested", "type": "object"}
+ ],
+ "enable_nested_fields": true
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+
+ nlohmann::json doc;
+ doc["title"] = "hello";
+ doc["nested"] = nlohmann::json::object();
+ doc["nested"]["hello"] = "world";
+
+ auto add_op = coll->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json schema_change = R"({
+ "fields": [
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["title"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ auto schema_change_op = coll->alter(schema_change);
+ ASSERT_TRUE(schema_change_op.ok());
+
+ auto search_res = coll->search("*", {}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5);
+
+ ASSERT_EQ(1, search_res.get()["found"].get<size_t>());
+ ASSERT_EQ(384, search_res.get()["hits"][0]["document"]["embedding"].get<std::vector<float>>().size());
+ ASSERT_EQ(1, search_res.get()["hits"][0]["document"]["nested"].size());
+ ASSERT_EQ(0, search_res.get()["hits"][0]["document"].count(".flat"));
+ ASSERT_EQ(0, search_res.get()["hits"][0]["document"].count("nested.hello"));
+ }
diff --cc test/collection_specific_more_test.cpp
index d6538c86,266eb0a6..c89d761a
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@@ -1856,6 -1846,62 +1856,62 @@@ TEST_F(CollectionSpecificMoreTest, Cons
ASSERT_EQ("0", res["hits"][1]["document"]["id"].get<std::string>());
}
+ TEST_F(CollectionSpecificMoreTest, DisableFieldCountForScoring) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "brand", "type": "string"}
+ ]
+ })"_json;
+
+ Collection *coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["name"] = "Alpha beta gamma";
+ doc["brand"] = "Alpha beta gamma";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["name"] = "Alpha beta gamma";
+ doc["brand"] = "Theta";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto res_op = coll1->search("beta", {"name", "brand"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, 30000, 2, true);
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true);
+
+
+ auto res = coll1->search("beta", {"name", "brand"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, 30000, 2, false).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, false).get();
+
+ size_t score1 = std::stoul(res["hits"][0]["text_match_info"]["score"].get<std::string>());
+ size_t score2 = std::stoul(res["hits"][1]["text_match_info"]["score"].get<std::string>());
+ ASSERT_TRUE(score1 == score2);
+
+ res = coll1->search("beta", {"name", "brand"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
- 100, 0, 0, 30000, 2, true).get();
++ 100, 0, 0, HASH, 30000, 2, "", {}, {}, right_to_left, true).get();
+
+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", res["hits"][1]["document"]["id"].get<std::string>());
+
+ score1 = std::stoul(res["hits"][0]["text_match_info"]["score"].get<std::string>());
+ score2 = std::stoul(res["hits"][1]["text_match_info"]["score"].get<std::string>());
+ ASSERT_TRUE(score1 > score2);
+ }
+
TEST_F(CollectionSpecificMoreTest, NonNestedFieldNameWithDot) {
nlohmann::json schema = R"({
"name": "coll1",
diff --cc test/collection_vector_search_test.cpp
index 0c31beae,c3a608ba..34f5a405
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -2038,4 -2019,200 +2038,199 @@@ TEST_F(CollectionVectorTest, TestMultil
0, spp::sparse_hash_set<std::string>());
ASSERT_TRUE(semantic_results.ok());
- }
+ }
+
+
+ TEST_F(CollectionVectorTest, TestTwoEmbeddingFieldsSamePrefix) {
+ nlohmann::json schema = R"({
+ "name": "docs",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ },
+ {
+ "name": "embedding_en",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "title": "john doe"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto semantic_results = coll1->search("john", {"embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>());
+
+ ASSERT_TRUE(semantic_results.ok());
+ }
+
+ TEST_F(CollectionVectorTest, TestOneEmbeddingOneKeywordFieldsHaveSamePrefix) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "title_vec",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "title": "john doe"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto keyword_results = coll1->search("john", {"title"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>());
+
+ ASSERT_TRUE(keyword_results.ok());
+ }
+
+ TEST_F(CollectionVectorTest, HybridSearchOnlyKeyworMatchDoNotHaveVectorDistance) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "embed": {
+ "from": [
+ "title"
+ ],
+ "model_config": {
+ "model_name": "ts/e5-small"
+ }
+ }
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "title": "john doe"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ // hybrid search with empty vector (to pass distance threshold param)
+ std::string vec_query = "embedding:([], distance_threshold: 0.05)";
+
+ auto hybrid_results = coll1->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, vec_query);
+
+ ASSERT_TRUE(hybrid_results.ok());
+
+ ASSERT_EQ(1, hybrid_results.get()["hits"].size());
+ ASSERT_EQ(0, hybrid_results.get()["hits"][0].count("vector_distance"));
+ }
+
-
+ TEST_F(CollectionVectorTest, QueryByNotAutoEmbeddingVectorField) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {
+ "name": "title",
+ "type": "string"
+ },
+ {
+ "name": "embedding",
+ "type": "float[]",
+ "num_dim": 384
+ }
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ auto coll = collection_create_op.get();
+
+ auto search_res = coll->search("john", {"title", "embedding"}, "", {}, {}, {0}, 20, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
+ fallback,
+ 4, {off}, 32767, 32767, 2,
+ false, true, "embedding:([0.96826, 0.94, 0.39557, 0.306488])");
+
+ ASSERT_FALSE(search_res.ok());
+
+ ASSERT_EQ("Vector field `embedding` is not an auto-embedding field, do not use `query_by` with it, use `vector_query` instead.", search_res.error());
-}
++}