Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Sort by reference field.

Harpreet Sangarf41afc155fb0
diff --git a/include/collection.h b/include/collection.h
index 88b8d750..205737d7 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -214,6 +214,11 @@ private:
bool is_wildcard_query,const bool is_vector_query,
bool is_group_by_query = false) const;
+ Option<bool> validate_and_standardize_sort_fields_with_lock(const std::vector<sort_by> & sort_fields,
+ std::vector<sort_by>& sort_fields_std,
+ bool is_wildcard_query,const bool is_vector_query,
+ bool is_group_by_query = false) const;
+
Option<bool> persist_collection_meta();
@@ -569,6 +574,12 @@ public:
std::vector<std::string>& reordered_search_fields) const;
Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);
+
+ void reference_populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,
+ std::vector<sort_by>& sort_fields_std,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;
+
+ int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;
};
template<class T>
diff --git a/include/field.h b/include/field.h
index a051c7ee..44920158 100644
--- a/include/field.h
+++ b/include/field.h
@@ -513,6 +513,8 @@ struct sort_by {
missing_values_t missing_values;
eval_t eval;
+ std::string reference_collection_name;
+
sort_by(const std::string & name, const std::string & order):
name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),
missing_values(normal) {
@@ -527,6 +529,20 @@ struct sort_by {
}
+ sort_by(const sort_by& other) {
+ if (&other == this)
+ return;
+ name = other.name;
+ order = other.order;
+ text_match_buckets = other.text_match_buckets;
+ geopoint = other.geopoint;
+ exclude_radius = other.exclude_radius;
+ geo_precision = other.geo_precision;
+ missing_values = other.missing_values;
+ eval = other.eval;
+ reference_collection_name = other.reference_collection_name;
+ }
+
sort_by& operator=(const sort_by& other) {
name = other.name;
order = other.order;
@@ -536,6 +552,7 @@ struct sort_by {
geo_precision = other.geo_precision;
missing_values = other.missing_values;
eval = other.eval;
+ reference_collection_name = other.reference_collection_name;
return *this;
}
};
diff --git a/include/index.h b/include/index.h
index bc961ec2..b4b6da0c 100644
--- a/include/index.h
+++ b/include/index.h
@@ -424,36 +424,36 @@ private:
size_t min_len_2typo,
size_t max_candidates) const;
- void search_all_candidates(const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<search_field_t>& the_fields,
- filter_result_iterator_t* const filter_result_iterator,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by>& sort_fields,
- std::vector<tok_candidates>& token_candidates_vec,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const size_t typo_tokens_threshold,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool exhaustive_search,
- const size_t max_candidates,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const;
+ Option<bool> search_all_candidates(const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<search_field_t>& the_fields,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<tok_candidates>& token_candidates_vec,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ const bool exhaustive_search,
+ const size_t max_candidates,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff) const;
void search_candidates(const uint8_t & field_id,
bool field_is_array,
@@ -696,20 +696,20 @@ public:
// the following methods are not synchronized because their parent calls are synchronized or they are const/static
- void search_wildcard(filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- filter_result_iterator_t* const filter_result_iterator,
- const size_t concurrency,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const;
+ Option<bool> search_wildcard(filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ const size_t concurrency,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const;
void search_infix(const std::string& query, const std::string& field_name, std::vector<uint32_t>& ids,
size_t max_extra_prefix, size_t max_extra_suffix) const;
@@ -722,6 +722,12 @@ public:
std::vector<sort_by>& sort_fields_std,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;
+ void populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>& geopoint_indices,
+ std::vector<sort_by>& sort_fields_std,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;
+
+ int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;
+
static void remove_matched_tokens(std::vector<std::string>& tokens, const std::set<std::string>& rule_token_set) ;
void compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,
@@ -741,100 +747,100 @@ public:
const std::vector<search_field_t>& search_fields, uint32_t*& exclude_token_ids,
size_t& exclude_token_ids_size) const;
- void do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
- const std::vector<enable_t>& infixes,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
- const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,
- filter_result_iterator_t* const filter_result_iterator,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;
-
- void do_synonym_search(const std::vector<search_field_t>& the_fields,
- const text_match_type_t match_type,
- filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
- const token_ordering& token_order,
- const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool exhaustive_search, const size_t concurrency,
- const std::vector<bool>& prefixes,
- size_t min_len_1typo,
- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* actual_topster,
- std::vector<std::vector<token_t>>& q_pos_synonyms,
- int syn_orig_num_tokens,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- filter_result_iterator_t* const filter_result_iterator,
- std::set<uint64>& query_hashes,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- tsl::htrie_map<char, token_leaf>& qtoken_set) const;
-
- void do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
- std::vector<query_tokens_t>& field_query_tokens,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- Topster* actual_topster,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- filter_result_iterator_t*& filter_result_iterator,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- const std::set<uint32_t>& curated_ids,
- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* curated_topster,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- bool is_wildcard_query) const;
-
- void fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
- const std::vector<token_t>& query_tokens,
- const std::vector<token_t>& dropped_tokens,
- const text_match_type_t match_type,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- filter_result_iterator_t* const filter_result_iterator,
- const std::vector<uint32_t>& curated_ids,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by>& sort_fields,
- const std::vector<uint32_t>& num_typos,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const size_t group_limit, const std::vector<std::string>& group_by_fields,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- std::set<uint64>& query_hashes,
- const token_ordering token_order,
- const std::vector<bool>& prefixes,
- const size_t typo_tokens_threshold,
- const bool exhaustive_search,
- const size_t max_candidates,
- size_t min_len_1typo,
- size_t min_len_2typo,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const;
+ Option<bool> do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
+ const std::vector<enable_t>& infixes,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
+ const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,
+ filter_result_iterator_t* const filter_result_iterator,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;
+
+ [[nodiscard]] Option<bool> do_synonym_search(const std::vector<search_field_t>& the_fields,
+ const text_match_type_t match_type,
+ filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
+ const token_ordering& token_order,
+ const size_t typo_tokens_threshold, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ const bool exhaustive_search, const size_t concurrency,
+ const std::vector<bool>& prefixes,
+ size_t min_len_1typo,
+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* actual_topster,
+ std::vector<std::vector<token_t>>& q_pos_synonyms,
+ int syn_orig_num_tokens,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ std::set<uint64>& query_hashes,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ tsl::htrie_map<char, token_leaf>& qtoken_set) const;
+
+ Option<bool> do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
+ std::vector<query_tokens_t>& field_query_tokens,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ Topster* actual_topster,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ filter_result_iterator_t*& filter_result_iterator,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::set<uint32_t>& curated_ids,
+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* curated_topster,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ bool is_wildcard_query) const;
+
+ [[nodiscard]] Option<bool> fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<token_t>& dropped_tokens,
+ const text_match_type_t match_type,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ filter_result_iterator_t* const filter_result_iterator,
+ const std::vector<uint32_t>& curated_ids,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by>& sort_fields,
+ const std::vector<uint32_t>& num_typos,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ std::set<uint64>& query_hashes,
+ const token_ordering token_order,
+ const std::vector<bool>& prefixes,
+ const size_t typo_tokens_threshold,
+ const bool exhaustive_search,
+ const size_t max_candidates,
+ size_t min_len_1typo,
+ size_t min_len_2typo,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const;
void find_across_fields(const token_t& previous_token,
const std::string& previous_token_str,
@@ -846,34 +852,34 @@ public:
std::vector<uint32_t>& prev_token_doc_ids,
std::vector<size_t>& top_prefix_field_ids) const;
- void search_across_fields(const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- const std::vector<search_field_t>& the_fields,
- const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<sort_by>& sort_fields,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- bool prioritize_exact_match,
- const bool search_all_candidates,
- filter_result_iterator_t* const filter_result_iterator,
- const uint32_t total_cost,
- const int syn_orig_num_tokens,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::vector<uint32_t>& id_buff,
- uint32_t*& all_result_ids,
- size_t& all_result_ids_len) const;
+ Option<bool> search_across_fields(const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ const std::vector<search_field_t>& the_fields,
+ const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<sort_by>& sort_fields,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ bool prioritize_exact_match,
+ const bool search_all_candidates,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t total_cost,
+ const int syn_orig_num_tokens,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::vector<uint32_t>& id_buff,
+ uint32_t*& all_result_ids,
+ size_t& all_result_ids_len) const;
void
search_fields(const std::vector<filter>& filters,
@@ -903,13 +909,14 @@ public:
filter_node_t*& filter_tree_root,
std::vector<const override_t*>& matched_dynamic_overrides) const;
- void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices, uint32_t seq_id,
- size_t filter_index,
- int64_t max_field_match_score,
- int64_t* scores,
- int64_t& match_score_index, float vector_distance = 0) const;
+ Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices, uint32_t seq_id,
+ const std::map<basic_string<char>, reference_filter_result_t>& references,
+ size_t filter_index,
+ int64_t max_field_match_score,
+ int64_t* scores,
+ int64_t& match_score_index, float vector_distance = 0) const;
void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,
diff --git a/include/topster.h b/include/topster.h
index 5985a48c..0b106f47 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -33,7 +33,7 @@ struct KV {
this->scores[1] = scores[1];
this->scores[2] = scores[2];
- if(match_score_index >= 0) {
+ if(match_score_index >= 0 && match_score_index <= 2){
this->text_match_score = scores[match_score_index];
}
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 65afff8e..6d259e01 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -757,6 +757,44 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
for(size_t i = 0; i < sort_fields.size(); i++) {
const sort_by& _sort_field = sort_fields[i];
+
+ if (_sort_field.name[0] == '$') {
+ // Reference sort_by
+ auto parenthesis_index = _sort_field.name.find('(');
+ std::string ref_collection_name = _sort_field.name.substr(1, parenthesis_index - 1);
+ auto& cm = CollectionManager::get_instance();
+ auto ref_collection = cm.get_collection(ref_collection_name);
+ if (ref_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `sort_by` not found.");
+ }
+
+ auto sort_by_str = _sort_field.name.substr(parenthesis_index + 1,
+ _sort_field.name.size() - parenthesis_index - 2);
+ std::vector<sort_by> ref_sort_fields;
+ bool parsed_sort_by = CollectionManager::parse_sort_by_str(sort_by_str, ref_sort_fields);
+ if (!parsed_sort_by) {
+ return Option<bool>(400, "Reference `sort_by` is malformed.");
+ }
+
+ std::vector<sort_by> ref_sort_fields_std;
+ auto sort_validation_op = ref_collection->validate_and_standardize_sort_fields_with_lock(ref_sort_fields,
+ ref_sort_fields_std,
+ is_wildcard_query,
+ is_vector_query,
+ is_group_by_query);
+ if (!sort_validation_op.ok()) {
+ return Option<bool>(sort_validation_op.code(), "Referenced collection `" + ref_collection_name + "`: " +
+ sort_validation_op.error());
+ }
+
+ for (auto& ref_sort_field_std: ref_sort_fields_std) {
+ ref_sort_field_std.reference_collection_name = ref_collection_name;
+ sort_fields_std.emplace_back(ref_sort_field_std);
+ }
+
+ continue;
+ }
+
sort_by sort_field_std(_sort_field.name, _sort_field.order);
if(sort_field_std.name.back() == ')') {
@@ -1012,6 +1050,214 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
return Option<bool>(true);
}
+Option<bool> Collection::validate_and_standardize_sort_fields_with_lock(const std::vector<sort_by> & sort_fields,
+ std::vector<sort_by>& sort_fields_std,
+ const bool is_wildcard_query,
+ const bool is_vector_query,
+ const bool is_group_by_query) const {
+ std::shared_lock lock(mutex);
+
+ for(const auto & sort_field : sort_fields) {
+ sort_by sort_field_std(sort_field.name, sort_field.order);
+
+ if(sort_field_std.name.back() == ')') {
+ // check if this is a geo field or text match field
+ size_t paran_start = 0;
+ while(paran_start < sort_field_std.name.size() && sort_field_std.name[paran_start] != '(') {
+ paran_start++;
+ }
+
+ const std::string& actual_field_name = sort_field_std.name.substr(0, paran_start);
+ const auto field_it = search_schema.find(actual_field_name);
+
+ if(actual_field_name == sort_field_const::text_match) {
+ std::vector<std::string> match_parts;
+ const std::string& match_config = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);
+ StringUtils::split(match_config, match_parts, ":");
+ if(match_parts.size() != 2 || match_parts[0] != "buckets") {
+ return Option<bool>(400, "Invalid sorting parameter passed for _text_match.");
+ }
+
+ if(!StringUtils::is_uint32_t(match_parts[1])) {
+ return Option<bool>(400, "Invalid value passed for _text_match `buckets` configuration.");
+ }
+
+ sort_field_std.name = actual_field_name;
+ sort_field_std.text_match_buckets = std::stoll(match_parts[1]);
+
+ } else if(actual_field_name == sort_field_const::eval) {
+ const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,
+ sort_field_std.name.size() - paran_start -
+ 2);
+ if(filter_exp.empty()) {
+ return Option<bool>(400, "The eval expression in sort_by is empty.");
+ }
+
+ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,
+ store, "", sort_field_std.eval.filter_tree_root);
+ if(!parse_filter_op.ok()) {
+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");
+ }
+
+ sort_field_std.name = actual_field_name;
+
+ } else {
+ if(field_it == search_schema.end()) {
+ std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";
+ return Option<bool>(404, error);
+ }
+
+ std::string error = "Bad syntax for sorting field `" + actual_field_name + "`";
+
+ if(!field_it.value().is_geopoint()) {
+ // check for null value order
+ const std::string& sort_params_str = sort_field_std.name.substr(paran_start + 1,
+ sort_field_std.name.size() -
+ paran_start - 2);
+
+ std::vector<std::string> param_parts;
+ StringUtils::split(sort_params_str, param_parts, ":");
+
+ if(param_parts.size() != 2) {
+ return Option<bool>(400, error);
+ }
+
+ if(param_parts[0] != sort_field_const::missing_values) {
+ return Option<bool>(400, error);
+ }
+
+ auto missing_values_op = magic_enum::enum_cast<sort_by::missing_values_t>(param_parts[1]);
+ if(missing_values_op.has_value()) {
+ sort_field_std.missing_values = missing_values_op.value();
+ } else {
+ return Option<bool>(400, error);
+ }
+ }
+
+ else {
+ const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);
+
+ // e.g. geopoint_field(lat1, lng1, exclude_radius: 10 miles)
+
+ std::vector<std::string> geo_parts;
+ StringUtils::split(geo_coordstr, geo_parts, ",");
+
+ if(geo_parts.size() != 2 && geo_parts.size() != 3) {
+ return Option<bool>(400, error);
+ }
+
+ if(!StringUtils::is_float(geo_parts[0]) || !StringUtils::is_float(geo_parts[1])) {
+ return Option<bool>(400, error);
+ }
+
+ if(geo_parts.size() == 3) {
+ // try to parse the exclude radius option
+ bool is_exclude_option = false;
+
+ if(StringUtils::begins_with(geo_parts[2], sort_field_const::exclude_radius)) {
+ is_exclude_option = true;
+ } else if(StringUtils::begins_with(geo_parts[2], sort_field_const::precision)) {
+ is_exclude_option = false;
+ } else {
+ return Option<bool>(400, error);
+ }
+
+ std::vector<std::string> param_parts;
+ StringUtils::split(geo_parts[2], param_parts, ":");
+
+ if(param_parts.size() != 2) {
+ return Option<bool>(400, error);
+ }
+
+ // param_parts[1] is the value, in either "20km" or "20 km" format
+
+ if(param_parts[1].size() < 2) {
+ return Option<bool>(400, error);
+ }
+
+ std::string unit = param_parts[1].substr(param_parts[1].size()-2, 2);
+
+ if(unit != "km" && unit != "mi") {
+ return Option<bool>(400, "Sort field's parameter unit must be either `km` or `mi`.");
+ }
+
+ std::vector<std::string> dist_values;
+ StringUtils::split(param_parts[1], dist_values, unit);
+
+ if(dist_values.size() != 1) {
+ return Option<bool>(400, error);
+ }
+
+ if(!StringUtils::is_float(dist_values[0])) {
+ return Option<bool>(400, error);
+ }
+
+ int32_t value_meters;
+
+ if(unit == "km") {
+ value_meters = std::stof(dist_values[0]) * 1000;
+ } else if(unit == "mi") {
+ value_meters = std::stof(dist_values[0]) * 1609.34;
+ } else {
+ return Option<bool>(400, "Sort field's parameter "
+ "unit must be either `km` or `mi`.");
+ }
+
+ if(value_meters <= 0) {
+ return Option<bool>(400, "Sort field's parameter must be a positive number.");
+ }
+
+ if(is_exclude_option) {
+ sort_field_std.exclude_radius = value_meters;
+ } else {
+ sort_field_std.geo_precision = value_meters;
+ }
+ }
+
+ double lat = std::stod(geo_parts[0]);
+ double lng = std::stod(geo_parts[1]);
+ int64_t lat_lng = GeoPoint::pack_lat_lng(lat, lng);
+ sort_field_std.geopoint = lat_lng;
+ }
+
+ sort_field_std.name = actual_field_name;
+ }
+ }
+
+ if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&
+ sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance) {
+
+ const auto field_it = search_schema.find(sort_field_std.name);
+ if(field_it == search_schema.end() || !field_it.value().sort || !field_it.value().index) {
+ std::string error = "Could not find a field named `" + sort_field_std.name +
+ "` in the schema for sorting.";
+ return Option<bool>(404, error);
+ }
+ }
+
+ if(sort_field_std.name == sort_field_const::group_found && is_group_by_query == false) {
+ std::string error = "group_by parameters should not be empty when using sort_by group_found";
+ return Option<bool>(404, error);
+ }
+
+ if(sort_field_std.name == sort_field_const::vector_distance && !is_vector_query) {
+ std::string error = "sort_by vector_distance is only supported for vector queries, semantic search and hybrid search.";
+ return Option<bool>(404, error);
+ }
+
+ StringUtils::toupper(sort_field_std.order);
+
+ if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {
+ std::string error = "Order for field` " + sort_field_std.name + "` should be either ASC or DESC.";
+ return Option<bool>(400, error);
+ }
+
+ sort_fields_std.emplace_back(sort_field_std);
+ }
+
+ return Option<bool>(true);
+}
+
Option<bool> Collection::extract_field_name(const std::string& field_name,
const tsl::htrie_map<char, field>& search_schema,
std::vector<std::string>& processed_search_fields,
@@ -1526,7 +1772,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
bool is_vector_query = !vector_query.field_name.empty();
if(curated_sort_by.empty()) {
- auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,
+ auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,
sort_fields_std, is_wildcard_query, is_vector_query, is_group_by_query);
if(!sort_validation_op.ok()) {
return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());
@@ -5113,3 +5359,16 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t k
return Option<bool>(true);
}
+
+void Collection::reference_populate_sort_mapping(int *sort_order, std::vector<size_t> &geopoint_indices,
+ std::vector<sort_by> &sort_fields_std,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t> *, 3> &field_values)
+ const {
+ std::shared_lock lock(mutex);
+ index->populate_sort_mapping_with_lock(sort_order, geopoint_indices, sort_fields_std, field_values);
+}
+
+int64_t Collection::reference_string_sort_score(const string &field_name, const uint32_t& seq_id) const {
+ std::shared_lock lock(mutex);
+ return index->reference_string_sort_score(field_name, seq_id);
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 7791b2a9..c47d0bc2 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -623,6 +623,33 @@ bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<s
char prev_non_space_char = 'a';
for(size_t i=0; i < sort_by_str.size(); i++) {
+ if (sort_field_expr.empty() && sort_by_str[i] == '$') {
+ // Sort by reference field
+ auto open_paren_pos = sort_by_str.find('(', i);
+ if (open_paren_pos == std::string::npos) {
+ return false;
+ }
+ sort_field_expr = sort_by_str.substr(i, open_paren_pos - i + 1);
+
+ i = open_paren_pos;
+ int paren_count = 1;
+ while (++i < sort_by_str.size() && paren_count > 0) {
+ if (sort_by_str[i] == '(') {
+ paren_count++;
+ } else if (sort_by_str[i] == ')') {
+ paren_count--;
+ }
+ sort_field_expr += sort_by_str[i];
+ }
+ if (paren_count != 0) {
+ return false;
+ }
+
+ sort_fields.emplace_back(sort_field_expr, "");
+ sort_field_expr = "";
+ continue;
+ }
+
if(i == sort_by_str.size()-1 || (sort_by_str[i] == ',' && !isdigit(prev_non_space_char))) {
if(i == sort_by_str.size()-1) {
sort_field_expr += sort_by_str[i];
diff --git a/src/index.cpp b/src/index.cpp
index db429001..10b73baf 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -22,6 +22,7 @@
#include <timsort.hpp>
#include "logger.h"
#include "validator.h"
+#include <collection_manager.h>
#define RETURN_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \
std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \
@@ -29,6 +30,12 @@
return ;\
}
+#define RETURN_CIRCUIT_BREAKER_OP if((std::chrono::duration_cast<std::chrono::microseconds>( \
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \
+ search_cutoff = true; \
+ return Option<bool>(true);\
+ }
+
#define BREAK_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \
std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \
search_cutoff = true; \
@@ -1431,36 +1438,36 @@ void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {
}
}
-void Index::search_all_candidates(const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<search_field_t>& the_fields,
- filter_result_iterator_t* const filter_result_iterator,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by>& sort_fields,
- std::vector<tok_candidates>& token_candidates_vec,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- const size_t typo_tokens_threshold,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool exhaustive_search,
- const size_t max_candidates,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::set<uint64>& query_hashes,
- std::vector<uint32_t>& id_buff) const {
+Option<bool> Index::search_all_candidates(const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<search_field_t>& the_fields,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<tok_candidates>& token_candidates_vec,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ const size_t typo_tokens_threshold,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ const bool exhaustive_search,
+ const size_t max_candidates,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::set<uint64>& query_hashes,
+ std::vector<uint32_t>& id_buff) const {
/*if(!token_candidates_vec.empty()) {
LOG(INFO) << "Prefix candidates size: " << token_candidates_vec.back().candidates.size();
@@ -1476,7 +1483,7 @@ void Index::search_all_candidates(const size_t num_search_fields,
std::max<size_t>(Index::COMBINATION_MIN_LIMIT, max_candidates);
for(long long n = 0; n < N && n < combination_limit; ++n) {
- RETURN_CIRCUIT_BREAKER
+ RETURN_CIRCUIT_BREAKER_OP
std::vector<token_t> query_suggestion(token_candidates_vec.size());
@@ -1499,19 +1506,26 @@ void Index::search_all_candidates(const size_t num_search_fields,
//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
- search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields, match_type,
- sort_fields, topster,groups_processed,
- searched_queries, qtoken_set, dropped_tokens, group_limit, group_by_fields,
- prioritize_exact_match, prioritize_token_position,
- filter_result_iterator,
- total_cost, syn_orig_num_tokens,
- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
- sort_order, field_values, geopoint_indices,
- id_buff, all_result_ids, all_result_ids_len);
+ auto search_across_fields_op = search_across_fields(query_suggestion, num_typos, prefixes, the_fields,
+ num_search_fields, match_type,
+ sort_fields, topster,groups_processed,
+ searched_queries, qtoken_set, dropped_tokens,
+ group_limit, group_by_fields,
+ prioritize_exact_match, prioritize_token_position,
+ filter_result_iterator,
+ total_cost, syn_orig_num_tokens,
+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
+ sort_order, field_values, geopoint_indices,
+ id_buff, all_result_ids, all_result_ids_len);
+ if (!search_across_fields_op.ok()) {
+ return search_across_fields_op;
+ }
query_hashes.insert(qhash);
filter_result_iterator->reset();
}
+
+ return Option<bool>(true);
}
void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
@@ -2258,16 +2272,21 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
// handle phrase searches
if (!field_query_tokens[0].q_phrases.empty()) {
- do_phrase_search(num_search_fields, the_fields, field_query_tokens,
- sort_fields_std, searched_queries, group_limit, group_by_fields,
- topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,
- filter_result_iterator, all_result_ids, all_result_ids_len, groups_processed, curated_ids,
- excluded_result_ids, excluded_result_ids_size, excluded_group_ids, curated_topster,
- included_ids_map, is_wildcard_query);
+ auto do_phrase_search_op = do_phrase_search(num_search_fields, the_fields, field_query_tokens,
+ sort_fields_std, searched_queries, group_limit, group_by_fields,
+ topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,
+ filter_result_iterator, all_result_ids, all_result_ids_len,
+ groups_processed, curated_ids,
+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
+ curated_topster, included_ids_map, is_wildcard_query);
filter_iterator_guard.release();
filter_iterator_guard.reset(filter_result_iterator);
+ if (!do_phrase_search_op.ok()) {
+ return do_phrase_search_op;
+ }
+
if (filter_result_iterator->approx_filter_ids_length == 0) {
goto process_search_results;
}
@@ -2397,6 +2416,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
for (auto& dist_result : dist_results) {
auto& seq_id = dist_result.second.seq_id;
+ auto references = std::move(dist_result.second.reference_filter_results);
if(vector_query.query_doc_given && vector_query.seq_id == seq_id) {
continue;
@@ -2420,12 +2440,16 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
int64_t scores[3] = {0};
int64_t match_score_index = -1;
- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices,
- seq_id, 0, 0, scores, match_score_index, vec_dist_score);
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
+ geopoint_indices, seq_id, references, 0, 0, scores,
+ match_score_index, vec_dist_score);
+ if (!compute_sort_scores_op.ok()) {
+ return compute_sort_scores_op;
+ }
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
kv.vector_distance = vec_dist_score;
- kv.reference_filter_results = std::move(dist_result.second.reference_filter_results);
+ kv.reference_filter_results = std::move(references);
int ret = topster->add(&kv);
if(group_limit != 0 && ret < 2) {
@@ -2447,13 +2471,16 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
filter_iterator_guard.reset(filter_result_iterator);
}
- search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,
- curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
- curated_ids, curated_ids_sorted,
- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
- all_result_ids, all_result_ids_len,
- filter_result_iterator, concurrency,
- sort_order, field_values, geopoint_indices);
+ auto search_wildcard_op = search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,
+ curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,
+ curated_ids, curated_ids_sorted,
+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
+ all_result_ids, all_result_ids_len,
+ filter_result_iterator, concurrency,
+ sort_order, field_values, geopoint_indices);
+ if (!search_wildcard_op.ok()) {
+ return search_wildcard_op;
+ }
filter_result_iterator->reset();
}
@@ -2499,15 +2526,20 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
}
- fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type, excluded_result_ids,
- excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,
- excluded_group_ids,
- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
- prioritize_token_position, query_hashes, token_order, prefixes,
- typo_tokens_threshold, exhaustive_search,
- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,
- field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type,
+ excluded_result_ids, excluded_result_ids_size,
+ filter_result_iterator, curated_ids_sorted,
+ excluded_group_ids, sort_fields_std, num_typos,
+ searched_queries, qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len,
+ group_limit, group_by_fields, prioritize_exact_match,
+ prioritize_token_position, query_hashes, token_order, prefixes,
+ typo_tokens_threshold, exhaustive_search,
+ max_candidates, min_len_1typo, min_len_2typo,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
// try split/joining tokens if no results are found
if(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {
@@ -2537,28 +2569,38 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
space_resolved_queries[0][j].size(), 0);
}
- fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,
- excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,
- excluded_group_ids,
- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
- prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,
- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,
+ excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,
+ excluded_group_ids,
+ sort_fields_std, num_typos, searched_queries,
+ qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len,
+ group_limit, group_by_fields, prioritize_exact_match,
+ prioritize_token_position, query_hashes, token_order,
+ prefixes, typo_tokens_threshold, exhaustive_search,
+ max_candidates, min_len_1typo, min_len_2typo,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
}
}
// do synonym based searches
- do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map, sort_fields_std,
- curated_topster, token_order,
- 0, group_limit, group_by_fields, prioritize_exact_match, prioritize_token_position,
- exhaustive_search, concurrency, prefixes,
- min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
- topster, q_pos_synonyms, syn_orig_num_tokens,
- groups_processed, searched_queries, all_result_ids, all_result_ids_len,
- filter_result_iterator, query_hashes,
- sort_order, field_values, geopoint_indices,
- qtoken_set);
+ auto do_synonym_search_op = do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map,
+ sort_fields_std, curated_topster, token_order, 0, group_limit,
+ group_by_fields, prioritize_exact_match, prioritize_token_position,
+ exhaustive_search, concurrency, prefixes,
+ min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,
+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,
+ topster, q_pos_synonyms, syn_orig_num_tokens,
+ groups_processed, searched_queries, all_result_ids, all_result_ids_len,
+ filter_result_iterator, query_hashes,
+ sort_order, field_values, geopoint_indices,
+ qtoken_set);
+ if (!do_synonym_search_op.ok()) {
+ return do_synonym_search_op;
+ }
filter_result_iterator->reset();
// gather up both original query and synonym queries and do drop tokens
@@ -2606,16 +2648,21 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
drop_token_prefixes.push_back(p && prefix_search);
}
- fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,
- excluded_result_ids, excluded_result_ids_size,
- filter_result_iterator,
- curated_ids_sorted, excluded_group_ids, sort_fields_std, num_typos, searched_queries,
- qtoken_set, topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields,
- prioritize_exact_match, prioritize_token_position, query_hashes,
- token_order, prefixes, typo_tokens_threshold,
- exhaustive_search, max_candidates, min_len_1typo,
- min_len_2typo, -1, sort_order, field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,
+ excluded_result_ids, excluded_result_ids_size,
+ filter_result_iterator,
+ curated_ids_sorted, excluded_group_ids,
+ sort_fields_std, num_typos, searched_queries,
+ qtoken_set, topster, groups_processed,
+ all_result_ids, all_result_ids_len,
+ group_limit, group_by_fields,
+ prioritize_exact_match, prioritize_token_position, query_hashes,
+ token_order, prefixes, typo_tokens_threshold,
+ exhaustive_search, max_candidates, min_len_1typo,
+ min_len_2typo, -1, sort_order, field_values, geopoint_indices);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
} else {
break;
@@ -2624,14 +2671,18 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
}
- do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std, searched_queries,
- group_limit, group_by_fields,
- max_extra_prefix, max_extra_suffix,
- field_query_tokens[0].q_include_tokens,
- topster, filter_result_iterator,
- sort_order, field_values, geopoint_indices,
- curated_ids_sorted, excluded_group_ids, all_result_ids, all_result_ids_len, groups_processed);
-
+ auto do_infix_search_op = do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std,
+ searched_queries,
+ group_limit, group_by_fields,
+ max_extra_prefix, max_extra_suffix,
+ field_query_tokens[0].q_include_tokens,
+ topster, filter_result_iterator,
+ sort_order, field_values, geopoint_indices,
+ curated_ids_sorted, excluded_group_ids,
+ all_result_ids, all_result_ids_len, groups_processed);
+ if (!do_infix_search_op.ok()) {
+ return do_infix_search_op;
+ }
filter_result_iterator->reset();
if(!vector_query.field_name.empty()) {
@@ -2701,6 +2752,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
for(size_t res_index = 0; res_index < vec_results.size(); res_index++) {
auto& vec_result = vec_results[res_index];
auto doc_id = vec_result.first;
+
+ filter_result_iterator->skip_to(doc_id);
+ auto references = std::move(filter_result_iterator->reference);
+ filter_result_iterator->reset();
auto result_it = topster->kv_map.find(doc_id);
if(result_it != topster->kv_map.end()) {
@@ -2719,8 +2774,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT));
int64_t match_score_index = -1;
int64_t scores[3] = {0};
-
- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);
+
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
+ geopoint_indices, doc_id, references, 0,
+ match_score, scores, match_score_index,
+ vec_result.second);
+ if (!compute_sort_scores_op.ok()) {
+ return compute_sort_scores_op;
+ }
for(int i = 0; i < 3; i++) {
result->scores[i] = scores[i];
@@ -2732,18 +2793,17 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
int64_t scores[3] = {0};
int64_t match_score = float_to_int64_t((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT);
int64_t match_score_index = -1;
- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);
- KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);
- kv.text_match_score = 0;
- kv.vector_distance = vec_result.second;
- if (filter_result_iterator->is_valid &&
- !filter_result_iterator->reference.empty()) {
- // The doc_id must be valid otherwise it would've been filtered out upstream.
- filter_result_iterator->skip_to(doc_id);
- kv.reference_filter_results = std::move(filter_result_iterator->reference);
- filter_result_iterator->reset();
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
+ geopoint_indices, doc_id, references, 0,
+ match_score, scores, match_score_index,
+ vec_result.second);
+ if (!compute_sort_scores_op.ok()) {
+ return compute_sort_scores_op;
}
+ KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores, std::move(references));
+ kv.text_match_score = 0;
+ kv.vector_distance = vec_result.second;
topster->add(&kv);
vec_search_ids.push_back(doc_id);
@@ -3046,36 +3106,36 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>
}
}
-void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
- const std::vector<token_t>& query_tokens,
- const std::vector<token_t>& dropped_tokens,
- const text_match_type_t match_type,
- const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- filter_result_iterator_t* const filter_result_iterator,
- const std::vector<uint32_t>& curated_ids,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const std::vector<sort_by> & sort_fields,
- const std::vector<uint32_t>& num_typos,
- std::vector<std::vector<art_leaf*>> & searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- uint32_t*& all_result_ids, size_t & all_result_ids_len,
- const size_t group_limit, const std::vector<std::string>& group_by_fields,
- bool prioritize_exact_match,
- const bool prioritize_token_position,
- std::set<uint64>& query_hashes,
- const token_ordering token_order,
- const std::vector<bool>& prefixes,
- const size_t typo_tokens_threshold,
- const bool exhaustive_search,
- const size_t max_candidates,
- size_t min_len_1typo,
- size_t min_len_2typo,
- int syn_orig_num_tokens,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const {
+Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
+ const std::vector<token_t>& query_tokens,
+ const std::vector<token_t>& dropped_tokens,
+ const text_match_type_t match_type,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ filter_result_iterator_t* const filter_result_iterator,
+ const std::vector<uint32_t>& curated_ids,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const std::vector<sort_by> & sort_fields,
+ const std::vector<uint32_t>& num_typos,
+ std::vector<std::vector<art_leaf*>> & searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ uint32_t*& all_result_ids, size_t & all_result_ids_len,
+ const size_t group_limit, const std::vector<std::string>& group_by_fields,
+ bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ std::set<uint64>& query_hashes,
+ const token_ordering token_order,
+ const std::vector<bool>& prefixes,
+ const size_t typo_tokens_threshold,
+ const bool exhaustive_search,
+ const size_t max_candidates,
+ size_t min_len_1typo,
+ size_t min_len_2typo,
+ int syn_orig_num_tokens,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const {
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
@@ -3112,7 +3172,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
const long long combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
while(n < N && n < combination_limit) {
- RETURN_CIRCUIT_BREAKER
+ RETURN_CIRCUIT_BREAKER_OP
//LOG(INFO) << "fuzzy_search_fields, n: " << n;
@@ -3290,7 +3350,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
if(token_to_costs[token_index].empty()) {
// we cannot proceed further, as this token is not found within cost limits
// and, dropping of tokens are done elsewhere.
- return ;
+ return Option<bool>(true);
}
}
@@ -3306,16 +3366,20 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
if(token_candidates_vec.size() == query_tokens.size()) {
std::vector<uint32_t> id_buff;
- search_all_candidates(num_search_fields, match_type, the_fields, filter_result_iterator,
- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
- sort_fields, token_candidates_vec, searched_queries, qtoken_set,
- dropped_tokens, topster,
- groups_processed, all_result_ids, all_result_ids_len,
- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
- num_typos, prefixes, prioritize_exact_match, prioritize_token_position,
- exhaustive_search, max_candidates,
- syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
- query_hashes, id_buff);
+ auto search_all_candidates_op = search_all_candidates(num_search_fields, match_type, the_fields,
+ filter_result_iterator,
+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,
+ sort_fields, token_candidates_vec, searched_queries, qtoken_set,
+ dropped_tokens, topster,
+ groups_processed, all_result_ids, all_result_ids_len,
+ typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
+ num_typos, prefixes, prioritize_exact_match, prioritize_token_position,
+ exhaustive_search, max_candidates,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
+ query_hashes, id_buff);
+ if (!search_all_candidates_op.ok()) {
+ return search_all_candidates_op;
+ }
if(id_buff.size() > 1) {
gfx::timsort(id_buff.begin(), id_buff.end());
@@ -3333,11 +3397,13 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
if(!exhaustive_search && all_result_ids_len >= typo_tokens_threshold) {
// if typo threshold is breached, we are done
- return ;
+ return Option<bool>(true);
}
n++;
}
+
+ return Option<bool>(true);
}
void Index::popular_fields_of_token(const spp::sparse_hash_map<std::string, art_tree*>& search_index,
@@ -3454,31 +3520,31 @@ void Index::find_across_fields(const token_t& previous_token,
}
}
-void Index::search_across_fields(const std::vector<token_t>& query_tokens,
- const std::vector<uint32_t>& num_typos,
- const std::vector<bool>& prefixes,
- const std::vector<search_field_t>& the_fields,
- const size_t num_search_fields,
- const text_match_type_t match_type,
- const std::vector<sort_by>& sort_fields,
- Topster* topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- tsl::htrie_map<char, token_leaf>& qtoken_set,
- const std::vector<token_t>& dropped_tokens,
- const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- const bool prioritize_exact_match,
- const bool prioritize_token_position,
- filter_result_iterator_t* const filter_result_iterator,
- const uint32_t total_cost, const int syn_orig_num_tokens,
- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- std::vector<uint32_t>& id_buff,
- uint32_t*& all_result_ids, size_t& all_result_ids_len) const {
+Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokens,
+ const std::vector<uint32_t>& num_typos,
+ const std::vector<bool>& prefixes,
+ const std::vector<search_field_t>& the_fields,
+ const size_t num_search_fields,
+ const text_match_type_t match_type,
+ const std::vector<sort_by>& sort_fields,
+ Topster* topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ tsl::htrie_map<char, token_leaf>& qtoken_set,
+ const std::vector<token_t>& dropped_tokens,
+ const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ const bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ filter_result_iterator_t* const filter_result_iterator,
+ const uint32_t total_cost, const int syn_orig_num_tokens,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ std::vector<uint32_t>& id_buff,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len) const {
std::vector<art_leaf*> query_suggestion;
@@ -3596,10 +3662,12 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
std::vector<uint32_t> result_ids;
size_t filter_index = 0;
+ Option<bool> status(true);
or_iterator_t::intersect(token_its, istate,
[&](single_filter_result_t& filter_result, const std::vector<or_iterator_t>& its) {
auto& seq_id = filter_result.seq_id;
+ auto references = std::move(filter_result.reference_filter_results);
//LOG(INFO) << "seq_id: " << seq_id;
// Convert [token -> fields] orientation to [field -> tokens] orientation
std::vector<std::vector<posting_list_t::iterator_t>> field_to_tokens(num_search_fields);
@@ -3667,8 +3735,13 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
int64_t scores[3] = {0};
int64_t match_score_index = -1;
- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,
- best_field_match_score, scores, match_score_index);
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
+ seq_id, references, filter_index, best_field_match_score,
+ scores, match_score_index);
+ if (!compute_sort_scores_op.ok()) {
+ status = Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());
+ return;
+ }
size_t query_len = query_tokens.size();
@@ -3719,8 +3792,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
<< ", num_matching_fields: " << num_matching_fields
<< ", aggregated_score: " << aggregated_score;*/
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- kv.reference_filter_results = std::move(filter_result.reference_filter_results);
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
if(match_score_index != -1) {
kv.scores[match_score_index] = aggregated_score;
@@ -3733,6 +3805,15 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
}
result_ids.push_back(seq_id);
});
+ if (!status.ok()) {
+ for(posting_list_t* plist: expanded_plists) {
+ delete plist;
+ }
+ for(posting_list_t* plist: expanded_dropped_plists) {
+ delete plist;
+ }
+ return status;
+ }
id_buff.insert(id_buff.end(), result_ids.begin(), result_ids.end());
@@ -3763,13 +3844,16 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
for(posting_list_t* plist: expanded_dropped_plists) {
delete plist;
}
+
+ return Option<bool>(true);
}
-void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- uint32_t seq_id, size_t filter_index, int64_t max_field_match_score,
- int64_t* scores, int64_t& match_score_index, float vector_distance) const {
+Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ uint32_t seq_id, const std::map<basic_string<char>, reference_filter_result_t>& references,
+ size_t filter_index, int64_t max_field_match_score, int64_t* scores,
+ int64_t& match_score_index, float vector_distance) const {
int64_t geopoint_distances[3];
@@ -3827,6 +3911,24 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
// avoiding loop
if (sort_fields.size() > 0) {
+ // In case of reference sort_by, we need to get the sort score of the reference doc id.
+ if (!sort_fields[0].reference_collection_name.empty()) {
+ auto& sort_field = sort_fields[0];
+ if (references.count(sort_field.reference_collection_name) == 0 ||
+ references.at(sort_field.reference_collection_name).count == 0) {
+ return Option<bool>(400, "No references found to sort by on `" +
+ sort_field.reference_collection_name + "." + sort_field.name + "`.");
+ }
+
+ auto& reference_result = references.at(sort_field.reference_collection_name);
+ if (reference_result.count > 1) {
+ return Option<bool>(400, "Multiple references found to sort by on `" +
+ sort_field.reference_collection_name + "." + sort_field.name + "`.");
+ }
+
+ seq_id = reference_result.docs[0];
+ }
+
if (field_values[0] == &text_match_sentinel_value) {
scores[0] = int64_t(max_field_match_score);
match_score_index = 0;
@@ -3835,7 +3937,19 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
} else if(field_values[0] == &geo_sentinel_value) {
scores[0] = geopoint_distances[0];
} else if(field_values[0] == &str_sentinel_value) {
- scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);
+ if (sort_fields[0].reference_collection_name.empty()) {
+ scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);
+ } else {
+ auto& cm = CollectionManager::get_instance();
+ auto ref_collection = cm.get_collection(sort_fields[0].reference_collection_name);
+ if (ref_collection == nullptr) {
+ return Option<bool>(400, "Referenced collection `" + sort_fields[0].reference_collection_name +
+ "` not found.");
+ }
+
+ scores[0] = ref_collection->reference_string_sort_score(sort_fields[0].name, seq_id);
+ }
+
if(scores[0] == adi_tree_t::NOT_FOUND) {
if(sort_fields[0].order == sort_field_const::asc &&
sort_fields[0].missing_values == sort_by::missing_values_t::first) {
@@ -3885,6 +3999,24 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
}
if(sort_fields.size() > 1) {
+ // In case of reference sort_by, we need to get the sort score of the reference doc id.
+ if (!sort_fields[1].reference_collection_name.empty()) {
+ auto& sort_field = sort_fields[1];
+ if (references.count(sort_field.reference_collection_name) == 0 ||
+ references.at(sort_field.reference_collection_name).count == 0) {
+ return Option<bool>(400, "No references found to sort by on `" +
+ sort_field.reference_collection_name + "." + sort_field.name + "`.");
+ }
+
+ auto& reference_result = references.at(sort_field.reference_collection_name);
+ if (reference_result.count > 1) {
+ return Option<bool>(400, "Multiple references found to sort by on `" +
+ sort_field.reference_collection_name + "." + sort_field.name + "`.");
+ }
+
+ seq_id = reference_result.docs[0];
+ }
+
if (field_values[1] == &text_match_sentinel_value) {
scores[1] = int64_t(max_field_match_score);
match_score_index = 1;
@@ -3939,6 +4071,24 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
}
if(sort_fields.size() > 2) {
+ // In case of reference sort_by, we need to get the sort score of the reference doc id.
+ if (!sort_fields[2].reference_collection_name.empty()) {
+ auto& sort_field = sort_fields[2];
+ if (references.count(sort_field.reference_collection_name) == 0 ||
+ references.at(sort_field.reference_collection_name).count == 0) {
+ return Option<bool>(400, "No references found to sort by on `" +
+ sort_field.reference_collection_name + "." + sort_field.name + "`.");
+ }
+
+ auto& reference_result = references.at(sort_field.reference_collection_name);
+ if (reference_result.count > 1) {
+ return Option<bool>(400, "Multiple references found to sort by on `" +
+ sort_field.reference_collection_name + "." + sort_field.name + "`.");
+ }
+
+ seq_id = reference_result.docs[0];
+ }
+
if (field_values[2] == &text_match_sentinel_value) {
scores[2] = int64_t(max_field_match_score);
match_score_index = 2;
@@ -3991,27 +4141,29 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
scores[2] = -scores[2];
}
}
+
+ return Option<bool>(true);
}
-void Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
- std::vector<query_tokens_t>& field_query_tokens,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields,
- Topster* actual_topster,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- filter_result_iterator_t*& filter_result_iterator,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- const std::set<uint32_t>& curated_ids,
- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* curated_topster,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- bool is_wildcard_query) const {
+Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,
+ std::vector<query_tokens_t>& field_query_tokens,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields,
+ Topster* actual_topster,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ filter_result_iterator_t*& filter_result_iterator,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ const std::set<uint32_t>& curated_ids,
+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* curated_topster,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ bool is_wildcard_query) const {
uint32_t* phrase_result_ids = nullptr;
uint32_t phrase_result_count = 0;
@@ -4114,7 +4266,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
if (!is_wildcard_query) {
// this means that the there are non-phrase tokens in the query
// so we cannot directly copy to the all_result_ids array
- return;
+ return Option<bool>(true);
}
all_result_ids_len = filter_result_iterator->to_filter_id_array(all_result_ids);
@@ -4124,15 +4276,19 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
// populate topster
for(size_t i = 0; i < std::min<size_t>(10000, all_result_ids_len); i++) {
auto seq_id = filter_result_iterator->seq_id;
- auto reference_filter_results = std::move(filter_result_iterator->reference);
+ auto references = std::move(filter_result_iterator->reference);
filter_result_iterator->next();
int64_t match_score = phrase_match_id_scores[seq_id];
int64_t scores[3] = {0};
int64_t match_score_index = -1;
- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,
- match_score, scores, match_score_index);
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
+ seq_id, references, filter_index, match_score, scores,
+ match_score_index);
+ if (!compute_sort_scores_op.ok()) {
+ return compute_sort_scores_op;
+ }
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
@@ -4142,8 +4298,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
}
}
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- kv.reference_filter_results = std::move(reference_filter_results);
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
int ret = actual_topster->add(&kv);
if(group_limit != 0 && ret < 2) {
@@ -4157,66 +4312,73 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s
filter_result_iterator->reset();
searched_queries.push_back({});
+ return Option<bool>(true);
}
-void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
- const text_match_type_t match_type,
- filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
- const token_ordering& token_order,
- const size_t typo_tokens_threshold, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
- const bool prioritize_token_position,
- const bool exhaustive_search, const size_t concurrency,
- const std::vector<bool>& prefixes,
- size_t min_len_1typo,
- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- Topster* actual_topster,
- std::vector<std::vector<token_t>>& q_pos_synonyms,
- int syn_orig_num_tokens,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- filter_result_iterator_t* const filter_result_iterator,
- std::set<uint64>& query_hashes,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices,
- tsl::htrie_map<char, token_leaf>& qtoken_set) const {
+Option<bool> Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
+ const text_match_type_t match_type,
+ filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,
+ const token_ordering& token_order,
+ const size_t typo_tokens_threshold, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
+ const bool prioritize_token_position,
+ const bool exhaustive_search, const size_t concurrency,
+ const std::vector<bool>& prefixes,
+ size_t min_len_1typo,
+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ Topster* actual_topster,
+ std::vector<std::vector<token_t>>& q_pos_synonyms,
+ int syn_orig_num_tokens,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ std::set<uint64>& query_hashes,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices,
+ tsl::htrie_map<char, token_leaf>& qtoken_set) const {
for (const auto& syn_tokens : q_pos_synonyms) {
query_hashes.clear();
- fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,
- exclude_token_ids_size, filter_result_iterator, curated_ids_sorted, excluded_group_ids,
- sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,
- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,
- prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold,
- exhaustive_search, max_candidates, min_len_1typo,
- min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,
+ exclude_token_ids_size, filter_result_iterator,
+ curated_ids_sorted, excluded_group_ids, sort_fields_std, {0},
+ searched_queries, qtoken_set, actual_topster, groups_processed,
+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,
+ prioritize_exact_match, prioritize_token_position, query_hashes,
+ token_order, prefixes, typo_tokens_threshold, exhaustive_search,
+ max_candidates, min_len_1typo, min_len_2typo,
+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices);
+ if (!fuzzy_search_fields_op.ok()) {
+ return fuzzy_search_fields_op;
+ }
}
collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
+ return Option<bool>(true);
}
-void Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
- const std::vector<enable_t>& infixes,
- const std::vector<sort_by>& sort_fields,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
- const size_t max_extra_suffix,
- const std::vector<token_t>& query_tokens, Topster* actual_topster,
- filter_result_iterator_t* const filter_result_iterator,
- const int sort_order[3],
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
- const std::vector<size_t>& geopoint_indices,
- const std::vector<uint32_t>& curated_ids_sorted,
- const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {
+Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
+ const std::vector<enable_t>& infixes,
+ const std::vector<sort_by>& sort_fields,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
+ const size_t max_extra_suffix,
+ const std::vector<token_t>& query_tokens, Topster* actual_topster,
+ filter_result_iterator_t* const filter_result_iterator,
+ const int sort_order[3],
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
+ const std::vector<size_t>& geopoint_indices,
+ const std::vector<uint32_t>& curated_ids_sorted,
+ const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
auto& field_name = the_fields[field_id].name;
@@ -4259,6 +4421,10 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
for(size_t i = 0; i < raw_infix_ids_length; i++) {
auto seq_id = raw_infix_ids[i];
+ std::map<basic_string<char>, reference_filter_result_t> references;
+ for (const auto& item: filtered_infix_ids.reference_filter_results) {
+ references[item.first] = item.second[i];
+ }
int64_t match_score = 0;
score_results2(sort_fields, searched_queries.size(), field_id, field_is_array,
@@ -4267,8 +4433,12 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
int64_t scores[3] = {0};
int64_t match_score_index = -1;
- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,
- 100, scores, match_score_index);
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values,
+ geopoint_indices, seq_id, references,
+ filter_index, 100, scores, match_score_index);
+ if (!compute_sort_scores_op.ok()) {
+ return compute_sort_scores_op;
+ }
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
@@ -4278,10 +4448,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
}
}
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- for (const auto& item: filtered_infix_ids.reference_filter_results) {
- kv.reference_filter_results[item.first] = item.second[i];
- }
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
int ret = actual_topster->add(&kv);
if(group_limit != 0 && ret < 2) {
@@ -4308,6 +4475,8 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
}
}
}
+
+ return Option<bool>(true);
}
void Index::handle_exclusion(const size_t num_search_fields, std::vector<query_tokens_t>& field_query_tokens,
@@ -4561,20 +4730,20 @@ void Index::curate_filtered_ids(const std::set<uint32_t>& curated_ids,
}
}
-void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
- uint32_t*& all_result_ids, size_t& all_result_ids_len,
- filter_result_iterator_t* const filter_result_iterator,
- const size_t concurrency,
- const int* sort_order,
- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
- const std::vector<size_t>& geopoint_indices) const {
+Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root,
+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,
+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,
+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,
+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
+ const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,
+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,
+ uint32_t*& all_result_ids, size_t& all_result_ids_len,
+ filter_result_iterator_t* const filter_result_iterator,
+ const size_t concurrency,
+ const int* sort_order,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
+ const std::vector<size_t>& geopoint_indices) const {
filter_result_iterator->compute_result();
auto const& approx_filter_ids_length = filter_result_iterator->approx_filter_ids_length;
@@ -4602,6 +4771,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
const auto parent_search_stop_ms = search_stop_us;
auto parent_search_cutoff = search_cutoff;
uint32_t excluded_result_index = 0;
+ Option<bool>* compute_sort_score_statuses[num_threads];
for(size_t thread_id = 0; thread_id < num_threads && filter_result_iterator->is_valid; thread_id++) {
filter_result_t batch_result;
@@ -4613,14 +4783,15 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
searched_queries.push_back({});
topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);
+ auto& compute_sort_score_status = compute_sort_score_statuses[thread_id] = nullptr;
thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,
- thread_id, &sort_fields, &searched_queries,
- &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,
- &sort_order, field_values, &geopoint_indices, &plists,
- check_for_circuit_break,
- batch_result,
- &num_processed, &m_process, &cv_process]() {
+ thread_id, &sort_fields, &searched_queries,
+ &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,
+ &sort_order, field_values, &geopoint_indices, &plists,
+ check_for_circuit_break,
+ batch_result,
+ &num_processed, &m_process, &cv_process, &compute_sort_score_status]() {
search_begin_us = parent_search_begin;
search_stop_us = parent_search_stop_ms;
@@ -4630,6 +4801,11 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
for(size_t i = 0; i < batch_result.count; i++) {
const uint32_t seq_id = batch_result.docs[i];
+ std::map<basic_string<char>, reference_filter_result_t> references;
+ for (const auto& item: batch_result.reference_filter_results) {
+ references[item.first] = item.second[i];
+ }
+
int64_t match_score = 0;
score_results2(sort_fields, (uint16_t) searched_queries.size(), 0, false, 0,
@@ -4638,8 +4814,13 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
int64_t scores[3] = {0};
int64_t match_score_index = -1;
- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,
- 100, scores, match_score_index);
+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
+ seq_id, references, filter_index, 100, scores,
+ match_score_index);
+ if (!compute_sort_scores_op.ok()) {
+ compute_sort_score_status = new Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());
+ break;
+ }
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
@@ -4649,10 +4830,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
}
}
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- for (const auto& item: batch_result.reference_filter_results) {
- kv.reference_filter_results[item.first] = item.second[i];
- }
+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));
int ret = topsters[thread_id]->add(&kv);
if(group_limit != 0 && ret < 2) {
@@ -4677,6 +4855,19 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
search_cutoff = parent_search_cutoff;
for(size_t thread_id = 0; thread_id < num_processed; thread_id++) {
+ if (compute_sort_score_statuses[thread_id] != nullptr) {
+ auto& status = compute_sort_score_statuses[thread_id];
+ auto return_value = Option<bool>(status->code(), status->error());
+
+ // Cleanup the remaining threads.
+ for (size_t i = thread_id; i < num_processed; i++) {
+ delete compute_sort_score_statuses[i];
+ delete topsters[i];
+ }
+
+ return return_value;
+ }
+
//groups_processed.insert(tgroups_processed[thread_id].begin(), tgroups_processed[thread_id].end());
for(const auto& it : tgroups_processed[thread_id]) {
groups_processed[it.first]+= it.second;
@@ -4691,12 +4882,38 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
filter_result_iterator->reset();
all_result_ids_len = filter_result_iterator->to_filter_id_array(all_result_ids);
+
+ return Option<bool>(true);
}
void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,
std::vector<sort_by>& sort_fields_std,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const {
for (size_t i = 0; i < sort_fields_std.size(); i++) {
+ if (!sort_fields_std[i].reference_collection_name.empty()) {
+ auto& cm = CollectionManager::get_instance();
+ auto ref_collection = cm.get_collection(sort_fields_std[i].reference_collection_name);
+
+ int ref_sort_order[1];
+ std::vector<size_t> ref_geopoint_indices;
+ std::vector<sort_by> ref_sort_fields_std;
+ ref_sort_fields_std.emplace_back(sort_fields_std[i]);
+ ref_sort_fields_std.front().reference_collection_name.clear();
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> ref_field_values;
+ ref_collection->reference_populate_sort_mapping(ref_sort_order, ref_geopoint_indices,
+ ref_sort_fields_std, ref_field_values);
+
+ sort_order[i] = ref_sort_order[0];
+ if (!ref_geopoint_indices.empty()) {
+ geopoint_indices.push_back(i);
+ }
+ sort_fields_std[i] = ref_sort_fields_std[0];
+ sort_fields_std[i].reference_collection_name = ref_collection->get_name();
+ field_values[i] = ref_field_values[0];
+
+ continue;
+ }
+
sort_order[i] = 1;
if (sort_fields_std[i].order == sort_field_const::asc) {
sort_order[i] = -1;
@@ -4734,6 +4951,13 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint
}
}
+void Index::populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>& geopoint_indices,
+ std::vector<sort_by>& sort_fields_std,
+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const {
+ std::shared_lock lock(mutex);
+ populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);
+}
+
void Index::search_field(const uint8_t & field_id,
const std::vector<token_t>& query_tokens,
const uint32_t* exclude_token_ids,
@@ -6168,6 +6392,11 @@ Option<uint32_t> Index::get_reference_doc_id_with_lock(const string& reference_h
return Option<uint32_t>(sort_index.at(reference_helper_field_name)->at(seq_id));
}
+int64_t Index::reference_string_sort_score(const string &field_name, const uint32_t &seq_id) const {
+ std::shared_lock lock(mutex);
+ return str_sort_index.at(field_name)->rank(seq_id);
+}
+
/*
// https://stackoverflow.com/questions/924171/geo-fencing-point-inside-outside-polygon
// NOTE: polygon and point should have been transformed with `transform_for_180th_meridian`
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 13d8b2a6..7ae07ce7 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -1364,6 +1364,36 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ nlohmann::json model_config = R"({
+ "model_name": "ts/e5-small"
+ })"_json;
+ auto query_embedding = TextEmbedderManager::get_instance().get_text_embedder(model_config).get()->Embed("natural products");
+ std::string vec_string = "[";
+ for (auto const& i : query_embedding.embedding) {
+ vec_string += std::to_string(i);
+ vec_string += ",";
+ }
+ vec_string[vec_string.size() - 1] = ']';
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},
+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
+ {"include_fields", "product_name, $Customers(product_price)"},
+ {"exclude_fields", ""}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+ ASSERT_EQ(1, res_obj["hits"].size());
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+
// Hybrid search - Both text match and vector match
req_params = {
{"collection", "Products"},
@@ -1383,7 +1413,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
- ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("rank_fusion_score"));
+ ASSERT_NE(0, res_obj["hits"][0].at("text_match"));
+ ASSERT_NE(0, res_obj["hits"][0].at("vector_distance"));
// Hybrid search - Only vector match
req_params = {
@@ -1404,7 +1435,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
- ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("rank_fusion_score"));
+ ASSERT_EQ(0, res_obj["hits"][0].at("text_match"));
+ ASSERT_NE(0, res_obj["hits"][0].at("vector_distance"));
// Infix search
req_params = {
@@ -1631,3 +1663,510 @@ TEST_F(CollectionJoinTest, CascadeDeletion) {
ASSERT_EQ("product_b", res_obj["hits"][0]["document"].at("product_id"));
ASSERT_EQ("user_a", res_obj["hits"][0]["document"].at("user_id"));
}
+
+TEST_F(CollectionJoinTest, SortByReference) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_id", "type": "string"},
+ {"name": "product_name", "type": "string", "sort": true, "infix": true},
+ {"name": "product_description", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "product_id": "product_a",
+ "product_name": "shampoo",
+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."
+ })"_json,
+ R"({
+ "product_id": "product_b",
+ "product_name": "soap",
+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."
+ })"_json
+ };
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Customers",
+ "fields": [
+ {"name": "customer_id", "type": "string"},
+ {"name": "customer_name", "type": "string"},
+ {"name": "product_price", "type": "float"},
+ {"name": "product_available", "type": "bool"},
+ {"name": "product_location", "type": "geopoint"},
+ {"name": "product_id", "type": "string", "reference": "Products.product_id", "sort": true}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "customer_id": "customer_a",
+ "customer_name": "Joe",
+ "product_price": 143,
+ "product_available": true,
+ "product_location": [48.872576479306765, 2.332291112241466],
+ "product_id": "product_a"
+ })"_json,
+ R"({
+ "customer_id": "customer_a",
+ "customer_name": "Joe",
+ "product_price": 73.5,
+ "product_available": false,
+ "product_location": [48.888286721920934, 2.342340862419206],
+ "product_id": "product_b"
+ })"_json,
+ R"({
+ "customer_id": "customer_b",
+ "customer_name": "Dan",
+ "product_price": 75,
+ "product_available": true,
+ "product_location": [48.872576479306765, 2.332291112241466],
+ "product_id": "product_a"
+ })"_json,
+ R"({
+ "customer_id": "customer_b",
+ "customer_name": "Dan",
+ "product_price": 140,
+ "product_available": false,
+ "product_location": [48.888286721920934, 2.342340862419206],
+ "product_id": "product_b"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ std::map<std::string, std::string> req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$foo(product_price:asc"}
+ };
+ nlohmann::json embedded_params;
+ std::string json_res;
+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Parameter `sort_by` is malformed.", search_op.error());
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(product_price)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Reference `sort_by` is malformed.", search_op.error());
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$foo(product_price:asc)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Referenced collection `foo` in `sort_by` not found.", search_op.error());
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(foo:asc)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Referenced collection `Customers`: Could not find a field named `foo` in the schema for sorting.",
+ search_op.error());
+
+ // Sort by reference numeric field
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(product_price:asc)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ auto res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_b", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_a", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][1]["document"].at("product_price"));
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+
+ // Sort by reference string field
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(product_id:asc)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+
+ // Sort by reference optional filtering.
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(_eval(product_available:true):asc)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_b", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_a", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][1]["document"].at("product_price"));
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(_eval(product_available:true):desc)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+
+ // Text search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "s"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+
+ // Phrase search
+ req_params = {
+ {"collection", "Products"},
+ {"q", R"("our")"},
+ {"query_by", "product_description"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+
+ // Vector search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "natural products"},
+ {"query_by", "embedding"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+ auto product_a_score = res_obj["hits"][0].at("vector_distance");
+ auto product_b_score = res_obj["hits"][1].at("vector_distance");
+ // product_b is a better match for the vector query but sort_by overrides the order.
+ ASSERT_TRUE(product_b_score < product_a_score);
+
+ nlohmann::json model_config = R"({
+ "model_name": "ts/e5-small"
+ })"_json;
+ auto query_embedding = TextEmbedderManager::get_instance().get_text_embedder(model_config).get()->Embed("natural products");
+ std::string vec_string = "[";
+ for (auto const& i : query_embedding.embedding) {
+ vec_string += std::to_string(i);
+ vec_string += ",";
+ }
+ vec_string[vec_string.size() - 1] = ']';
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+ product_a_score = res_obj["hits"][0].at("vector_distance");
+ product_b_score = res_obj["hits"][1].at("vector_distance");
+ // product_b is a better match for the vector query but sort_by overrides the order.
+ ASSERT_TRUE(product_b_score < product_a_score);
+
+ // Hybrid search - Both text match and vector match
+ req_params = {
+ {"collection", "Products"},
+ {"q", "soap"},
+ {"query_by", "product_name, embedding"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+ product_a_score = res_obj["hits"][0].at("text_match");
+ product_b_score = res_obj["hits"][1].at("text_match");
+ ASSERT_TRUE(product_b_score > product_a_score);
+ product_a_score = res_obj["hits"][0].at("vector_distance");
+ product_b_score = res_obj["hits"][1].at("vector_distance");
+ ASSERT_TRUE(product_b_score < product_a_score);
+
+ // Hybrid search - Only vector match
+ req_params = {
+ {"collection", "Products"},
+ {"q", "natural products"},
+ {"query_by", "product_name, embedding"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+ product_a_score = res_obj["hits"][0].at("vector_distance");
+ product_b_score = res_obj["hits"][1].at("vector_distance");
+ // product_b is a better match for the vector query but sort_by overrides the order.
+ ASSERT_TRUE(product_b_score < product_a_score);
+
+ // Infix search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "p"},
+ {"query_by", "product_name"},
+ {"infix", "always"},
+ {"filter_by", "$Customers(customer_id:=customer_a)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+ ASSERT_EQ(2, res_obj["hits"].size());
+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));
+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));
+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));
+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));
+
+ // Multiple references - Wildcard search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(product_price: >0)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+
+ // Multiple references - Text search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "s"},
+ {"query_by", "product_name"},
+ {"filter_by", "$Customers(product_price: >0)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ {"include_fields", "product_id, $Customers(product_price)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+
+ // Multiple references - Phrase search
+ req_params = {
+ {"collection", "Products"},
+ {"q", R"("our")"},
+ {"query_by", "product_description"},
+ {"filter_by", "$Customers(product_price: >0)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+
+ // Multiple references - Vector search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "natural products"},
+ {"query_by", "embedding"},
+ {"filter_by", "$Customers(product_price:>0)"},
+ {"include_fields", "product_name, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "*"},
+ {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},
+ {"filter_by", "$Customers(product_price: >0)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+
+ // Multiple references - Hybrid search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "soap"},
+ {"query_by", "product_name, embedding"},
+ {"filter_by", "$Customers(product_price: >0)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+
+ req_params = {
+ {"collection", "Products"},
+ {"q", "natural products"},
+ {"query_by", "product_name, embedding"},
+ {"filter_by", "$Customers(product_price: >0)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+
+ // Multiple references - Infix search
+ req_params = {
+ {"collection", "Products"},
+ {"q", "p"},
+ {"query_by", "product_name"},
+ {"infix", "always"},
+ {"filter_by", "$Customers(product_price: >0)"},
+ {"include_fields", "product_id, $Customers(product_price)"},
+ {"sort_by", "$Customers(product_price:desc)"},
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());
+}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 3c18db44..5f63ba39 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -1110,6 +1110,23 @@ TEST_F(CollectionManagerTest, ParseSortByClause) {
ASSERT_EQ("points", sort_fields[1].name);
ASSERT_EQ("DESC", sort_fields[1].order);
+ sort_fields.clear();
+ sort_by_parsed = CollectionManager::parse_sort_by_str("points:desc,loc(24.56,10.45):ASC,"
+ "$Customers(product_price:DESC)", sort_fields);
+ ASSERT_TRUE(sort_by_parsed);
+ ASSERT_EQ(3, sort_fields.size());
+ ASSERT_EQ("points", sort_fields[0].name);
+ ASSERT_EQ("DESC", sort_fields[0].order);
+ ASSERT_EQ("loc(24.56,10.45)", sort_fields[1].name);
+ ASSERT_EQ("ASC", sort_fields[1].order);
+ ASSERT_EQ("$Customers(product_price:DESC)", sort_fields[2].name);
+
+ sort_fields.clear();
+ sort_by_parsed = CollectionManager::parse_sort_by_str("$foo( _eval(brand:nike && foo:bar):DESC,points:desc) ",
+ sort_fields);
+ ASSERT_TRUE(sort_by_parsed);
+ ASSERT_EQ("$foo( _eval(brand:nike && foo:bar):DESC,points:desc)", sort_fields[0].name);
+
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("", sort_fields);
ASSERT_TRUE(sort_by_parsed);