Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Sort by reference field.
Harpreet Sangarf41afc155fb0
diff --git a/include/collection.h b/include/collection.hindex 88b8d750..205737d7 100644--- a/include/collection.h+++ b/include/collection.h@@ -214,6 +214,11 @@ private:bool is_wildcard_query,const bool is_vector_query,bool is_group_by_query = false) const;+ Option<bool> validate_and_standardize_sort_fields_with_lock(const std::vector<sort_by> & sort_fields,+ std::vector<sort_by>& sort_fields_std,+ bool is_wildcard_query,const bool is_vector_query,+ bool is_group_by_query = false) const;+Option<bool> persist_collection_meta();@@ -569,6 +574,12 @@ public:std::vector<std::string>& reordered_search_fields) const;Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);++ void reference_populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,+ std::vector<sort_by>& sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;++ int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;};template<class T>diff --git a/include/field.h b/include/field.hindex a051c7ee..44920158 100644--- a/include/field.h+++ b/include/field.h@@ -513,6 +513,8 @@ struct sort_by {missing_values_t missing_values;eval_t eval;+ std::string reference_collection_name;+sort_by(const std::string & name, const std::string & order):name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),missing_values(normal) {@@ -527,6 +529,20 @@ struct sort_by {}+ sort_by(const sort_by& other) {+ if (&other == this)+ return;+ name = other.name;+ order = other.order;+ text_match_buckets = other.text_match_buckets;+ geopoint = other.geopoint;+ exclude_radius = other.exclude_radius;+ geo_precision = other.geo_precision;+ missing_values = other.missing_values;+ eval = other.eval;+ reference_collection_name = other.reference_collection_name;+ }+sort_by& operator=(const sort_by& other) {name = other.name;order = other.order;@@ -536,6 +552,7 @@ struct sort_by {geo_precision = other.geo_precision;missing_values = other.missing_values;eval = other.eval;+ reference_collection_name = other.reference_collection_name;return *this;}};diff --git a/include/index.h b/include/index.hindex bc961ec2..b4b6da0c 100644--- a/include/index.h+++ b/include/index.h@@ -424,36 +424,36 @@ private:size_t min_len_2typo,size_t max_candidates) const;- void search_all_candidates(const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<search_field_t>& the_fields,- filter_result_iterator_t* const filter_result_iterator,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by>& sort_fields,- std::vector<tok_candidates>& token_candidates_vec,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const size_t typo_tokens_threshold,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool exhaustive_search,- const size_t max_candidates,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::set<uint64>& query_hashes,- std::vector<uint32_t>& id_buff) const;+ Option<bool> search_all_candidates(const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<search_field_t>& the_fields,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by>& sort_fields,+ std::vector<tok_candidates>& token_candidates_vec,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const size_t typo_tokens_threshold,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ bool prioritize_exact_match,+ const bool prioritize_token_position,+ const bool exhaustive_search,+ const size_t max_candidates,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::set<uint64>& query_hashes,+ std::vector<uint32_t>& id_buff) const;void search_candidates(const uint8_t & field_id,bool field_is_array,@@ -696,20 +696,20 @@ public:// the following methods are not synchronized because their parent calls are synchronized or they are const/static- void search_wildcard(filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- filter_result_iterator_t* const filter_result_iterator,- const size_t concurrency,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const;+ Option<bool> search_wildcard(filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ const size_t concurrency,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices) const;void search_infix(const std::string& query, const std::string& field_name, std::vector<uint32_t>& ids,size_t max_extra_prefix, size_t max_extra_suffix) const;@@ -722,6 +722,12 @@ public:std::vector<sort_by>& sort_fields_std,std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;+ void populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>& geopoint_indices,+ std::vector<sort_by>& sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;++ int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;+static void remove_matched_tokens(std::vector<std::string>& tokens, const std::set<std::string>& rule_token_set) ;void compute_facet_infos(const std::vector<facet>& facets, facet_query_t& facet_query,@@ -741,100 +747,100 @@ public:const std::vector<search_field_t>& search_fields, uint32_t*& exclude_token_ids,size_t& exclude_token_ids_size) const;- void do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,- const std::vector<enable_t>& infixes,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,- const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,- filter_result_iterator_t* const filter_result_iterator,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;-- void do_synonym_search(const std::vector<search_field_t>& the_fields,- const text_match_type_t match_type,- filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,- const token_ordering& token_order,- const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,- const bool prioritize_token_position,- const bool exhaustive_search, const size_t concurrency,- const std::vector<bool>& prefixes,- size_t min_len_1typo,- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* actual_topster,- std::vector<std::vector<token_t>>& q_pos_synonyms,- int syn_orig_num_tokens,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- filter_result_iterator_t* const filter_result_iterator,- std::set<uint64>& query_hashes,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- tsl::htrie_map<char, token_leaf>& qtoken_set) const;-- void do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,- std::vector<query_tokens_t>& field_query_tokens,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- Topster* actual_topster,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- filter_result_iterator_t*& filter_result_iterator,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- const std::set<uint32_t>& curated_ids,- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* curated_topster,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- bool is_wildcard_query) const;-- void fuzzy_search_fields(const std::vector<search_field_t>& the_fields,- const std::vector<token_t>& query_tokens,- const std::vector<token_t>& dropped_tokens,- const text_match_type_t match_type,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- filter_result_iterator_t* const filter_result_iterator,- const std::vector<uint32_t>& curated_ids,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by>& sort_fields,- const std::vector<uint32_t>& num_typos,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const size_t group_limit, const std::vector<std::string>& group_by_fields,- bool prioritize_exact_match,- const bool prioritize_token_position,- std::set<uint64>& query_hashes,- const token_ordering token_order,- const std::vector<bool>& prefixes,- const size_t typo_tokens_threshold,- const bool exhaustive_search,- const size_t max_candidates,- size_t min_len_1typo,- size_t min_len_2typo,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const;+ Option<bool> do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,+ const std::vector<enable_t>& infixes,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,+ const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,+ filter_result_iterator_t* const filter_result_iterator,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const;++ [[nodiscard]] Option<bool> do_synonym_search(const std::vector<search_field_t>& the_fields,+ const text_match_type_t match_type,+ filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,+ const token_ordering& token_order,+ const size_t typo_tokens_threshold, const size_t group_limit,+ const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const bool prioritize_token_position,+ const bool exhaustive_search, const size_t concurrency,+ const std::vector<bool>& prefixes,+ size_t min_len_1typo,+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* actual_topster,+ std::vector<std::vector<token_t>>& q_pos_synonyms,+ int syn_orig_num_tokens,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ std::set<uint64>& query_hashes,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ tsl::htrie_map<char, token_leaf>& qtoken_set) const;++ Option<bool> do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,+ std::vector<query_tokens_t>& field_query_tokens,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ Topster* actual_topster,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ filter_result_iterator_t*& filter_result_iterator,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ const std::set<uint32_t>& curated_ids,+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* curated_topster,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ bool is_wildcard_query) const;++ [[nodiscard]] Option<bool> fuzzy_search_fields(const std::vector<search_field_t>& the_fields,+ const std::vector<token_t>& query_tokens,+ const std::vector<token_t>& dropped_tokens,+ const text_match_type_t match_type,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ filter_result_iterator_t* const filter_result_iterator,+ const std::vector<uint32_t>& curated_ids,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by>& sort_fields,+ const std::vector<uint32_t>& num_typos,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,+ bool prioritize_exact_match,+ const bool prioritize_token_position,+ std::set<uint64>& query_hashes,+ const token_ordering token_order,+ const std::vector<bool>& prefixes,+ const size_t typo_tokens_threshold,+ const bool exhaustive_search,+ const size_t max_candidates,+ size_t min_len_1typo,+ size_t min_len_2typo,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices) const;void find_across_fields(const token_t& previous_token,const std::string& previous_token_str,@@ -846,34 +852,34 @@ public:std::vector<uint32_t>& prev_token_doc_ids,std::vector<size_t>& top_prefix_field_ids) const;- void search_across_fields(const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- const std::vector<search_field_t>& the_fields,- const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<sort_by>& sort_fields,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- bool prioritize_exact_match,- const bool search_all_candidates,- filter_result_iterator_t* const filter_result_iterator,- const uint32_t total_cost,- const int syn_orig_num_tokens,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::vector<uint32_t>& id_buff,- uint32_t*& all_result_ids,- size_t& all_result_ids_len) const;+ Option<bool> search_across_fields(const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ const std::vector<search_field_t>& the_fields,+ const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<sort_by>& sort_fields,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ bool prioritize_exact_match,+ const bool search_all_candidates,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t total_cost,+ const int syn_orig_num_tokens,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::vector<uint32_t>& id_buff,+ uint32_t*& all_result_ids,+ size_t& all_result_ids_len) const;voidsearch_fields(const std::vector<filter>& filters,@@ -903,13 +909,14 @@ public:filter_node_t*& filter_tree_root,std::vector<const override_t*>& matched_dynamic_overrides) const;- void compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices, uint32_t seq_id,- size_t filter_index,- int64_t max_field_match_score,- int64_t* scores,- int64_t& match_score_index, float vector_distance = 0) const;+ Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices, uint32_t seq_id,+ const std::map<basic_string<char>, reference_filter_result_t>& references,+ size_t filter_index,+ int64_t max_field_match_score,+ int64_t* scores,+ int64_t& match_score_index, float vector_distance = 0) const;void process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,const std::vector<uint32_t>& excluded_ids, const std::vector<std::string>& group_by_fields,diff --git a/include/topster.h b/include/topster.hindex 5985a48c..0b106f47 100644--- a/include/topster.h+++ b/include/topster.h@@ -33,7 +33,7 @@ struct KV {this->scores[1] = scores[1];this->scores[2] = scores[2];- if(match_score_index >= 0) {+ if(match_score_index >= 0 && match_score_index <= 2){this->text_match_score = scores[match_score_index];}}diff --git a/src/collection.cpp b/src/collection.cppindex 65afff8e..6d259e01 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -757,6 +757,44 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<for(size_t i = 0; i < sort_fields.size(); i++) {const sort_by& _sort_field = sort_fields[i];++ if (_sort_field.name[0] == '$') {+ // Reference sort_by+ auto parenthesis_index = _sort_field.name.find('(');+ std::string ref_collection_name = _sort_field.name.substr(1, parenthesis_index - 1);+ auto& cm = CollectionManager::get_instance();+ auto ref_collection = cm.get_collection(ref_collection_name);+ if (ref_collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + ref_collection_name + "` in `sort_by` not found.");+ }++ auto sort_by_str = _sort_field.name.substr(parenthesis_index + 1,+ _sort_field.name.size() - parenthesis_index - 2);+ std::vector<sort_by> ref_sort_fields;+ bool parsed_sort_by = CollectionManager::parse_sort_by_str(sort_by_str, ref_sort_fields);+ if (!parsed_sort_by) {+ return Option<bool>(400, "Reference `sort_by` is malformed.");+ }++ std::vector<sort_by> ref_sort_fields_std;+ auto sort_validation_op = ref_collection->validate_and_standardize_sort_fields_with_lock(ref_sort_fields,+ ref_sort_fields_std,+ is_wildcard_query,+ is_vector_query,+ is_group_by_query);+ if (!sort_validation_op.ok()) {+ return Option<bool>(sort_validation_op.code(), "Referenced collection `" + ref_collection_name + "`: " ++ sort_validation_op.error());+ }++ for (auto& ref_sort_field_std: ref_sort_fields_std) {+ ref_sort_field_std.reference_collection_name = ref_collection_name;+ sort_fields_std.emplace_back(ref_sort_field_std);+ }++ continue;+ }+sort_by sort_field_std(_sort_field.name, _sort_field.order);if(sort_field_std.name.back() == ')') {@@ -1012,6 +1050,214 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<return Option<bool>(true);}+Option<bool> Collection::validate_and_standardize_sort_fields_with_lock(const std::vector<sort_by> & sort_fields,+ std::vector<sort_by>& sort_fields_std,+ const bool is_wildcard_query,+ const bool is_vector_query,+ const bool is_group_by_query) const {+ std::shared_lock lock(mutex);++ for(const auto & sort_field : sort_fields) {+ sort_by sort_field_std(sort_field.name, sort_field.order);++ if(sort_field_std.name.back() == ')') {+ // check if this is a geo field or text match field+ size_t paran_start = 0;+ while(paran_start < sort_field_std.name.size() && sort_field_std.name[paran_start] != '(') {+ paran_start++;+ }++ const std::string& actual_field_name = sort_field_std.name.substr(0, paran_start);+ const auto field_it = search_schema.find(actual_field_name);++ if(actual_field_name == sort_field_const::text_match) {+ std::vector<std::string> match_parts;+ const std::string& match_config = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);+ StringUtils::split(match_config, match_parts, ":");+ if(match_parts.size() != 2 || match_parts[0] != "buckets") {+ return Option<bool>(400, "Invalid sorting parameter passed for _text_match.");+ }++ if(!StringUtils::is_uint32_t(match_parts[1])) {+ return Option<bool>(400, "Invalid value passed for _text_match `buckets` configuration.");+ }++ sort_field_std.name = actual_field_name;+ sort_field_std.text_match_buckets = std::stoll(match_parts[1]);++ } else if(actual_field_name == sort_field_const::eval) {+ const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,+ sort_field_std.name.size() - paran_start -+ 2);+ if(filter_exp.empty()) {+ return Option<bool>(400, "The eval expression in sort_by is empty.");+ }++ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,+ store, "", sort_field_std.eval.filter_tree_root);+ if(!parse_filter_op.ok()) {+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");+ }++ sort_field_std.name = actual_field_name;++ } else {+ if(field_it == search_schema.end()) {+ std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";+ return Option<bool>(404, error);+ }++ std::string error = "Bad syntax for sorting field `" + actual_field_name + "`";++ if(!field_it.value().is_geopoint()) {+ // check for null value order+ const std::string& sort_params_str = sort_field_std.name.substr(paran_start + 1,+ sort_field_std.name.size() -+ paran_start - 2);++ std::vector<std::string> param_parts;+ StringUtils::split(sort_params_str, param_parts, ":");++ if(param_parts.size() != 2) {+ return Option<bool>(400, error);+ }++ if(param_parts[0] != sort_field_const::missing_values) {+ return Option<bool>(400, error);+ }++ auto missing_values_op = magic_enum::enum_cast<sort_by::missing_values_t>(param_parts[1]);+ if(missing_values_op.has_value()) {+ sort_field_std.missing_values = missing_values_op.value();+ } else {+ return Option<bool>(400, error);+ }+ }++ else {+ const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);++ // e.g. geopoint_field(lat1, lng1, exclude_radius: 10 miles)++ std::vector<std::string> geo_parts;+ StringUtils::split(geo_coordstr, geo_parts, ",");++ if(geo_parts.size() != 2 && geo_parts.size() != 3) {+ return Option<bool>(400, error);+ }++ if(!StringUtils::is_float(geo_parts[0]) || !StringUtils::is_float(geo_parts[1])) {+ return Option<bool>(400, error);+ }++ if(geo_parts.size() == 3) {+ // try to parse the exclude radius option+ bool is_exclude_option = false;++ if(StringUtils::begins_with(geo_parts[2], sort_field_const::exclude_radius)) {+ is_exclude_option = true;+ } else if(StringUtils::begins_with(geo_parts[2], sort_field_const::precision)) {+ is_exclude_option = false;+ } else {+ return Option<bool>(400, error);+ }++ std::vector<std::string> param_parts;+ StringUtils::split(geo_parts[2], param_parts, ":");++ if(param_parts.size() != 2) {+ return Option<bool>(400, error);+ }++ // param_parts[1] is the value, in either "20km" or "20 km" format++ if(param_parts[1].size() < 2) {+ return Option<bool>(400, error);+ }++ std::string unit = param_parts[1].substr(param_parts[1].size()-2, 2);++ if(unit != "km" && unit != "mi") {+ return Option<bool>(400, "Sort field's parameter unit must be either `km` or `mi`.");+ }++ std::vector<std::string> dist_values;+ StringUtils::split(param_parts[1], dist_values, unit);++ if(dist_values.size() != 1) {+ return Option<bool>(400, error);+ }++ if(!StringUtils::is_float(dist_values[0])) {+ return Option<bool>(400, error);+ }++ int32_t value_meters;++ if(unit == "km") {+ value_meters = std::stof(dist_values[0]) * 1000;+ } else if(unit == "mi") {+ value_meters = std::stof(dist_values[0]) * 1609.34;+ } else {+ return Option<bool>(400, "Sort field's parameter "+ "unit must be either `km` or `mi`.");+ }++ if(value_meters <= 0) {+ return Option<bool>(400, "Sort field's parameter must be a positive number.");+ }++ if(is_exclude_option) {+ sort_field_std.exclude_radius = value_meters;+ } else {+ sort_field_std.geo_precision = value_meters;+ }+ }++ double lat = std::stod(geo_parts[0]);+ double lng = std::stod(geo_parts[1]);+ int64_t lat_lng = GeoPoint::pack_lat_lng(lat, lng);+ sort_field_std.geopoint = lat_lng;+ }++ sort_field_std.name = actual_field_name;+ }+ }++ if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&+ sort_field_std.name != sort_field_const::seq_id && sort_field_std.name != sort_field_const::group_found && sort_field_std.name != sort_field_const::vector_distance) {++ const auto field_it = search_schema.find(sort_field_std.name);+ if(field_it == search_schema.end() || !field_it.value().sort || !field_it.value().index) {+ std::string error = "Could not find a field named `" + sort_field_std.name ++ "` in the schema for sorting.";+ return Option<bool>(404, error);+ }+ }++ if(sort_field_std.name == sort_field_const::group_found && is_group_by_query == false) {+ std::string error = "group_by parameters should not be empty when using sort_by group_found";+ return Option<bool>(404, error);+ }++ if(sort_field_std.name == sort_field_const::vector_distance && !is_vector_query) {+ std::string error = "sort_by vector_distance is only supported for vector queries, semantic search and hybrid search.";+ return Option<bool>(404, error);+ }++ StringUtils::toupper(sort_field_std.order);++ if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {+ std::string error = "Order for field` " + sort_field_std.name + "` should be either ASC or DESC.";+ return Option<bool>(400, error);+ }++ sort_fields_std.emplace_back(sort_field_std);+ }++ return Option<bool>(true);+}+Option<bool> Collection::extract_field_name(const std::string& field_name,const tsl::htrie_map<char, field>& search_schema,std::vector<std::string>& processed_search_fields,@@ -1526,7 +1772,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,bool is_vector_query = !vector_query.field_name.empty();if(curated_sort_by.empty()) {- auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,+ auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields,sort_fields_std, is_wildcard_query, is_vector_query, is_group_by_query);if(!sort_validation_op.ok()) {return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());@@ -5113,3 +5359,16 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t kreturn Option<bool>(true);}++void Collection::reference_populate_sort_mapping(int *sort_order, std::vector<size_t> &geopoint_indices,+ std::vector<sort_by> &sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t> *, 3> &field_values)+ const {+ std::shared_lock lock(mutex);+ index->populate_sort_mapping_with_lock(sort_order, geopoint_indices, sort_fields_std, field_values);+}++int64_t Collection::reference_string_sort_score(const string &field_name, const uint32_t& seq_id) const {+ std::shared_lock lock(mutex);+ return index->reference_string_sort_score(field_name, seq_id);+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 7791b2a9..c47d0bc2 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -623,6 +623,33 @@ bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<schar prev_non_space_char = 'a';for(size_t i=0; i < sort_by_str.size(); i++) {+ if (sort_field_expr.empty() && sort_by_str[i] == '$') {+ // Sort by reference field+ auto open_paren_pos = sort_by_str.find('(', i);+ if (open_paren_pos == std::string::npos) {+ return false;+ }+ sort_field_expr = sort_by_str.substr(i, open_paren_pos - i + 1);++ i = open_paren_pos;+ int paren_count = 1;+ while (++i < sort_by_str.size() && paren_count > 0) {+ if (sort_by_str[i] == '(') {+ paren_count++;+ } else if (sort_by_str[i] == ')') {+ paren_count--;+ }+ sort_field_expr += sort_by_str[i];+ }+ if (paren_count != 0) {+ return false;+ }++ sort_fields.emplace_back(sort_field_expr, "");+ sort_field_expr = "";+ continue;+ }+if(i == sort_by_str.size()-1 || (sort_by_str[i] == ',' && !isdigit(prev_non_space_char))) {if(i == sort_by_str.size()-1) {sort_field_expr += sort_by_str[i];diff --git a/src/index.cpp b/src/index.cppindex db429001..10b73baf 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -22,6 +22,7 @@#include <timsort.hpp>#include "logger.h"#include "validator.h"+#include <collection_manager.h>#define RETURN_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \@@ -29,6 +30,12 @@return ;\}+#define RETURN_CIRCUIT_BREAKER_OP if((std::chrono::duration_cast<std::chrono::microseconds>( \+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \+ search_cutoff = true; \+ return Option<bool>(true);\+ }+#define BREAK_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \search_cutoff = true; \@@ -1431,36 +1438,36 @@ void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {}}-void Index::search_all_candidates(const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<search_field_t>& the_fields,- filter_result_iterator_t* const filter_result_iterator,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by>& sort_fields,- std::vector<tok_candidates>& token_candidates_vec,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- const size_t typo_tokens_threshold,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- bool prioritize_exact_match,- const bool prioritize_token_position,- const bool exhaustive_search,- const size_t max_candidates,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::set<uint64>& query_hashes,- std::vector<uint32_t>& id_buff) const {+Option<bool> Index::search_all_candidates(const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<search_field_t>& the_fields,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by>& sort_fields,+ std::vector<tok_candidates>& token_candidates_vec,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ const size_t typo_tokens_threshold,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ bool prioritize_exact_match,+ const bool prioritize_token_position,+ const bool exhaustive_search,+ const size_t max_candidates,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::set<uint64>& query_hashes,+ std::vector<uint32_t>& id_buff) const {/*if(!token_candidates_vec.empty()) {LOG(INFO) << "Prefix candidates size: " << token_candidates_vec.back().candidates.size();@@ -1476,7 +1483,7 @@ void Index::search_all_candidates(const size_t num_search_fields,std::max<size_t>(Index::COMBINATION_MIN_LIMIT, max_candidates);for(long long n = 0; n < N && n < combination_limit; ++n) {- RETURN_CIRCUIT_BREAKER+ RETURN_CIRCUIT_BREAKER_OPstd::vector<token_t> query_suggestion(token_candidates_vec.size());@@ -1499,19 +1506,26 @@ void Index::search_all_candidates(const size_t num_search_fields,//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;- search_across_fields(query_suggestion, num_typos, prefixes, the_fields, num_search_fields, match_type,- sort_fields, topster,groups_processed,- searched_queries, qtoken_set, dropped_tokens, group_limit, group_by_fields,- prioritize_exact_match, prioritize_token_position,- filter_result_iterator,- total_cost, syn_orig_num_tokens,- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,- sort_order, field_values, geopoint_indices,- id_buff, all_result_ids, all_result_ids_len);+ auto search_across_fields_op = search_across_fields(query_suggestion, num_typos, prefixes, the_fields,+ num_search_fields, match_type,+ sort_fields, topster,groups_processed,+ searched_queries, qtoken_set, dropped_tokens,+ group_limit, group_by_fields,+ prioritize_exact_match, prioritize_token_position,+ filter_result_iterator,+ total_cost, syn_orig_num_tokens,+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,+ sort_order, field_values, geopoint_indices,+ id_buff, all_result_ids, all_result_ids_len);+ if (!search_across_fields_op.ok()) {+ return search_across_fields_op;+ }query_hashes.insert(qhash);filter_result_iterator->reset();}++ return Option<bool>(true);}void Index::search_candidates(const uint8_t & field_id, bool field_is_array,@@ -2258,16 +2272,21 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons// handle phrase searchesif (!field_query_tokens[0].q_phrases.empty()) {- do_phrase_search(num_search_fields, the_fields, field_query_tokens,- sort_fields_std, searched_queries, group_limit, group_by_fields,- topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,- filter_result_iterator, all_result_ids, all_result_ids_len, groups_processed, curated_ids,- excluded_result_ids, excluded_result_ids_size, excluded_group_ids, curated_topster,- included_ids_map, is_wildcard_query);+ auto do_phrase_search_op = do_phrase_search(num_search_fields, the_fields, field_query_tokens,+ sort_fields_std, searched_queries, group_limit, group_by_fields,+ topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,+ filter_result_iterator, all_result_ids, all_result_ids_len,+ groups_processed, curated_ids,+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,+ curated_topster, included_ids_map, is_wildcard_query);filter_iterator_guard.release();filter_iterator_guard.reset(filter_result_iterator);+ if (!do_phrase_search_op.ok()) {+ return do_phrase_search_op;+ }+if (filter_result_iterator->approx_filter_ids_length == 0) {goto process_search_results;}@@ -2397,6 +2416,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor (auto& dist_result : dist_results) {auto& seq_id = dist_result.second.seq_id;+ auto references = std::move(dist_result.second.reference_filter_results);if(vector_query.query_doc_given && vector_query.seq_id == seq_id) {continue;@@ -2420,12 +2440,16 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consint64_t scores[3] = {0};int64_t match_score_index = -1;- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices,- seq_id, 0, 0, scores, match_score_index, vec_dist_score);+ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,+ geopoint_indices, seq_id, references, 0, 0, scores,+ match_score_index, vec_dist_score);+ if (!compute_sort_scores_op.ok()) {+ return compute_sort_scores_op;+ }KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);kv.vector_distance = vec_dist_score;- kv.reference_filter_results = std::move(dist_result.second.reference_filter_results);+ kv.reference_filter_results = std::move(references);int ret = topster->add(&kv);if(group_limit != 0 && ret < 2) {@@ -2447,13 +2471,16 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfilter_iterator_guard.reset(filter_result_iterator);}- search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,- curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,- curated_ids, curated_ids_sorted,- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,- all_result_ids, all_result_ids_len,- filter_result_iterator, concurrency,- sort_order, field_values, geopoint_indices);+ auto search_wildcard_op = search_wildcard(filter_tree_root, included_ids_map, sort_fields_std, topster,+ curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,+ curated_ids, curated_ids_sorted,+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,+ all_result_ids, all_result_ids_len,+ filter_result_iterator, concurrency,+ sort_order, field_values, geopoint_indices);+ if (!search_wildcard_op.ok()) {+ return search_wildcard_op;+ }filter_result_iterator->reset();}@@ -2499,15 +2526,20 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}}- fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type, excluded_result_ids,- excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,- excluded_group_ids,- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,- prioritize_token_position, query_hashes, token_order, prefixes,- typo_tokens_threshold, exhaustive_search,- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order,- field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, {}, match_type,+ excluded_result_ids, excluded_result_ids_size,+ filter_result_iterator, curated_ids_sorted,+ excluded_group_ids, sort_fields_std, num_typos,+ searched_queries, qtoken_set, topster, groups_processed,+ all_result_ids, all_result_ids_len,+ group_limit, group_by_fields, prioritize_exact_match,+ prioritize_token_position, query_hashes, token_order, prefixes,+ typo_tokens_threshold, exhaustive_search,+ max_candidates, min_len_1typo, min_len_2typo,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }// try split/joining tokens if no results are foundif(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {@@ -2537,28 +2569,38 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consspace_resolved_queries[0][j].size(), 0);}- fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,- excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,- excluded_group_ids,- sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,- prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold, exhaustive_search,- max_candidates, min_len_1typo, min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, resolved_tokens, {}, match_type, excluded_result_ids,+ excluded_result_ids_size, filter_result_iterator, curated_ids_sorted,+ excluded_group_ids,+ sort_fields_std, num_typos, searched_queries,+ qtoken_set, topster, groups_processed,+ all_result_ids, all_result_ids_len,+ group_limit, group_by_fields, prioritize_exact_match,+ prioritize_token_position, query_hashes, token_order,+ prefixes, typo_tokens_threshold, exhaustive_search,+ max_candidates, min_len_1typo, min_len_2typo,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }}}// do synonym based searches- do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map, sort_fields_std,- curated_topster, token_order,- 0, group_limit, group_by_fields, prioritize_exact_match, prioritize_token_position,- exhaustive_search, concurrency, prefixes,- min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,- excluded_result_ids, excluded_result_ids_size, excluded_group_ids,- topster, q_pos_synonyms, syn_orig_num_tokens,- groups_processed, searched_queries, all_result_ids, all_result_ids_len,- filter_result_iterator, query_hashes,- sort_order, field_values, geopoint_indices,- qtoken_set);+ auto do_synonym_search_op = do_synonym_search(the_fields, match_type, filter_tree_root, included_ids_map,+ sort_fields_std, curated_topster, token_order, 0, group_limit,+ group_by_fields, prioritize_exact_match, prioritize_token_position,+ exhaustive_search, concurrency, prefixes,+ min_len_1typo, min_len_2typo, max_candidates, curated_ids, curated_ids_sorted,+ excluded_result_ids, excluded_result_ids_size, excluded_group_ids,+ topster, q_pos_synonyms, syn_orig_num_tokens,+ groups_processed, searched_queries, all_result_ids, all_result_ids_len,+ filter_result_iterator, query_hashes,+ sort_order, field_values, geopoint_indices,+ qtoken_set);+ if (!do_synonym_search_op.ok()) {+ return do_synonym_search_op;+ }filter_result_iterator->reset();// gather up both original query and synonym queries and do drop tokens@@ -2606,16 +2648,21 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consdrop_token_prefixes.push_back(p && prefix_search);}- fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,- excluded_result_ids, excluded_result_ids_size,- filter_result_iterator,- curated_ids_sorted, excluded_group_ids, sort_fields_std, num_typos, searched_queries,- qtoken_set, topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields,- prioritize_exact_match, prioritize_token_position, query_hashes,- token_order, prefixes, typo_tokens_threshold,- exhaustive_search, max_candidates, min_len_1typo,- min_len_2typo, -1, sort_order, field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, truncated_tokens, dropped_tokens, match_type,+ excluded_result_ids, excluded_result_ids_size,+ filter_result_iterator,+ curated_ids_sorted, excluded_group_ids,+ sort_fields_std, num_typos, searched_queries,+ qtoken_set, topster, groups_processed,+ all_result_ids, all_result_ids_len,+ group_limit, group_by_fields,+ prioritize_exact_match, prioritize_token_position, query_hashes,+ token_order, prefixes, typo_tokens_threshold,+ exhaustive_search, max_candidates, min_len_1typo,+ min_len_2typo, -1, sort_order, field_values, geopoint_indices);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }} else {break;@@ -2624,14 +2671,18 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}}- do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std, searched_queries,- group_limit, group_by_fields,- max_extra_prefix, max_extra_suffix,- field_query_tokens[0].q_include_tokens,- topster, filter_result_iterator,- sort_order, field_values, geopoint_indices,- curated_ids_sorted, excluded_group_ids, all_result_ids, all_result_ids_len, groups_processed);-+ auto do_infix_search_op = do_infix_search(num_search_fields, the_fields, infixes, sort_fields_std,+ searched_queries,+ group_limit, group_by_fields,+ max_extra_prefix, max_extra_suffix,+ field_query_tokens[0].q_include_tokens,+ topster, filter_result_iterator,+ sort_order, field_values, geopoint_indices,+ curated_ids_sorted, excluded_group_ids,+ all_result_ids, all_result_ids_len, groups_processed);+ if (!do_infix_search_op.ok()) {+ return do_infix_search_op;+ }filter_result_iterator->reset();if(!vector_query.field_name.empty()) {@@ -2701,6 +2752,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor(size_t res_index = 0; res_index < vec_results.size(); res_index++) {auto& vec_result = vec_results[res_index];auto doc_id = vec_result.first;++ filter_result_iterator->skip_to(doc_id);+ auto references = std::move(filter_result_iterator->reference);+ filter_result_iterator->reset();auto result_it = topster->kv_map.find(doc_id);if(result_it != topster->kv_map.end()) {@@ -2719,8 +2774,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT));int64_t match_score_index = -1;int64_t scores[3] = {0};-- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);++ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,+ geopoint_indices, doc_id, references, 0,+ match_score, scores, match_score_index,+ vec_result.second);+ if (!compute_sort_scores_op.ok()) {+ return compute_sort_scores_op;+ }for(int i = 0; i < 3; i++) {result->scores[i] = scores[i];@@ -2732,18 +2793,17 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consint64_t scores[3] = {0};int64_t match_score = float_to_int64_t((1.0 / (res_index + 1)) * VECTOR_SEARCH_WEIGHT);int64_t match_score_index = -1;- compute_sort_scores(sort_fields_std, sort_order, field_values, geopoint_indices, doc_id, 0, match_score, scores, match_score_index, vec_result.second);- KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);- kv.text_match_score = 0;- kv.vector_distance = vec_result.second;- if (filter_result_iterator->is_valid &&- !filter_result_iterator->reference.empty()) {- // The doc_id must be valid otherwise it would've been filtered out upstream.- filter_result_iterator->skip_to(doc_id);- kv.reference_filter_results = std::move(filter_result_iterator->reference);- filter_result_iterator->reset();+ auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,+ geopoint_indices, doc_id, references, 0,+ match_score, scores, match_score_index,+ vec_result.second);+ if (!compute_sort_scores_op.ok()) {+ return compute_sort_scores_op;}+ KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores, std::move(references));+ kv.text_match_score = 0;+ kv.vector_distance = vec_result.second;topster->add(&kv);vec_search_ids.push_back(doc_id);@@ -3046,36 +3106,36 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>}}-void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,- const std::vector<token_t>& query_tokens,- const std::vector<token_t>& dropped_tokens,- const text_match_type_t match_type,- const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- filter_result_iterator_t* const filter_result_iterator,- const std::vector<uint32_t>& curated_ids,- const std::unordered_set<uint32_t>& excluded_group_ids,- const std::vector<sort_by> & sort_fields,- const std::vector<uint32_t>& num_typos,- std::vector<std::vector<art_leaf*>> & searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- uint32_t*& all_result_ids, size_t & all_result_ids_len,- const size_t group_limit, const std::vector<std::string>& group_by_fields,- bool prioritize_exact_match,- const bool prioritize_token_position,- std::set<uint64>& query_hashes,- const token_ordering token_order,- const std::vector<bool>& prefixes,- const size_t typo_tokens_threshold,- const bool exhaustive_search,- const size_t max_candidates,- size_t min_len_1typo,- size_t min_len_2typo,- int syn_orig_num_tokens,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const {+Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,+ const std::vector<token_t>& query_tokens,+ const std::vector<token_t>& dropped_tokens,+ const text_match_type_t match_type,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ filter_result_iterator_t* const filter_result_iterator,+ const std::vector<uint32_t>& curated_ids,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const std::vector<sort_by> & sort_fields,+ const std::vector<uint32_t>& num_typos,+ std::vector<std::vector<art_leaf*>> & searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ Topster* topster, spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ uint32_t*& all_result_ids, size_t & all_result_ids_len,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,+ bool prioritize_exact_match,+ const bool prioritize_token_position,+ std::set<uint64>& query_hashes,+ const token_ordering token_order,+ const std::vector<bool>& prefixes,+ const size_t typo_tokens_threshold,+ const bool exhaustive_search,+ const size_t max_candidates,+ size_t min_len_1typo,+ size_t min_len_2typo,+ int syn_orig_num_tokens,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices) const {// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens@@ -3112,7 +3172,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,const long long combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;while(n < N && n < combination_limit) {- RETURN_CIRCUIT_BREAKER+ RETURN_CIRCUIT_BREAKER_OP//LOG(INFO) << "fuzzy_search_fields, n: " << n;@@ -3290,7 +3350,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,if(token_to_costs[token_index].empty()) {// we cannot proceed further, as this token is not found within cost limits// and, dropping of tokens are done elsewhere.- return ;+ return Option<bool>(true);}}@@ -3306,16 +3366,20 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,if(token_candidates_vec.size() == query_tokens.size()) {std::vector<uint32_t> id_buff;- search_all_candidates(num_search_fields, match_type, the_fields, filter_result_iterator,- exclude_token_ids, exclude_token_ids_size, excluded_group_ids,- sort_fields, token_candidates_vec, searched_queries, qtoken_set,- dropped_tokens, topster,- groups_processed, all_result_ids, all_result_ids_len,- typo_tokens_threshold, group_limit, group_by_fields, query_tokens,- num_typos, prefixes, prioritize_exact_match, prioritize_token_position,- exhaustive_search, max_candidates,- syn_orig_num_tokens, sort_order, field_values, geopoint_indices,- query_hashes, id_buff);+ auto search_all_candidates_op = search_all_candidates(num_search_fields, match_type, the_fields,+ filter_result_iterator,+ exclude_token_ids, exclude_token_ids_size, excluded_group_ids,+ sort_fields, token_candidates_vec, searched_queries, qtoken_set,+ dropped_tokens, topster,+ groups_processed, all_result_ids, all_result_ids_len,+ typo_tokens_threshold, group_limit, group_by_fields, query_tokens,+ num_typos, prefixes, prioritize_exact_match, prioritize_token_position,+ exhaustive_search, max_candidates,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices,+ query_hashes, id_buff);+ if (!search_all_candidates_op.ok()) {+ return search_all_candidates_op;+ }if(id_buff.size() > 1) {gfx::timsort(id_buff.begin(), id_buff.end());@@ -3333,11 +3397,13 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,if(!exhaustive_search && all_result_ids_len >= typo_tokens_threshold) {// if typo threshold is breached, we are done- return ;+ return Option<bool>(true);}n++;}++ return Option<bool>(true);}void Index::popular_fields_of_token(const spp::sparse_hash_map<std::string, art_tree*>& search_index,@@ -3454,31 +3520,31 @@ void Index::find_across_fields(const token_t& previous_token,}}-void Index::search_across_fields(const std::vector<token_t>& query_tokens,- const std::vector<uint32_t>& num_typos,- const std::vector<bool>& prefixes,- const std::vector<search_field_t>& the_fields,- const size_t num_search_fields,- const text_match_type_t match_type,- const std::vector<sort_by>& sort_fields,- Topster* topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- tsl::htrie_map<char, token_leaf>& qtoken_set,- const std::vector<token_t>& dropped_tokens,- const size_t group_limit,- const std::vector<std::string>& group_by_fields,- const bool prioritize_exact_match,- const bool prioritize_token_position,- filter_result_iterator_t* const filter_result_iterator,- const uint32_t total_cost, const int syn_orig_num_tokens,- const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- std::vector<uint32_t>& id_buff,- uint32_t*& all_result_ids, size_t& all_result_ids_len) const {+Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokens,+ const std::vector<uint32_t>& num_typos,+ const std::vector<bool>& prefixes,+ const std::vector<search_field_t>& the_fields,+ const size_t num_search_fields,+ const text_match_type_t match_type,+ const std::vector<sort_by>& sort_fields,+ Topster* topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ tsl::htrie_map<char, token_leaf>& qtoken_set,+ const std::vector<token_t>& dropped_tokens,+ const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ const bool prioritize_exact_match,+ const bool prioritize_token_position,+ filter_result_iterator_t* const filter_result_iterator,+ const uint32_t total_cost, const int syn_orig_num_tokens,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ std::vector<uint32_t>& id_buff,+ uint32_t*& all_result_ids, size_t& all_result_ids_len) const {std::vector<art_leaf*> query_suggestion;@@ -3596,10 +3662,12 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,std::vector<uint32_t> result_ids;size_t filter_index = 0;+ Option<bool> status(true);or_iterator_t::intersect(token_its, istate,[&](single_filter_result_t& filter_result, const std::vector<or_iterator_t>& its) {auto& seq_id = filter_result.seq_id;+ auto references = std::move(filter_result.reference_filter_results);//LOG(INFO) << "seq_id: " << seq_id;// Convert [token -> fields] orientation to [field -> tokens] orientationstd::vector<std::vector<posting_list_t::iterator_t>> field_to_tokens(num_search_fields);@@ -3667,8 +3735,13 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,int64_t scores[3] = {0};int64_t match_score_index = -1;- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,- best_field_match_score, scores, match_score_index);+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,+ seq_id, references, filter_index, best_field_match_score,+ scores, match_score_index);+ if (!compute_sort_scores_op.ok()) {+ status = Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());+ return;+ }size_t query_len = query_tokens.size();@@ -3719,8 +3792,7 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,<< ", num_matching_fields: " << num_matching_fields<< ", aggregated_score: " << aggregated_score;*/- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- kv.reference_filter_results = std::move(filter_result.reference_filter_results);+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));if(match_score_index != -1) {kv.scores[match_score_index] = aggregated_score;@@ -3733,6 +3805,15 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,}result_ids.push_back(seq_id);});+ if (!status.ok()) {+ for(posting_list_t* plist: expanded_plists) {+ delete plist;+ }+ for(posting_list_t* plist: expanded_dropped_plists) {+ delete plist;+ }+ return status;+ }id_buff.insert(id_buff.end(), result_ids.begin(), result_ids.end());@@ -3763,13 +3844,16 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,for(posting_list_t* plist: expanded_dropped_plists) {delete plist;}++ return Option<bool>(true);}-void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- uint32_t seq_id, size_t filter_index, int64_t max_field_match_score,- int64_t* scores, int64_t& match_score_index, float vector_distance) const {+Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ uint32_t seq_id, const std::map<basic_string<char>, reference_filter_result_t>& references,+ size_t filter_index, int64_t max_field_match_score, int64_t* scores,+ int64_t& match_score_index, float vector_distance) const {int64_t geopoint_distances[3];@@ -3827,6 +3911,24 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i// avoiding loopif (sort_fields.size() > 0) {+ // In case of reference sort_by, we need to get the sort score of the reference doc id.+ if (!sort_fields[0].reference_collection_name.empty()) {+ auto& sort_field = sort_fields[0];+ if (references.count(sort_field.reference_collection_name) == 0 ||+ references.at(sort_field.reference_collection_name).count == 0) {+ return Option<bool>(400, "No references found to sort by on `" ++ sort_field.reference_collection_name + "." + sort_field.name + "`.");+ }++ auto& reference_result = references.at(sort_field.reference_collection_name);+ if (reference_result.count > 1) {+ return Option<bool>(400, "Multiple references found to sort by on `" ++ sort_field.reference_collection_name + "." + sort_field.name + "`.");+ }++ seq_id = reference_result.docs[0];+ }+if (field_values[0] == &text_match_sentinel_value) {scores[0] = int64_t(max_field_match_score);match_score_index = 0;@@ -3835,7 +3937,19 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i} else if(field_values[0] == &geo_sentinel_value) {scores[0] = geopoint_distances[0];} else if(field_values[0] == &str_sentinel_value) {- scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);+ if (sort_fields[0].reference_collection_name.empty()) {+ scores[0] = str_sort_index.at(sort_fields[0].name)->rank(seq_id);+ } else {+ auto& cm = CollectionManager::get_instance();+ auto ref_collection = cm.get_collection(sort_fields[0].reference_collection_name);+ if (ref_collection == nullptr) {+ return Option<bool>(400, "Referenced collection `" + sort_fields[0].reference_collection_name ++ "` not found.");+ }++ scores[0] = ref_collection->reference_string_sort_score(sort_fields[0].name, seq_id);+ }+if(scores[0] == adi_tree_t::NOT_FOUND) {if(sort_fields[0].order == sort_field_const::asc &&sort_fields[0].missing_values == sort_by::missing_values_t::first) {@@ -3885,6 +3999,24 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i}if(sort_fields.size() > 1) {+ // In case of reference sort_by, we need to get the sort score of the reference doc id.+ if (!sort_fields[1].reference_collection_name.empty()) {+ auto& sort_field = sort_fields[1];+ if (references.count(sort_field.reference_collection_name) == 0 ||+ references.at(sort_field.reference_collection_name).count == 0) {+ return Option<bool>(400, "No references found to sort by on `" ++ sort_field.reference_collection_name + "." + sort_field.name + "`.");+ }++ auto& reference_result = references.at(sort_field.reference_collection_name);+ if (reference_result.count > 1) {+ return Option<bool>(400, "Multiple references found to sort by on `" ++ sort_field.reference_collection_name + "." + sort_field.name + "`.");+ }++ seq_id = reference_result.docs[0];+ }+if (field_values[1] == &text_match_sentinel_value) {scores[1] = int64_t(max_field_match_score);match_score_index = 1;@@ -3939,6 +4071,24 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i}if(sort_fields.size() > 2) {+ // In case of reference sort_by, we need to get the sort score of the reference doc id.+ if (!sort_fields[2].reference_collection_name.empty()) {+ auto& sort_field = sort_fields[2];+ if (references.count(sort_field.reference_collection_name) == 0 ||+ references.at(sort_field.reference_collection_name).count == 0) {+ return Option<bool>(400, "No references found to sort by on `" ++ sort_field.reference_collection_name + "." + sort_field.name + "`.");+ }++ auto& reference_result = references.at(sort_field.reference_collection_name);+ if (reference_result.count > 1) {+ return Option<bool>(400, "Multiple references found to sort by on `" ++ sort_field.reference_collection_name + "." + sort_field.name + "`.");+ }++ seq_id = reference_result.docs[0];+ }+if (field_values[2] == &text_match_sentinel_value) {scores[2] = int64_t(max_field_match_score);match_score_index = 2;@@ -3991,27 +4141,29 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const iscores[2] = -scores[2];}}++ return Option<bool>(true);}-void Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,- std::vector<query_tokens_t>& field_query_tokens,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields,- Topster* actual_topster,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- filter_result_iterator_t*& filter_result_iterator,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- const std::set<uint32_t>& curated_ids,- const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* curated_topster,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- bool is_wildcard_query) const {+Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::vector<search_field_t>& search_fields,+ std::vector<query_tokens_t>& field_query_tokens,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields,+ Topster* actual_topster,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ filter_result_iterator_t*& filter_result_iterator,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ const std::set<uint32_t>& curated_ids,+ const uint32_t* excluded_result_ids, size_t excluded_result_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* curated_topster,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ bool is_wildcard_query) const {uint32_t* phrase_result_ids = nullptr;uint32_t phrase_result_count = 0;@@ -4114,7 +4266,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<sif (!is_wildcard_query) {// this means that the there are non-phrase tokens in the query// so we cannot directly copy to the all_result_ids array- return;+ return Option<bool>(true);}all_result_ids_len = filter_result_iterator->to_filter_id_array(all_result_ids);@@ -4124,15 +4276,19 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s// populate topsterfor(size_t i = 0; i < std::min<size_t>(10000, all_result_ids_len); i++) {auto seq_id = filter_result_iterator->seq_id;- auto reference_filter_results = std::move(filter_result_iterator->reference);+ auto references = std::move(filter_result_iterator->reference);filter_result_iterator->next();int64_t match_score = phrase_match_id_scores[seq_id];int64_t scores[3] = {0};int64_t match_score_index = -1;- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,- match_score, scores, match_score_index);+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,+ seq_id, references, filter_index, match_score, scores,+ match_score_index);+ if (!compute_sort_scores_op.ok()) {+ return compute_sort_scores_op;+ }uint64_t distinct_id = seq_id;if(group_limit != 0) {@@ -4142,8 +4298,7 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<s}}- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- kv.reference_filter_results = std::move(reference_filter_results);+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = actual_topster->add(&kv);if(group_limit != 0 && ret < 2) {@@ -4157,66 +4312,73 @@ void Index::do_phrase_search(const size_t num_search_fields, const std::vector<sfilter_result_iterator->reset();searched_queries.push_back({});+ return Option<bool>(true);}-void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,- const text_match_type_t match_type,- filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,- const token_ordering& token_order,- const size_t typo_tokens_threshold, const size_t group_limit,- const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,- const bool prioritize_token_position,- const bool exhaustive_search, const size_t concurrency,- const std::vector<bool>& prefixes,- size_t min_len_1typo,- size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size,- const std::unordered_set<uint32_t>& excluded_group_ids,- Topster* actual_topster,- std::vector<std::vector<token_t>>& q_pos_synonyms,- int syn_orig_num_tokens,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- filter_result_iterator_t* const filter_result_iterator,- std::set<uint64>& query_hashes,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices,- tsl::htrie_map<char, token_leaf>& qtoken_set) const {+Option<bool> Index::do_synonym_search(const std::vector<search_field_t>& the_fields,+ const text_match_type_t match_type,+ filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields_std, Topster* curated_topster,+ const token_ordering& token_order,+ const size_t typo_tokens_threshold, const size_t group_limit,+ const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,+ const bool prioritize_token_position,+ const bool exhaustive_search, const size_t concurrency,+ const std::vector<bool>& prefixes,+ size_t min_len_1typo,+ size_t min_len_2typo, const size_t max_candidates, const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ Topster* actual_topster,+ std::vector<std::vector<token_t>>& q_pos_synonyms,+ int syn_orig_num_tokens,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ std::set<uint64>& query_hashes,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices,+ tsl::htrie_map<char, token_leaf>& qtoken_set) const {for (const auto& syn_tokens : q_pos_synonyms) {query_hashes.clear();- fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,- exclude_token_ids_size, filter_result_iterator, curated_ids_sorted, excluded_group_ids,- sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,- all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,- prioritize_token_position, query_hashes, token_order, prefixes, typo_tokens_threshold,- exhaustive_search, max_candidates, min_len_1typo,- min_len_2typo, syn_orig_num_tokens, sort_order, field_values, geopoint_indices);+ auto fuzzy_search_fields_op = fuzzy_search_fields(the_fields, syn_tokens, {}, match_type, exclude_token_ids,+ exclude_token_ids_size, filter_result_iterator,+ curated_ids_sorted, excluded_group_ids, sort_fields_std, {0},+ searched_queries, qtoken_set, actual_topster, groups_processed,+ all_result_ids, all_result_ids_len, group_limit, group_by_fields,+ prioritize_exact_match, prioritize_token_position, query_hashes,+ token_order, prefixes, typo_tokens_threshold, exhaustive_search,+ max_candidates, min_len_1typo, min_len_2typo,+ syn_orig_num_tokens, sort_order, field_values, geopoint_indices);+ if (!fuzzy_search_fields_op.ok()) {+ return fuzzy_search_fields_op;+ }}collate_included_ids({}, included_ids_map, curated_topster, searched_queries);+ return Option<bool>(true);}-void Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,- const std::vector<enable_t>& infixes,- const std::vector<sort_by>& sort_fields,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,- const size_t max_extra_suffix,- const std::vector<token_t>& query_tokens, Topster* actual_topster,- filter_result_iterator_t* const filter_result_iterator,- const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,- const std::vector<size_t>& geopoint_indices,- const std::vector<uint32_t>& curated_ids_sorted,- const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {+Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,+ const std::vector<enable_t>& infixes,+ const std::vector<sort_by>& sort_fields,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,+ const size_t max_extra_suffix,+ const std::vector<token_t>& query_tokens, Topster* actual_topster,+ filter_result_iterator_t* const filter_result_iterator,+ const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,+ const std::vector<uint32_t>& curated_ids_sorted,+ const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed) const {for(size_t field_id = 0; field_id < num_search_fields; field_id++) {auto& field_name = the_fields[field_id].name;@@ -4259,6 +4421,10 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<sefor(size_t i = 0; i < raw_infix_ids_length; i++) {auto seq_id = raw_infix_ids[i];+ std::map<basic_string<char>, reference_filter_result_t> references;+ for (const auto& item: filtered_infix_ids.reference_filter_results) {+ references[item.first] = item.second[i];+ }int64_t match_score = 0;score_results2(sort_fields, searched_queries.size(), field_id, field_is_array,@@ -4267,8 +4433,12 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<seint64_t scores[3] = {0};int64_t match_score_index = -1;- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,- 100, scores, match_score_index);+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values,+ geopoint_indices, seq_id, references,+ filter_index, 100, scores, match_score_index);+ if (!compute_sort_scores_op.ok()) {+ return compute_sort_scores_op;+ }uint64_t distinct_id = seq_id;if(group_limit != 0) {@@ -4278,10 +4448,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se}}- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- for (const auto& item: filtered_infix_ids.reference_filter_results) {- kv.reference_filter_results[item.first] = item.second[i];- }+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = actual_topster->add(&kv);if(group_limit != 0 && ret < 2) {@@ -4308,6 +4475,8 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se}}}++ return Option<bool>(true);}void Index::handle_exclusion(const size_t num_search_fields, std::vector<query_tokens_t>& field_query_tokens,@@ -4561,20 +4730,20 @@ void Index::curate_filtered_ids(const std::set<uint32_t>& curated_ids,}}-void Index::search_wildcard(filter_node_t const* const& filter_tree_root,- const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,- const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,- spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,- std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,- const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,- const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,- size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,- uint32_t*& all_result_ids, size_t& all_result_ids_len,- filter_result_iterator_t* const filter_result_iterator,- const size_t concurrency,- const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,- const std::vector<size_t>& geopoint_indices) const {+Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root,+ const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,+ const std::vector<sort_by>& sort_fields, Topster* topster, Topster* curated_topster,+ spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,+ std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,+ const std::vector<std::string>& group_by_fields, const std::set<uint32_t>& curated_ids,+ const std::vector<uint32_t>& curated_ids_sorted, const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size, const std::unordered_set<uint32_t>& excluded_group_ids,+ uint32_t*& all_result_ids, size_t& all_result_ids_len,+ filter_result_iterator_t* const filter_result_iterator,+ const size_t concurrency,+ const int* sort_order,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ const std::vector<size_t>& geopoint_indices) const {filter_result_iterator->compute_result();auto const& approx_filter_ids_length = filter_result_iterator->approx_filter_ids_length;@@ -4602,6 +4771,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,const auto parent_search_stop_ms = search_stop_us;auto parent_search_cutoff = search_cutoff;uint32_t excluded_result_index = 0;+ Option<bool>* compute_sort_score_statuses[num_threads];for(size_t thread_id = 0; thread_id < num_threads && filter_result_iterator->is_valid; thread_id++) {filter_result_t batch_result;@@ -4613,14 +4783,15 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,searched_queries.push_back({});topsters[thread_id] = new Topster(topster->MAX_SIZE, topster->distinct);+ auto& compute_sort_score_status = compute_sort_score_statuses[thread_id] = nullptr;thread_pool->enqueue([this, &parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,- thread_id, &sort_fields, &searched_queries,- &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,- &sort_order, field_values, &geopoint_indices, &plists,- check_for_circuit_break,- batch_result,- &num_processed, &m_process, &cv_process]() {+ thread_id, &sort_fields, &searched_queries,+ &group_limit, &group_by_fields, &topsters, &tgroups_processed, &excluded_group_ids,+ &sort_order, field_values, &geopoint_indices, &plists,+ check_for_circuit_break,+ batch_result,+ &num_processed, &m_process, &cv_process, &compute_sort_score_status]() {search_begin_us = parent_search_begin;search_stop_us = parent_search_stop_ms;@@ -4630,6 +4801,11 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,for(size_t i = 0; i < batch_result.count; i++) {const uint32_t seq_id = batch_result.docs[i];+ std::map<basic_string<char>, reference_filter_result_t> references;+ for (const auto& item: batch_result.reference_filter_results) {+ references[item.first] = item.second[i];+ }+int64_t match_score = 0;score_results2(sort_fields, (uint16_t) searched_queries.size(), 0, false, 0,@@ -4638,8 +4814,13 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,int64_t scores[3] = {0};int64_t match_score_index = -1;- compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices, seq_id, filter_index,- 100, scores, match_score_index);+ auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,+ seq_id, references, filter_index, 100, scores,+ match_score_index);+ if (!compute_sort_scores_op.ok()) {+ compute_sort_score_status = new Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());+ break;+ }uint64_t distinct_id = seq_id;if(group_limit != 0) {@@ -4649,10 +4830,7 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,}}- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- for (const auto& item: batch_result.reference_filter_results) {- kv.reference_filter_results[item.first] = item.second[i];- }+ KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, std::move(references));int ret = topsters[thread_id]->add(&kv);if(group_limit != 0 && ret < 2) {@@ -4677,6 +4855,19 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,search_cutoff = parent_search_cutoff;for(size_t thread_id = 0; thread_id < num_processed; thread_id++) {+ if (compute_sort_score_statuses[thread_id] != nullptr) {+ auto& status = compute_sort_score_statuses[thread_id];+ auto return_value = Option<bool>(status->code(), status->error());++ // Cleanup the remaining threads.+ for (size_t i = thread_id; i < num_processed; i++) {+ delete compute_sort_score_statuses[i];+ delete topsters[i];+ }++ return return_value;+ }+//groups_processed.insert(tgroups_processed[thread_id].begin(), tgroups_processed[thread_id].end());for(const auto& it : tgroups_processed[thread_id]) {groups_processed[it.first]+= it.second;@@ -4691,12 +4882,38 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,filter_result_iterator->reset();all_result_ids_len = filter_result_iterator->to_filter_id_array(all_result_ids);++ return Option<bool>(true);}void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,std::vector<sort_by>& sort_fields_std,std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const {for (size_t i = 0; i < sort_fields_std.size(); i++) {+ if (!sort_fields_std[i].reference_collection_name.empty()) {+ auto& cm = CollectionManager::get_instance();+ auto ref_collection = cm.get_collection(sort_fields_std[i].reference_collection_name);++ int ref_sort_order[1];+ std::vector<size_t> ref_geopoint_indices;+ std::vector<sort_by> ref_sort_fields_std;+ ref_sort_fields_std.emplace_back(sort_fields_std[i]);+ ref_sort_fields_std.front().reference_collection_name.clear();+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> ref_field_values;+ ref_collection->reference_populate_sort_mapping(ref_sort_order, ref_geopoint_indices,+ ref_sort_fields_std, ref_field_values);++ sort_order[i] = ref_sort_order[0];+ if (!ref_geopoint_indices.empty()) {+ geopoint_indices.push_back(i);+ }+ sort_fields_std[i] = ref_sort_fields_std[0];+ sort_fields_std[i].reference_collection_name = ref_collection->get_name();+ field_values[i] = ref_field_values[0];++ continue;+ }+sort_order[i] = 1;if (sort_fields_std[i].order == sort_field_const::asc) {sort_order[i] = -1;@@ -4734,6 +4951,13 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint}}+void Index::populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>& geopoint_indices,+ std::vector<sort_by>& sort_fields_std,+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const {+ std::shared_lock lock(mutex);+ populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);+}+void Index::search_field(const uint8_t & field_id,const std::vector<token_t>& query_tokens,const uint32_t* exclude_token_ids,@@ -6168,6 +6392,11 @@ Option<uint32_t> Index::get_reference_doc_id_with_lock(const string& reference_hreturn Option<uint32_t>(sort_index.at(reference_helper_field_name)->at(seq_id));}+int64_t Index::reference_string_sort_score(const string &field_name, const uint32_t &seq_id) const {+ std::shared_lock lock(mutex);+ return str_sort_index.at(field_name)->rank(seq_id);+}+/*// https://stackoverflow.com/questions/924171/geo-fencing-point-inside-outside-polygon// NOTE: polygon and point should have been transformed with `transform_for_180th_meridian`diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex 13d8b2a6..7ae07ce7 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -1364,6 +1364,36 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ nlohmann::json model_config = R"({+ "model_name": "ts/e5-small"+ })"_json;+ auto query_embedding = TextEmbedderManager::get_instance().get_text_embedder(model_config).get()->Embed("natural products");+ std::string vec_string = "[";+ for (auto const& i : query_embedding.embedding) {+ vec_string += std::to_string(i);+ vec_string += ",";+ }+ vec_string[vec_string.size() - 1] = ']';++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},+ {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},+ {"include_fields", "product_name, $Customers(product_price)"},+ {"exclude_fields", ""}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(1, res_obj["found"].get<size_t>());+ ASSERT_EQ(1, res_obj["hits"].size());+ ASSERT_EQ(2, res_obj["hits"][0]["document"].size());+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));+ ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+// Hybrid search - Both text match and vector matchreq_params = {{"collection", "Products"},@@ -1383,7 +1413,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));- ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("rank_fusion_score"));+ ASSERT_NE(0, res_obj["hits"][0].at("text_match"));+ ASSERT_NE(0, res_obj["hits"][0].at("vector_distance"));// Hybrid search - Only vector matchreq_params = {@@ -1404,7 +1435,8 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference_SingleMatch) {ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));- ASSERT_NE(0, res_obj["hits"][0]["hybrid_search_info"].at("rank_fusion_score"));+ ASSERT_EQ(0, res_obj["hits"][0].at("text_match"));+ ASSERT_NE(0, res_obj["hits"][0].at("vector_distance"));// Infix searchreq_params = {@@ -1631,3 +1663,510 @@ TEST_F(CollectionJoinTest, CascadeDeletion) {ASSERT_EQ("product_b", res_obj["hits"][0]["document"].at("product_id"));ASSERT_EQ("user_a", res_obj["hits"][0]["document"].at("user_id"));}++TEST_F(CollectionJoinTest, SortByReference) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_id", "type": "string"},+ {"name": "product_name", "type": "string", "sort": true, "infix": true},+ {"name": "product_description", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "product_id": "product_a",+ "product_name": "shampoo",+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair."+ })"_json,+ R"({+ "product_id": "product_b",+ "product_name": "soap",+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients."+ })"_json+ };++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Customers",+ "fields": [+ {"name": "customer_id", "type": "string"},+ {"name": "customer_name", "type": "string"},+ {"name": "product_price", "type": "float"},+ {"name": "product_available", "type": "bool"},+ {"name": "product_location", "type": "geopoint"},+ {"name": "product_id", "type": "string", "reference": "Products.product_id", "sort": true}+ ]+ })"_json;+ documents = {+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 143,+ "product_available": true,+ "product_location": [48.872576479306765, 2.332291112241466],+ "product_id": "product_a"+ })"_json,+ R"({+ "customer_id": "customer_a",+ "customer_name": "Joe",+ "product_price": 73.5,+ "product_available": false,+ "product_location": [48.888286721920934, 2.342340862419206],+ "product_id": "product_b"+ })"_json,+ R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 75,+ "product_available": true,+ "product_location": [48.872576479306765, 2.332291112241466],+ "product_id": "product_a"+ })"_json,+ R"({+ "customer_id": "customer_b",+ "customer_name": "Dan",+ "product_price": 140,+ "product_available": false,+ "product_location": [48.888286721920934, 2.342340862419206],+ "product_id": "product_b"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ ASSERT_TRUE(add_op.ok());+ }++ std::map<std::string, std::string> req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$foo(product_price:asc"}+ };+ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Parameter `sort_by` is malformed.", search_op.error());++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(product_price)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Reference `sort_by` is malformed.", search_op.error());++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$foo(product_price:asc)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Referenced collection `foo` in `sort_by` not found.", search_op.error());++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(foo:asc)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Referenced collection `Customers`: Could not find a field named `foo` in the schema for sorting.",+ search_op.error());++ // Sort by reference numeric field+ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(product_price:asc)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ auto res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_b", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_a", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][1]["document"].at("product_price"));++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(product_price:desc)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));++ // Sort by reference string field+ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(product_id:asc)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));++ // Sort by reference optional filtering.+ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(_eval(product_available:true):asc)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_b", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_a", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][1]["document"].at("product_price"));++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(_eval(product_available:true):desc)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));++ // Text search+ req_params = {+ {"collection", "Products"},+ {"q", "s"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"sort_by", "$Customers(product_price:desc)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));++ // Phrase search+ req_params = {+ {"collection", "Products"},+ {"q", R"("our")"},+ {"query_by", "product_description"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));++ // Vector search+ req_params = {+ {"collection", "Products"},+ {"q", "natural products"},+ {"query_by", "embedding"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));+ auto product_a_score = res_obj["hits"][0].at("vector_distance");+ auto product_b_score = res_obj["hits"][1].at("vector_distance");+ // product_b is a better match for the vector query but sort_by overrides the order.+ ASSERT_TRUE(product_b_score < product_a_score);++ nlohmann::json model_config = R"({+ "model_name": "ts/e5-small"+ })"_json;+ auto query_embedding = TextEmbedderManager::get_instance().get_text_embedder(model_config).get()->Embed("natural products");+ std::string vec_string = "[";+ for (auto const& i : query_embedding.embedding) {+ vec_string += std::to_string(i);+ vec_string += ",";+ }+ vec_string[vec_string.size() - 1] = ']';++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));+ product_a_score = res_obj["hits"][0].at("vector_distance");+ product_b_score = res_obj["hits"][1].at("vector_distance");+ // product_b is a better match for the vector query but sort_by overrides the order.+ ASSERT_TRUE(product_b_score < product_a_score);++ // Hybrid search - Both text match and vector match+ req_params = {+ {"collection", "Products"},+ {"q", "soap"},+ {"query_by", "product_name, embedding"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));+ product_a_score = res_obj["hits"][0].at("text_match");+ product_b_score = res_obj["hits"][1].at("text_match");+ ASSERT_TRUE(product_b_score > product_a_score);+ product_a_score = res_obj["hits"][0].at("vector_distance");+ product_b_score = res_obj["hits"][1].at("vector_distance");+ ASSERT_TRUE(product_b_score < product_a_score);++ // Hybrid search - Only vector match+ req_params = {+ {"collection", "Products"},+ {"q", "natural products"},+ {"query_by", "product_name, embedding"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));+ product_a_score = res_obj["hits"][0].at("vector_distance");+ product_b_score = res_obj["hits"][1].at("vector_distance");+ // product_b is a better match for the vector query but sort_by overrides the order.+ ASSERT_TRUE(product_b_score < product_a_score);++ // Infix search+ req_params = {+ {"collection", "Products"},+ {"q", "p"},+ {"query_by", "product_name"},+ {"infix", "always"},+ {"filter_by", "$Customers(customer_id:=customer_a)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_TRUE(search_op.ok());++ res_obj = nlohmann::json::parse(json_res);+ ASSERT_EQ(2, res_obj["found"].get<size_t>());+ ASSERT_EQ(2, res_obj["hits"].size());+ ASSERT_EQ("product_a", res_obj["hits"][0]["document"].at("product_id"));+ ASSERT_EQ(143, res_obj["hits"][0]["document"].at("product_price"));+ ASSERT_EQ("product_b", res_obj["hits"][1]["document"].at("product_id"));+ ASSERT_EQ(73.5, res_obj["hits"][1]["document"].at("product_price"));++ // Multiple references - Wildcard search+ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(product_price: >0)"},+ {"sort_by", "$Customers(product_price:desc)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());++ // Multiple references - Text search+ req_params = {+ {"collection", "Products"},+ {"q", "s"},+ {"query_by", "product_name"},+ {"filter_by", "$Customers(product_price: >0)"},+ {"sort_by", "$Customers(product_price:desc)"},+ {"include_fields", "product_id, $Customers(product_price)"}+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());++ // Multiple references - Phrase search+ req_params = {+ {"collection", "Products"},+ {"q", R"("our")"},+ {"query_by", "product_description"},+ {"filter_by", "$Customers(product_price: >0)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());++ // Multiple references - Vector search+ req_params = {+ {"collection", "Products"},+ {"q", "natural products"},+ {"query_by", "embedding"},+ {"filter_by", "$Customers(product_price:>0)"},+ {"include_fields", "product_name, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());++ req_params = {+ {"collection", "Products"},+ {"q", "*"},+ {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},+ {"filter_by", "$Customers(product_price: >0)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());++ // Multiple references - Hybrid search+ req_params = {+ {"collection", "Products"},+ {"q", "soap"},+ {"query_by", "product_name, embedding"},+ {"filter_by", "$Customers(product_price: >0)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());++ req_params = {+ {"collection", "Products"},+ {"q", "natural products"},+ {"query_by", "product_name, embedding"},+ {"filter_by", "$Customers(product_price: >0)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());++ // Multiple references - Infix search+ req_params = {+ {"collection", "Products"},+ {"q", "p"},+ {"query_by", "product_name"},+ {"infix", "always"},+ {"filter_by", "$Customers(product_price: >0)"},+ {"include_fields", "product_id, $Customers(product_price)"},+ {"sort_by", "$Customers(product_price:desc)"},+ };+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Multiple references found to sort by on `Customers.product_price`.", search_op.error());+}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 3c18db44..5f63ba39 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -1110,6 +1110,23 @@ TEST_F(CollectionManagerTest, ParseSortByClause) {ASSERT_EQ("points", sort_fields[1].name);ASSERT_EQ("DESC", sort_fields[1].order);+ sort_fields.clear();+ sort_by_parsed = CollectionManager::parse_sort_by_str("points:desc,loc(24.56,10.45):ASC,"+ "$Customers(product_price:DESC)", sort_fields);+ ASSERT_TRUE(sort_by_parsed);+ ASSERT_EQ(3, sort_fields.size());+ ASSERT_EQ("points", sort_fields[0].name);+ ASSERT_EQ("DESC", sort_fields[0].order);+ ASSERT_EQ("loc(24.56,10.45)", sort_fields[1].name);+ ASSERT_EQ("ASC", sort_fields[1].order);+ ASSERT_EQ("$Customers(product_price:DESC)", sort_fields[2].name);++ sort_fields.clear();+ sort_by_parsed = CollectionManager::parse_sort_by_str("$foo( _eval(brand:nike && foo:bar):DESC,points:desc) ",+ sort_fields);+ ASSERT_TRUE(sort_by_parsed);+ ASSERT_EQ("$foo( _eval(brand:nike && foo:bar):DESC,points:desc)", sort_fields[0].name);+sort_fields.clear();sort_by_parsed = CollectionManager::parse_sort_by_str("", sort_fields);ASSERT_TRUE(sort_by_parsed);