Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support filter scoring.
Harpreet Sangar024dcc1b6867
diff --git a/include/field.h b/include/field.hindex ccdb8784..34664a62 100644--- a/include/field.h+++ b/include/field.h@@ -496,12 +496,14 @@ struct sort_by {};struct eval_t {- filter_node_t* filter_tree_root = nullptr;- uint32_t* ids = nullptr;- uint32_t size = 0;+ filter_node_t* filter_trees = nullptr;+ std::vector<uint32_t*> eval_ids_vec;+ std::vector<uint32_t> eval_ids_count_vec;+ std::vector<int64_t> scores;};std::string name;+ std::vector<std::string> eval_expressions;std::string order;// for text_match score bucketing@@ -523,6 +525,13 @@ struct sort_by {}+ sort_by(std::vector<std::string> eval_expressions, std::vector<int64_t> scores, std::string order):+ eval_expressions(std::move(eval_expressions)), order(std::move(order)), text_match_buckets(0), geopoint(0), exclude_radius(0),+ geo_precision(0), missing_values(normal) {+ name = sort_field_const::eval;+ eval.scores = std::move(scores);+ }+sort_by(const std::string &name, const std::string &order, uint32_t text_match_buckets, int64_t geopoint,uint32_t exclude_radius, uint32_t geo_precision) :name(name), order(order), text_match_buckets(text_match_buckets),@@ -535,6 +544,7 @@ struct sort_by {if (&other == this)return;name = other.name;+ eval_expressions = other.eval_expressions;order = other.order;text_match_buckets = other.text_match_buckets;geopoint = other.geopoint;@@ -547,6 +557,7 @@ struct sort_by {sort_by& operator=(const sort_by& other) {name = other.name;+ eval_expressions = other.eval_expressions;order = other.order;text_match_buckets = other.text_match_buckets;geopoint = other.geopoint;diff --git a/include/filter.h b/include/filter.hindex 01469d5e..486b7a5e 100644--- a/include/filter.h+++ b/include/filter.h@@ -67,7 +67,9 @@ struct filter_node_t {filter_node_t* left = nullptr;filter_node_t* right = nullptr;- filter_node_t(filter filter_exp)+ filter_node_t() = default;++ explicit filter_node_t(filter filter_exp): filter_exp(std::move(filter_exp)),isOperator(false),left(nullptr),@@ -85,4 +87,25 @@ struct filter_node_t {delete left;delete right;}++ filter_node_t& operator=(filter_node_t&& obj) noexcept {+ if (&obj == this) {+ return *this;+ }++ if (obj.isOperator) {+ isOperator = true;+ filter_operator = obj.filter_operator;+ left = obj.left;+ right = obj.right;++ obj.left = nullptr;+ obj.right = nullptr;+ } else {+ isOperator = false;+ filter_exp = obj.filter_exp;+ }++ return *this;+ }};diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.hindex 25f05ad2..8f4abb46 100644--- a/include/filter_result_iterator.h+++ b/include/filter_result_iterator.h@@ -210,6 +210,8 @@ public:/// iterator reaching it's end. (is_valid would be false in both these cases)uint32_t approx_filter_ids_length = 0;+ filter_result_iterator_t() = default;+explicit filter_result_iterator_t(uint32_t* ids, const uint32_t& ids_count);explicit filter_result_iterator_t(const std::string collection_name,diff --git a/include/index.h b/include/index.hindex a78cf783..c768c1e1 100644--- a/include/index.h+++ b/include/index.h@@ -957,7 +957,7 @@ public:std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,const std::vector<size_t>& geopoint_indices, uint32_t seq_id,const std::map<basic_string<char>, reference_filter_result_t>& references,- size_t filter_index,+ std::vector<uint32_t>& filter_indexes,int64_t max_field_match_score,int64_t* scores,int64_t& match_score_index, float vector_distance = 0,diff --git a/src/collection.cpp b/src/collection.cppindex 9eb782c3..f699edad 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -28,12 +28,10 @@ struct sort_fields_guard_t {~sort_fields_guard_t() {for(auto& sort_by_clause: sort_fields_std) {- delete sort_by_clause.eval.filter_tree_root;- if(sort_by_clause.eval.ids) {- delete [] sort_by_clause.eval.ids;- sort_by_clause.eval.ids = nullptr;- sort_by_clause.eval.size = 0;+ for (auto& eval_ids: sort_by_clause.eval.eval_ids_vec) {+ delete [] eval_ids;}+ delete [] sort_by_clause.eval.filter_trees;}}};@@ -774,7 +772,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<const bool is_vector_query,const bool is_group_by_query) const {- size_t num_sort_expressions = 0;+ uint32_t eval_sort_count = 0;for(size_t i = 0; i < sort_fields.size(); i++) {const sort_by& _sort_field = sort_fields[i];@@ -813,6 +811,37 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_fields_std.emplace_back(ref_sort_field_std);}+ continue;+ } else if (_sort_field.name == sort_field_const::eval) {+ sort_by sort_field_std(sort_field_const::eval, _sort_field.order);++ auto const& count = _sort_field.eval_expressions.size();+ sort_field_std.eval.filter_trees = new filter_node_t[count];+ std::unique_ptr<filter_node_t []> filter_trees_guard(sort_field_std.eval.filter_trees);++ for (uint32_t j = 0; j < count; j++) {+ auto const& filter_exp = _sort_field.eval_expressions[j];+ if (filter_exp.empty()) {+ return Option<bool>(400, "The eval expression in sort_by is empty.");+ }++ filter_node_t* filter_tree_root = nullptr;+ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,+ store, "", filter_tree_root);+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);++ if (!parse_filter_op.ok()) {+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");+ }++ sort_field_std.eval.filter_trees[j] = std::move(*filter_tree_root);+ }++ eval_sort_count++;+ sort_field_std.eval_expressions = _sort_field.eval_expressions;+ sort_field_std.eval.scores = _sort_field.eval.scores;+ sort_fields_std.emplace_back(sort_field_std);+ filter_trees_guard.release();continue;}@@ -843,23 +872,6 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_field_std.name = actual_field_name;sort_field_std.text_match_buckets = std::stoll(match_parts[1]);- } else if(actual_field_name == sort_field_const::eval) {- const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,- sort_field_std.name.size() - paran_start -- 2);- if(filter_exp.empty()) {- return Option<bool>(400, "The eval expression in sort_by is empty.");- }-- Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,- store, "", sort_field_std.eval.filter_tree_root);- if(!parse_filter_op.ok()) {- return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");- }-- sort_field_std.name = actual_field_name;- num_sort_expressions++;-} else {if(field_it == search_schema.end()) {std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";@@ -1063,7 +1075,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<return Option<bool>(422, message);}- if(num_sort_expressions > 1) {+ if(eval_sort_count > 1) {std::string message = "Only one sorting eval expression is allowed.";return Option<bool>(422, message);}@@ -1106,22 +1118,6 @@ Option<bool> Collection::validate_and_standardize_sort_fields_with_lock(const stsort_field_std.name = actual_field_name;sort_field_std.text_match_buckets = std::stoll(match_parts[1]);- } else if(actual_field_name == sort_field_const::eval) {- const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,- sort_field_std.name.size() - paran_start -- 2);- if(filter_exp.empty()) {- return Option<bool>(400, "The eval expression in sort_by is empty.");- }-- Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,- store, "", sort_field_std.eval.filter_tree_root);- if(!parse_filter_op.ok()) {- return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");- }-- sort_field_std.name = actual_field_name;-} else {if(field_it == search_schema.end()) {std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";@@ -1243,6 +1239,35 @@ Option<bool> Collection::validate_and_standardize_sort_fields_with_lock(const stsort_field_std.name = actual_field_name;}+ } else if (sort_field.name == sort_field_const::eval) {+ auto const& count = sort_field.eval_expressions.size();+ sort_field_std.eval.filter_trees = new filter_node_t[count];+ std::unique_ptr<filter_node_t []> filter_trees_guard(sort_field_std.eval.filter_trees);++ for (uint32_t j = 0; j < count; j++) {+ auto const& filter_exp = sort_field.eval_expressions[j];+ if (filter_exp.empty()) {+ return Option<bool>(400, "The eval expression in sort_by is empty.");+ }++ filter_node_t* filter_tree_root = nullptr;+ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,+ store, "", filter_tree_root);+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);++ if (!parse_filter_op.ok()) {+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");+ }++ sort_field_std.eval.filter_trees[j] = std::move(*filter_tree_root);+ }++ sort_field_std.name = sort_field.name;+ sort_field_std.eval_expressions = sort_field.eval_expressions;+ sort_field_std.eval.scores = sort_field.eval.scores;+ sort_fields_std.emplace_back(sort_field_std);+ filter_trees_guard.release();+ continue;}if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 537bd9f1..3d5c306d 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -652,11 +652,120 @@ AuthManager& CollectionManager::getAuthManager() {return auth_manager;}+bool parse_multi_eval(const std::string& sort_by_str, uint32_t& index, std::vector<sort_by>& sort_fields) {+ // FORMAT:+ // _eval([ (<expr_1>): <score_1>, (<expr_2>): <score_2> ]):<order>++ std::vector<std::string> eval_expressions;+ std::vector<std::int64_t> scores;+ while (true) {+ if (index >= sort_by_str.size()) {+ return false;+ } else if (sort_by_str[index] == ']') {+ break;+ }++ auto open_paren_pos = sort_by_str.find('(', index);+ if (open_paren_pos == std::string::npos) {+ return false;+ }+ index = open_paren_pos;+ std::string eval_expr = "(";+ int paren_count = 1;+ while (++index < sort_by_str.size() && paren_count > 0) {+ if (sort_by_str[index] == '(') {+ paren_count++;+ } else if (sort_by_str[index] == ')') {+ paren_count--;+ }+ eval_expr += sort_by_str[index];+ }++ // Removing outer parenthesis.+ eval_expr = eval_expr.substr(1, eval_expr.size() - 2);+ if (paren_count != 0 || index >= sort_by_str.size()) {+ return false;+ }++ while (sort_by_str[index] != ':' && ++index < sort_by_str.size());+ if (index >= sort_by_str.size()) {+ return false;+ }++ std::string score;+ while (++index < sort_by_str.size() && !(sort_by_str[index] == ',' || sort_by_str[index] == ']')) {+ score += sort_by_str[index];+ }+ StringUtils::trim(score);+ if (!StringUtils::is_int64_t(score)) {+ return false;+ }++ eval_expressions.emplace_back(eval_expr);+ scores.emplace_back(std::stoll(score));+ }++ while (++index < sort_by_str.size() && sort_by_str[index] != ':');+ if (index >= sort_by_str.size()) {+ return false;+ }++ std::string order_str;+ while (++index < sort_by_str.size() && sort_by_str[index] != ',') {+ order_str += sort_by_str[index];+ }+ StringUtils::trim(order_str);+ StringUtils::toupper(order_str);++ sort_fields.emplace_back(eval_expressions, scores, order_str);+ return true;+}++bool parse_eval(const std::string& sort_by_str, uint32_t& index, std::vector<sort_by>& sort_fields) {+ // FORMAT:+ // _eval(<expr>):<order>+ std::string eval_expr = "(";+ int paren_count = 1;+ while (++index < sort_by_str.size() && paren_count > 0) {+ if (sort_by_str[index] == '(') {+ paren_count++;+ } else if (sort_by_str[index] == ')') {+ paren_count--;+ }+ eval_expr += sort_by_str[index];+ }++ // Removing outer parenthesis.+ eval_expr = eval_expr.substr(1, eval_expr.size() - 2);++ if (paren_count != 0 || index >= sort_by_str.size()) {+ return false;+ }++ while (sort_by_str[index] != ':' && ++index < sort_by_str.size());+ if (index >= sort_by_str.size()) {+ return false;+ }++ std::string order_str;+ while (++index < sort_by_str.size() && sort_by_str[index] != ',') {+ order_str += sort_by_str[index];+ }+ StringUtils::trim(order_str);+ StringUtils::toupper(order_str);++ std::vector<std::string> eval_expressions = {eval_expr};+ std::vector<int64_t> scores = {1};+ sort_fields.emplace_back(eval_expressions, scores, order_str);++ return true;+}+bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<sort_by>& sort_fields) {std::string sort_field_expr;char prev_non_space_char = 'a';- for(size_t i=0; i < sort_by_str.size(); i++) {+ for(uint32_t i=0; i < sort_by_str.size(); i++) {if (sort_field_expr.empty()) {if (sort_by_str[i] == '$') {// Sort by reference field@@ -689,36 +798,15 @@ bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<sif (open_paren_pos == std::string::npos) {return false;}- sort_field_expr = sort_field_const::eval + "(";i = open_paren_pos;- int paren_count = 1;- while (++i < sort_by_str.size() && paren_count > 0) {- if (sort_by_str[i] == '(') {- paren_count++;- } else if (sort_by_str[i] == ')') {- paren_count--;- }- sort_field_expr += sort_by_str[i];- }- if (paren_count != 0 || i >= sort_by_str.size()) {- return false;- }+ while(sort_by_str[++i] == ' ');- while (sort_by_str[i] != ':' && ++i < sort_by_str.size());- if (i >= sort_by_str.size()) {+ auto result = sort_by_str[i] == '[' ? parse_multi_eval(sort_by_str, i, sort_fields) :+ parse_eval(sort_by_str, --i, sort_fields);+ if (!result) {return false;}-- std::string order_str;- while (++i < sort_by_str.size() && sort_by_str[i] != ',') {- order_str += sort_by_str[i];- }- StringUtils::trim(order_str);- StringUtils::toupper(order_str);-- sort_fields.emplace_back(sort_field_expr, order_str);- sort_field_expr = "";continue;}}diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 3eff372e..ed3bbfcd 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -1596,9 +1596,10 @@ filter_result_iterator_t::~filter_result_iterator_t() {delete right_it;}-filter_result_iterator_t &filter_result_iterator_t::operator=(filter_result_iterator_t &&obj) noexcept {- if (&obj == this)+filter_result_iterator_t& filter_result_iterator_t::operator=(filter_result_iterator_t&& obj) noexcept {+ if (&obj == this) {return *this;+ }// In case the filter was on string field.for(auto expanded_plist: expanded_plists) {diff --git a/src/index.cpp b/src/index.cppindex 63d1ced1..9c9299ec 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2478,7 +2478,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}}- std::vector<uint32_t> nearest_ids;+ std::vector<uint32_t> nearest_ids, filter_indexes;for (auto& dist_result : dist_results) {auto& seq_id = dist_result.second.seq_id;@@ -2507,8 +2507,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consint64_t match_score_index = -1;auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,- geopoint_indices, seq_id, references, 0, 0, scores,- match_score_index, vec_dist_score, collection_name);+ geopoint_indices, seq_id, references, filter_indexes,+ 0, scores, match_score_index, vec_dist_score,+ collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;}@@ -2847,7 +2848,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consresult->scores[result->match_score_index] = float_to_int64_t((1.0 / (i + 1)) * TEXT_MATCH_WEIGHT);}- std::vector<uint32_t> vec_search_ids; // list of IDs found only in vector search+ std::vector<uint32_t> vec_search_ids, filter_indexes; // list of IDs found only in vector searchfor(size_t res_index = 0; res_index < vec_results.size(); res_index++) {auto& vec_result = vec_results[res_index];@@ -2888,7 +2889,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consint64_t scores[3] = {0};auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,- geopoint_indices, seq_id, references, 0,+ geopoint_indices, seq_id, references, filter_indexes,match_score, scores, match_score_index,vec_result.second, collection_name);if (!compute_sort_scores_op.ok()) {@@ -2909,7 +2910,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consint64_t match_score_index = -1;auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,- geopoint_indices, seq_id, references, 0,+ geopoint_indices, seq_id, references, filter_indexes,match_score, scores, match_score_index,vec_result.second, collection_name);if (!compute_sort_scores_op.ok()) {@@ -3807,8 +3808,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokentoken_its.push_back(std::move(token_fields));}- std::vector<uint32_t> result_ids;- size_t filter_index = 0;+ std::vector<uint32_t> result_ids, filter_indexes;Option<bool> status(true);or_iterator_t::intersect(token_its, istate,@@ -3905,7 +3905,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokenint64_t match_score_index = -1;auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,- seq_id, references, filter_index, best_field_match_score,+ seq_id, references, filter_indexes, best_field_match_score,scores, match_score_index, 0, collection_name);if (!compute_sort_scores_op.ok()) {status = Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());@@ -4013,7 +4013,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,const std::vector<size_t>& geopoint_indices,uint32_t seq_id, const std::map<basic_string<char>, reference_filter_result_t>& references,- size_t filter_index, int64_t max_field_match_score, int64_t* scores,+ std::vector<uint32_t>& filter_indexes, int64_t max_field_match_score, int64_t* scores,int64_t& match_score_index, float vector_distance,const std::string& collection_name) const {@@ -4189,22 +4189,32 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,}}} else if(field_values[0] == &eval_sentinel_value) {- // Returns iterator to the first element that is >= to value or last if no such element is found.- bool found = false;- if (filter_index == 0 || filter_index < sort_fields[0].eval.size) {- size_t found_index = std::lower_bound(sort_fields[0].eval.ids + filter_index,- sort_fields[0].eval.ids + sort_fields[0].eval.size, seq_id) -- sort_fields[0].eval.ids;+ auto const& count = sort_fields[0].eval_expressions.size();+ if (filter_indexes.empty()) {+ filter_indexes = std::vector<uint32_t>(count, 0);+ }- if (found_index != sort_fields[0].eval.size && sort_fields[0].eval.ids[found_index] == seq_id) {- filter_index = found_index + 1;- found = true;+ bool found = false;+ uint32_t index = 0;+ auto const& eval = sort_fields[0].eval;+ for (; index < count; index++) {+ auto& filter_index = filter_indexes[index];+ auto const& eval_ids = eval.eval_ids_vec[index];+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];+ if (filter_index == 0 || filter_index < eval_ids_count) {+ // Returns iterator to the first element that is >= to value or last if no such element is found.+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ eval_ids;++ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ filter_index++;+ found = true;+ break;+ }}-- filter_index = found_index;}- scores[0] = int64_t(found);+ scores[0] = found ? eval.scores[index] : 0;} else if(field_values[0] == &vector_distance_sentinel_value) {scores[0] = float_to_int64_t(vector_distance);} else {@@ -4342,22 +4352,32 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,}}} else if(field_values[1] == &eval_sentinel_value) {- // Returns iterator to the first element that is >= to value or last if no such element is found.- bool found = false;- if (filter_index == 0 || filter_index < sort_fields[1].eval.size) {- size_t found_index = std::lower_bound(sort_fields[1].eval.ids + filter_index,- sort_fields[1].eval.ids + sort_fields[1].eval.size, seq_id) -- sort_fields[1].eval.ids;+ auto const& count = sort_fields[1].eval_expressions.size();+ if (filter_indexes.empty()) {+ filter_indexes = std::vector<uint32_t>(count, 0);+ }- if (found_index != sort_fields[1].eval.size && sort_fields[1].eval.ids[found_index] == seq_id) {- filter_index = found_index + 1;- found = true;+ bool found = false;+ uint32_t index = 0;+ auto const& eval = sort_fields[1].eval;+ for (; index < count; index++) {+ auto& filter_index = filter_indexes[index];+ auto const& eval_ids = eval.eval_ids_vec[index];+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];+ if (filter_index == 0 || filter_index < eval_ids_count) {+ // Returns iterator to the first element that is >= to value or last if no such element is found.+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ eval_ids;++ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ filter_index++;+ found = true;+ break;+ }}-- filter_index = found_index;}- scores[1] = int64_t(found);+ scores[1] = found ? eval.scores[index] : 0;} else if(field_values[1] == &vector_distance_sentinel_value) {scores[1] = float_to_int64_t(vector_distance);} else {@@ -4491,22 +4511,32 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,}}} else if(field_values[2] == &eval_sentinel_value) {- // Returns iterator to the first element that is >= to value or last if no such element is found.- bool found = false;- if (filter_index == 0 || filter_index < sort_fields[2].eval.size) {- size_t found_index = std::lower_bound(sort_fields[2].eval.ids + filter_index,- sort_fields[2].eval.ids + sort_fields[2].eval.size, seq_id) -- sort_fields[2].eval.ids;+ auto const& count = sort_fields[2].eval_expressions.size();+ if (filter_indexes.empty()) {+ filter_indexes = std::vector<uint32_t>(count, 0);+ }- if (found_index != sort_fields[2].eval.size && sort_fields[2].eval.ids[found_index] == seq_id) {- filter_index = found_index + 1;- found = true;+ bool found = false;+ uint32_t index = 0;+ auto const& eval = sort_fields[2].eval;+ for (; index < count; index++) {+ auto& filter_index = filter_indexes[index];+ auto const& eval_ids = eval.eval_ids_vec[index];+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];+ if (filter_index == 0 || filter_index < eval_ids_count) {+ // Returns iterator to the first element that is >= to value or last if no such element is found.+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -+ eval_ids;++ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {+ filter_index++;+ found = true;+ break;+ }}-- filter_index = found_index;}- scores[2] = int64_t(found);+ scores[2] = found ? eval.scores[index] : 0;} else if(field_values[2] == &vector_distance_sentinel_value) {scores[2] = float_to_int64_t(vector_distance);} else {@@ -4654,7 +4684,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::all_result_ids_len = filter_result_iterator->to_filter_id_array(all_result_ids);filter_result_iterator->reset();- size_t filter_index = 0;+ std::vector<uint32_t> filter_indexes;// populate topsterfor(size_t i = 0; i < std::min<size_t>(10000, all_result_ids_len); i++) {auto seq_id = filter_result_iterator->seq_id;@@ -4666,7 +4696,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::int64_t match_score_index = -1;auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,- seq_id, references, filter_index, match_score, scores,+ seq_id, references, filter_indexes, match_score, scores,match_score_index, 0, collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;@@ -4810,7 +4840,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::v}bool field_is_array = search_schema.at(the_fields[field_id].name).is_array();- size_t filter_index = 0;+ std::vector<uint32_t> filter_indexes;for(size_t i = 0; i < raw_infix_ids_length; i++) {auto seq_id = raw_infix_ids[i];@@ -4828,7 +4858,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vauto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values,geopoint_indices, seq_id, references,- filter_index, 100, scores, match_score_index,+ filter_indexes, 100, scores, match_score_index,0, collection_name);if (!compute_sort_scores_op.ok()) {return compute_sort_scores_op;@@ -5196,7 +5226,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootsearch_stop_us = parent_search_stop_ms;search_cutoff = parent_search_cutoff;- size_t filter_index = 0;+ std::vector<uint32_t> filter_indexes;for(size_t i = 0; i < batch_result->count; i++) {const uint32_t seq_id = batch_result->docs[i];@@ -5214,7 +5244,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootint64_t match_score_index = -1;auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,- seq_id, references, filter_index, 100, scores,+ seq_id, references, filter_indexes, 100, scores,match_score_index, 0, collection_name);if (!compute_sort_scores_op.ok()) {compute_sort_score_status = new Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());@@ -5325,12 +5355,20 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopointfield_values[i] = &seq_id_sentinel_value;} else if (sort_fields_std[i].name == sort_field_const::eval) {field_values[i] = &eval_sentinel_value;- auto filter_result_iterator = filter_result_iterator_t("", this, sort_fields_std[i].eval.filter_tree_root);- auto filter_init_op = filter_result_iterator.init_status();- if (!filter_init_op.ok()) {- return;+ auto& eval_exp = sort_fields_std[i].eval;+ auto count = sort_fields_std[i].eval_expressions.size();+ for (uint32_t j = 0; j < count; j++) {+ auto filter_result_iterator = filter_result_iterator_t("", this, &eval_exp.filter_trees[j]);+ auto filter_init_op = filter_result_iterator.init_status();+ if (!filter_init_op.ok()) {+ return;+ }+ uint32_t* eval_ids = nullptr;+ auto eval_ids_count = filter_result_iterator.to_filter_id_array(eval_ids);++ eval_exp.eval_ids_vec.push_back(eval_ids);+ eval_exp.eval_ids_count_vec.push_back(eval_ids_count);}- sort_fields_std[i].eval.size = filter_result_iterator.to_filter_id_array(sort_fields_std[i].eval.ids);} else if(sort_fields_std[i].name == sort_field_const::vector_distance) {field_values[i] = &vector_distance_sentinel_value;} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex e0f382fb..421ca276 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -1177,11 +1177,27 @@ TEST_F(CollectionManagerTest, ParseSortByClause) {sort_fields.clear();sort_by_parsed = CollectionManager::parse_sort_by_str("_eval(brand:nike && foo:bar):DESC,points:desc ", sort_fields);ASSERT_TRUE(sort_by_parsed);- ASSERT_EQ("_eval(brand:nike && foo:bar)", sort_fields[0].name);+ ASSERT_EQ("_eval", sort_fields[0].name);+ ASSERT_FALSE(sort_fields[0].eval_expressions.empty());+ ASSERT_EQ("brand:nike && foo:bar", sort_fields[0].eval_expressions[0]);+ ASSERT_EQ(1, sort_fields[0].eval.scores.size());+ ASSERT_EQ(1, sort_fields[0].eval.scores[0]);ASSERT_EQ("DESC", sort_fields[0].order);ASSERT_EQ("points", sort_fields[1].name);ASSERT_EQ("DESC", sort_fields[1].order);+ sort_fields.clear();+ sort_by_parsed = CollectionManager::parse_sort_by_str("_eval([(brand:nike || brand:air):3, (brand:adidas):2]):DESC", sort_fields);+ ASSERT_TRUE(sort_by_parsed);+ ASSERT_EQ("_eval", sort_fields[0].name);+ ASSERT_EQ(2, sort_fields[0].eval_expressions.size());+ ASSERT_EQ("brand:nike || brand:air", sort_fields[0].eval_expressions[0]);+ ASSERT_EQ("brand:adidas", sort_fields[0].eval_expressions[1]);+ ASSERT_EQ(2, sort_fields[0].eval.scores.size());+ ASSERT_EQ(3, sort_fields[0].eval.scores[0]);+ ASSERT_EQ(2, sort_fields[0].eval.scores[1]);+ ASSERT_EQ("DESC", sort_fields[0].order);+sort_fields.clear();sort_by_parsed = CollectionManager::parse_sort_by_str("points:desc,loc(24.56,10.45):ASC,""$Customers(product_price:DESC)", sort_fields);diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 4ae097d0..3b729c26 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -2007,7 +2007,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {}std::vector<sort_by> sort_fields = {- sort_by("_eval(brand:nike)", "DESC"),+ sort_by({"brand:nike"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2021,7 +2021,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {// compound querysort_fields = {- sort_by("_eval(brand:nike && points:0)", "DESC"),+ sort_by({"brand:nike && points:0"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2033,9 +2033,22 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());}+ sort_fields = {+ sort_by({"brand:nike", "points:1"}, {2, 1}, "DESC"),+ sort_by("points", "DESC"),+ };++ results = coll1->search("*", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();+ ASSERT_EQ(5, results["hits"].size());++ expected_ids = {"3", "0", "1", "4", "2"};+ for(size_t i = 0; i < expected_ids.size(); i++) {+ ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());+ }+// when no results are found for eval querysort_fields = {- sort_by("_eval(brand:foobar)", "DESC"),+ sort_by({"brand:foobar"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2049,7 +2062,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {// bad syntax for eval querysort_fields = {- sort_by("_eval(brandnike || points:0)", "DESC"),+ sort_by({"brandnike || points:0"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2058,13 +2071,20 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {ASSERT_EQ("Error parsing eval expression in sort_by clause.", res_op.error());// when eval condition is empty- sort_fields = {- sort_by("_eval()", "DESC"),- sort_by("points", "DESC"),++ std::map<std::string, std::string> req_params = {+ {"collection", "coll1"},+ {"q", "*"},+ {"query_by", "title"},+ {"sort_by", "_eval():desc"},};- res_op = coll1->search("*", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10);- ASSERT_FALSE(res_op.ok());- ASSERT_EQ("The eval expression in sort_by is empty.", res_op.error());+ nlohmann::json embedded_params;+ std::string json_res;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("The eval expression in sort_by is empty.", search_op.error());// more bad syntax!sort_fields = {@@ -2078,8 +2098,8 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {// don't allow multiple sorting eval expressionssort_fields = {- sort_by("_eval(brand: nike || points:0)", "DESC"),- sort_by("_eval(brand: nike || points:0)", "DESC"),+ sort_by({"brand: nike || points:0"}, {1}, "DESC"),+ sort_by({"brand: nike || points:0"}, {1}, "DESC"),};res_op = coll1->search("*", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10);@@ -2114,7 +2134,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {}std::vector<sort_by> sort_fields = {- sort_by("_eval(brand:nike)", "DESC"),+ sort_by({"brand:nike"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2128,7 +2148,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {// compound querysort_fields = {- sort_by("_eval(brand:nike && points:0)", "DESC"),+ sort_by({"brand:nike && points:0"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2158,7 +2178,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {// when no results are found for eval querysort_fields = {- sort_by("_eval(brand:foobar)", "DESC"),+ sort_by({"brand:foobar"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2172,7 +2192,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {// bad syntax for eval querysort_fields = {- sort_by("_eval(brandnike || points:0)", "DESC"),+ sort_by({"brandnike || points:0"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2237,7 +2257,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSecondThirdParams) {std::vector<sort_by> sort_fields = {sort_by("val", "DESC"),- sort_by("_eval(brand:nike)", "DESC"),+ sort_by({"brand:nike"}, {1}, "DESC"),sort_by("points", "DESC"),};@@ -2253,7 +2273,32 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSecondThirdParams) {sort_fields = {sort_by("val", "DESC"),sort_by("val", "DESC"),- sort_by("_eval(brand:nike)", "DESC"),+ sort_by({"brand:nike"}, {1}, "DESC"),+ };++ results = coll1->search("title", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();+ ASSERT_EQ(5, results["hits"].size());+ for(size_t i = 0; i < expected_ids.size(); i++) {+ ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());+ }++ sort_fields = {+ sort_by("val", "DESC"),+ sort_by({"brand:adidas", "brand:nike"}, {2, 1}, "DESC"),+ sort_by("points", "DESC"),+ };+ expected_ids = {"4", "2", "1","3", "0"};++ results = coll1->search("title", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();+ ASSERT_EQ(5, results["hits"].size());+ for(size_t i = 0; i < expected_ids.size(); i++) {+ ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());+ }++ sort_fields = {+ sort_by("val", "DESC"),+ sort_by("val", "DESC"),+ sort_by({"brand:adidas", "brand:nike"}, {2, 1}, "DESC"),};results = coll1->search("title", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();