Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support filter scoring.

Harpreet Sangar024dcc1b6867
diff --git a/include/field.h b/include/field.h
index ccdb8784..34664a62 100644
--- a/include/field.h
+++ b/include/field.h
@@ -496,12 +496,14 @@ struct sort_by {
};
struct eval_t {
- filter_node_t* filter_tree_root = nullptr;
- uint32_t* ids = nullptr;
- uint32_t size = 0;
+ filter_node_t* filter_trees = nullptr;
+ std::vector<uint32_t*> eval_ids_vec;
+ std::vector<uint32_t> eval_ids_count_vec;
+ std::vector<int64_t> scores;
};
std::string name;
+ std::vector<std::string> eval_expressions;
std::string order;
// for text_match score bucketing
@@ -523,6 +525,13 @@ struct sort_by {
}
+ sort_by(std::vector<std::string> eval_expressions, std::vector<int64_t> scores, std::string order):
+ eval_expressions(std::move(eval_expressions)), order(std::move(order)), text_match_buckets(0), geopoint(0), exclude_radius(0),
+ geo_precision(0), missing_values(normal) {
+ name = sort_field_const::eval;
+ eval.scores = std::move(scores);
+ }
+
sort_by(const std::string &name, const std::string &order, uint32_t text_match_buckets, int64_t geopoint,
uint32_t exclude_radius, uint32_t geo_precision) :
name(name), order(order), text_match_buckets(text_match_buckets),
@@ -535,6 +544,7 @@ struct sort_by {
if (&other == this)
return;
name = other.name;
+ eval_expressions = other.eval_expressions;
order = other.order;
text_match_buckets = other.text_match_buckets;
geopoint = other.geopoint;
@@ -547,6 +557,7 @@ struct sort_by {
sort_by& operator=(const sort_by& other) {
name = other.name;
+ eval_expressions = other.eval_expressions;
order = other.order;
text_match_buckets = other.text_match_buckets;
geopoint = other.geopoint;
diff --git a/include/filter.h b/include/filter.h
index 01469d5e..486b7a5e 100644
--- a/include/filter.h
+++ b/include/filter.h
@@ -67,7 +67,9 @@ struct filter_node_t {
filter_node_t* left = nullptr;
filter_node_t* right = nullptr;
- filter_node_t(filter filter_exp)
+ filter_node_t() = default;
+
+ explicit filter_node_t(filter filter_exp)
: filter_exp(std::move(filter_exp)),
isOperator(false),
left(nullptr),
@@ -85,4 +87,25 @@ struct filter_node_t {
delete left;
delete right;
}
+
+ filter_node_t& operator=(filter_node_t&& obj) noexcept {
+ if (&obj == this) {
+ return *this;
+ }
+
+ if (obj.isOperator) {
+ isOperator = true;
+ filter_operator = obj.filter_operator;
+ left = obj.left;
+ right = obj.right;
+
+ obj.left = nullptr;
+ obj.right = nullptr;
+ } else {
+ isOperator = false;
+ filter_exp = obj.filter_exp;
+ }
+
+ return *this;
+ }
};
diff --git a/include/filter_result_iterator.h b/include/filter_result_iterator.h
index 25f05ad2..8f4abb46 100644
--- a/include/filter_result_iterator.h
+++ b/include/filter_result_iterator.h
@@ -210,6 +210,8 @@ public:
/// iterator reaching it's end. (is_valid would be false in both these cases)
uint32_t approx_filter_ids_length = 0;
+ filter_result_iterator_t() = default;
+
explicit filter_result_iterator_t(uint32_t* ids, const uint32_t& ids_count);
explicit filter_result_iterator_t(const std::string collection_name,
diff --git a/include/index.h b/include/index.h
index a78cf783..c768c1e1 100644
--- a/include/index.h
+++ b/include/index.h
@@ -957,7 +957,7 @@ public:
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices, uint32_t seq_id,
const std::map<basic_string<char>, reference_filter_result_t>& references,
- size_t filter_index,
+ std::vector<uint32_t>& filter_indexes,
int64_t max_field_match_score,
int64_t* scores,
int64_t& match_score_index, float vector_distance = 0,
diff --git a/src/collection.cpp b/src/collection.cpp
index 9eb782c3..f699edad 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -28,12 +28,10 @@ struct sort_fields_guard_t {
~sort_fields_guard_t() {
for(auto& sort_by_clause: sort_fields_std) {
- delete sort_by_clause.eval.filter_tree_root;
- if(sort_by_clause.eval.ids) {
- delete [] sort_by_clause.eval.ids;
- sort_by_clause.eval.ids = nullptr;
- sort_by_clause.eval.size = 0;
+ for (auto& eval_ids: sort_by_clause.eval.eval_ids_vec) {
+ delete [] eval_ids;
}
+ delete [] sort_by_clause.eval.filter_trees;
}
}
};
@@ -774,7 +772,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
const bool is_vector_query,
const bool is_group_by_query) const {
- size_t num_sort_expressions = 0;
+ uint32_t eval_sort_count = 0;
for(size_t i = 0; i < sort_fields.size(); i++) {
const sort_by& _sort_field = sort_fields[i];
@@ -813,6 +811,37 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
sort_fields_std.emplace_back(ref_sort_field_std);
}
+ continue;
+ } else if (_sort_field.name == sort_field_const::eval) {
+ sort_by sort_field_std(sort_field_const::eval, _sort_field.order);
+
+ auto const& count = _sort_field.eval_expressions.size();
+ sort_field_std.eval.filter_trees = new filter_node_t[count];
+ std::unique_ptr<filter_node_t []> filter_trees_guard(sort_field_std.eval.filter_trees);
+
+ for (uint32_t j = 0; j < count; j++) {
+ auto const& filter_exp = _sort_field.eval_expressions[j];
+ if (filter_exp.empty()) {
+ return Option<bool>(400, "The eval expression in sort_by is empty.");
+ }
+
+ filter_node_t* filter_tree_root = nullptr;
+ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,
+ store, "", filter_tree_root);
+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
+
+ if (!parse_filter_op.ok()) {
+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");
+ }
+
+ sort_field_std.eval.filter_trees[j] = std::move(*filter_tree_root);
+ }
+
+ eval_sort_count++;
+ sort_field_std.eval_expressions = _sort_field.eval_expressions;
+ sort_field_std.eval.scores = _sort_field.eval.scores;
+ sort_fields_std.emplace_back(sort_field_std);
+ filter_trees_guard.release();
continue;
}
@@ -843,23 +872,6 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
sort_field_std.name = actual_field_name;
sort_field_std.text_match_buckets = std::stoll(match_parts[1]);
- } else if(actual_field_name == sort_field_const::eval) {
- const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,
- sort_field_std.name.size() - paran_start -
- 2);
- if(filter_exp.empty()) {
- return Option<bool>(400, "The eval expression in sort_by is empty.");
- }
-
- Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,
- store, "", sort_field_std.eval.filter_tree_root);
- if(!parse_filter_op.ok()) {
- return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");
- }
-
- sort_field_std.name = actual_field_name;
- num_sort_expressions++;
-
} else {
if(field_it == search_schema.end()) {
std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";
@@ -1063,7 +1075,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
return Option<bool>(422, message);
}
- if(num_sort_expressions > 1) {
+ if(eval_sort_count > 1) {
std::string message = "Only one sorting eval expression is allowed.";
return Option<bool>(422, message);
}
@@ -1106,22 +1118,6 @@ Option<bool> Collection::validate_and_standardize_sort_fields_with_lock(const st
sort_field_std.name = actual_field_name;
sort_field_std.text_match_buckets = std::stoll(match_parts[1]);
- } else if(actual_field_name == sort_field_const::eval) {
- const std::string& filter_exp = sort_field_std.name.substr(paran_start + 1,
- sort_field_std.name.size() - paran_start -
- 2);
- if(filter_exp.empty()) {
- return Option<bool>(400, "The eval expression in sort_by is empty.");
- }
-
- Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,
- store, "", sort_field_std.eval.filter_tree_root);
- if(!parse_filter_op.ok()) {
- return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");
- }
-
- sort_field_std.name = actual_field_name;
-
} else {
if(field_it == search_schema.end()) {
std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";
@@ -1243,6 +1239,35 @@ Option<bool> Collection::validate_and_standardize_sort_fields_with_lock(const st
sort_field_std.name = actual_field_name;
}
+ } else if (sort_field.name == sort_field_const::eval) {
+ auto const& count = sort_field.eval_expressions.size();
+ sort_field_std.eval.filter_trees = new filter_node_t[count];
+ std::unique_ptr<filter_node_t []> filter_trees_guard(sort_field_std.eval.filter_trees);
+
+ for (uint32_t j = 0; j < count; j++) {
+ auto const& filter_exp = sort_field.eval_expressions[j];
+ if (filter_exp.empty()) {
+ return Option<bool>(400, "The eval expression in sort_by is empty.");
+ }
+
+ filter_node_t* filter_tree_root = nullptr;
+ Option<bool> parse_filter_op = filter::parse_filter_query(filter_exp, search_schema,
+ store, "", filter_tree_root);
+ std::unique_ptr<filter_node_t> filter_tree_root_guard(filter_tree_root);
+
+ if (!parse_filter_op.ok()) {
+ return Option<bool>(parse_filter_op.code(), "Error parsing eval expression in sort_by clause.");
+ }
+
+ sort_field_std.eval.filter_trees[j] = std::move(*filter_tree_root);
+ }
+
+ sort_field_std.name = sort_field.name;
+ sort_field_std.eval_expressions = sort_field.eval_expressions;
+ sort_field_std.eval.scores = sort_field.eval.scores;
+ sort_fields_std.emplace_back(sort_field_std);
+ filter_trees_guard.release();
+ continue;
}
if (sort_field_std.name != sort_field_const::text_match && sort_field_std.name != sort_field_const::eval &&
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 537bd9f1..3d5c306d 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -652,11 +652,120 @@ AuthManager& CollectionManager::getAuthManager() {
return auth_manager;
}
+bool parse_multi_eval(const std::string& sort_by_str, uint32_t& index, std::vector<sort_by>& sort_fields) {
+ // FORMAT:
+ // _eval([ (<expr_1>): <score_1>, (<expr_2>): <score_2> ]):<order>
+
+ std::vector<std::string> eval_expressions;
+ std::vector<std::int64_t> scores;
+ while (true) {
+ if (index >= sort_by_str.size()) {
+ return false;
+ } else if (sort_by_str[index] == ']') {
+ break;
+ }
+
+ auto open_paren_pos = sort_by_str.find('(', index);
+ if (open_paren_pos == std::string::npos) {
+ return false;
+ }
+ index = open_paren_pos;
+ std::string eval_expr = "(";
+ int paren_count = 1;
+ while (++index < sort_by_str.size() && paren_count > 0) {
+ if (sort_by_str[index] == '(') {
+ paren_count++;
+ } else if (sort_by_str[index] == ')') {
+ paren_count--;
+ }
+ eval_expr += sort_by_str[index];
+ }
+
+ // Removing outer parenthesis.
+ eval_expr = eval_expr.substr(1, eval_expr.size() - 2);
+ if (paren_count != 0 || index >= sort_by_str.size()) {
+ return false;
+ }
+
+ while (sort_by_str[index] != ':' && ++index < sort_by_str.size());
+ if (index >= sort_by_str.size()) {
+ return false;
+ }
+
+ std::string score;
+ while (++index < sort_by_str.size() && !(sort_by_str[index] == ',' || sort_by_str[index] == ']')) {
+ score += sort_by_str[index];
+ }
+ StringUtils::trim(score);
+ if (!StringUtils::is_int64_t(score)) {
+ return false;
+ }
+
+ eval_expressions.emplace_back(eval_expr);
+ scores.emplace_back(std::stoll(score));
+ }
+
+ while (++index < sort_by_str.size() && sort_by_str[index] != ':');
+ if (index >= sort_by_str.size()) {
+ return false;
+ }
+
+ std::string order_str;
+ while (++index < sort_by_str.size() && sort_by_str[index] != ',') {
+ order_str += sort_by_str[index];
+ }
+ StringUtils::trim(order_str);
+ StringUtils::toupper(order_str);
+
+ sort_fields.emplace_back(eval_expressions, scores, order_str);
+ return true;
+}
+
+bool parse_eval(const std::string& sort_by_str, uint32_t& index, std::vector<sort_by>& sort_fields) {
+ // FORMAT:
+ // _eval(<expr>):<order>
+ std::string eval_expr = "(";
+ int paren_count = 1;
+ while (++index < sort_by_str.size() && paren_count > 0) {
+ if (sort_by_str[index] == '(') {
+ paren_count++;
+ } else if (sort_by_str[index] == ')') {
+ paren_count--;
+ }
+ eval_expr += sort_by_str[index];
+ }
+
+ // Removing outer parenthesis.
+ eval_expr = eval_expr.substr(1, eval_expr.size() - 2);
+
+ if (paren_count != 0 || index >= sort_by_str.size()) {
+ return false;
+ }
+
+ while (sort_by_str[index] != ':' && ++index < sort_by_str.size());
+ if (index >= sort_by_str.size()) {
+ return false;
+ }
+
+ std::string order_str;
+ while (++index < sort_by_str.size() && sort_by_str[index] != ',') {
+ order_str += sort_by_str[index];
+ }
+ StringUtils::trim(order_str);
+ StringUtils::toupper(order_str);
+
+ std::vector<std::string> eval_expressions = {eval_expr};
+ std::vector<int64_t> scores = {1};
+ sort_fields.emplace_back(eval_expressions, scores, order_str);
+
+ return true;
+}
+
bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<sort_by>& sort_fields) {
std::string sort_field_expr;
char prev_non_space_char = 'a';
- for(size_t i=0; i < sort_by_str.size(); i++) {
+ for(uint32_t i=0; i < sort_by_str.size(); i++) {
if (sort_field_expr.empty()) {
if (sort_by_str[i] == '$') {
// Sort by reference field
@@ -689,36 +798,15 @@ bool CollectionManager::parse_sort_by_str(std::string sort_by_str, std::vector<s
if (open_paren_pos == std::string::npos) {
return false;
}
- sort_field_expr = sort_field_const::eval + "(";
i = open_paren_pos;
- int paren_count = 1;
- while (++i < sort_by_str.size() && paren_count > 0) {
- if (sort_by_str[i] == '(') {
- paren_count++;
- } else if (sort_by_str[i] == ')') {
- paren_count--;
- }
- sort_field_expr += sort_by_str[i];
- }
- if (paren_count != 0 || i >= sort_by_str.size()) {
- return false;
- }
+ while(sort_by_str[++i] == ' ');
- while (sort_by_str[i] != ':' && ++i < sort_by_str.size());
- if (i >= sort_by_str.size()) {
+ auto result = sort_by_str[i] == '[' ? parse_multi_eval(sort_by_str, i, sort_fields) :
+ parse_eval(sort_by_str, --i, sort_fields);
+ if (!result) {
return false;
}
-
- std::string order_str;
- while (++i < sort_by_str.size() && sort_by_str[i] != ',') {
- order_str += sort_by_str[i];
- }
- StringUtils::trim(order_str);
- StringUtils::toupper(order_str);
-
- sort_fields.emplace_back(sort_field_expr, order_str);
- sort_field_expr = "";
continue;
}
}
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cpp
index 3eff372e..ed3bbfcd 100644
--- a/src/filter_result_iterator.cpp
+++ b/src/filter_result_iterator.cpp
@@ -1596,9 +1596,10 @@ filter_result_iterator_t::~filter_result_iterator_t() {
delete right_it;
}
-filter_result_iterator_t &filter_result_iterator_t::operator=(filter_result_iterator_t &&obj) noexcept {
- if (&obj == this)
+filter_result_iterator_t& filter_result_iterator_t::operator=(filter_result_iterator_t&& obj) noexcept {
+ if (&obj == this) {
return *this;
+ }
// In case the filter was on string field.
for(auto expanded_plist: expanded_plists) {
diff --git a/src/index.cpp b/src/index.cpp
index 63d1ced1..9c9299ec 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2478,7 +2478,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
}
- std::vector<uint32_t> nearest_ids;
+ std::vector<uint32_t> nearest_ids, filter_indexes;
for (auto& dist_result : dist_results) {
auto& seq_id = dist_result.second.seq_id;
@@ -2507,8 +2507,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
int64_t match_score_index = -1;
auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
- geopoint_indices, seq_id, references, 0, 0, scores,
- match_score_index, vec_dist_score, collection_name);
+ geopoint_indices, seq_id, references, filter_indexes,
+ 0, scores, match_score_index, vec_dist_score,
+ collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
}
@@ -2847,7 +2848,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
result->scores[result->match_score_index] = float_to_int64_t((1.0 / (i + 1)) * TEXT_MATCH_WEIGHT);
}
- std::vector<uint32_t> vec_search_ids; // list of IDs found only in vector search
+ std::vector<uint32_t> vec_search_ids, filter_indexes; // list of IDs found only in vector search
for(size_t res_index = 0; res_index < vec_results.size(); res_index++) {
auto& vec_result = vec_results[res_index];
@@ -2888,7 +2889,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
int64_t scores[3] = {0};
auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
- geopoint_indices, seq_id, references, 0,
+ geopoint_indices, seq_id, references, filter_indexes,
match_score, scores, match_score_index,
vec_result.second, collection_name);
if (!compute_sort_scores_op.ok()) {
@@ -2909,7 +2910,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
int64_t match_score_index = -1;
auto compute_sort_scores_op = compute_sort_scores(sort_fields_std, sort_order, field_values,
- geopoint_indices, seq_id, references, 0,
+ geopoint_indices, seq_id, references, filter_indexes,
match_score, scores, match_score_index,
vec_result.second, collection_name);
if (!compute_sort_scores_op.ok()) {
@@ -3807,8 +3808,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
token_its.push_back(std::move(token_fields));
}
- std::vector<uint32_t> result_ids;
- size_t filter_index = 0;
+ std::vector<uint32_t> result_ids, filter_indexes;
Option<bool> status(true);
or_iterator_t::intersect(token_its, istate,
@@ -3905,7 +3905,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token
int64_t match_score_index = -1;
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
- seq_id, references, filter_index, best_field_match_score,
+ seq_id, references, filter_indexes, best_field_match_score,
scores, match_score_index, 0, collection_name);
if (!compute_sort_scores_op.ok()) {
status = Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());
@@ -4013,7 +4013,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
uint32_t seq_id, const std::map<basic_string<char>, reference_filter_result_t>& references,
- size_t filter_index, int64_t max_field_match_score, int64_t* scores,
+ std::vector<uint32_t>& filter_indexes, int64_t max_field_match_score, int64_t* scores,
int64_t& match_score_index, float vector_distance,
const std::string& collection_name) const {
@@ -4189,22 +4189,32 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
}
}
} else if(field_values[0] == &eval_sentinel_value) {
- // Returns iterator to the first element that is >= to value or last if no such element is found.
- bool found = false;
- if (filter_index == 0 || filter_index < sort_fields[0].eval.size) {
- size_t found_index = std::lower_bound(sort_fields[0].eval.ids + filter_index,
- sort_fields[0].eval.ids + sort_fields[0].eval.size, seq_id) -
- sort_fields[0].eval.ids;
+ auto const& count = sort_fields[0].eval_expressions.size();
+ if (filter_indexes.empty()) {
+ filter_indexes = std::vector<uint32_t>(count, 0);
+ }
- if (found_index != sort_fields[0].eval.size && sort_fields[0].eval.ids[found_index] == seq_id) {
- filter_index = found_index + 1;
- found = true;
+ bool found = false;
+ uint32_t index = 0;
+ auto const& eval = sort_fields[0].eval;
+ for (; index < count; index++) {
+ auto& filter_index = filter_indexes[index];
+ auto const& eval_ids = eval.eval_ids_vec[index];
+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];
+ if (filter_index == 0 || filter_index < eval_ids_count) {
+ // Returns iterator to the first element that is >= to value or last if no such element is found.
+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -
+ eval_ids;
+
+ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {
+ filter_index++;
+ found = true;
+ break;
+ }
}
-
- filter_index = found_index;
}
- scores[0] = int64_t(found);
+ scores[0] = found ? eval.scores[index] : 0;
} else if(field_values[0] == &vector_distance_sentinel_value) {
scores[0] = float_to_int64_t(vector_distance);
} else {
@@ -4342,22 +4352,32 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
}
}
} else if(field_values[1] == &eval_sentinel_value) {
- // Returns iterator to the first element that is >= to value or last if no such element is found.
- bool found = false;
- if (filter_index == 0 || filter_index < sort_fields[1].eval.size) {
- size_t found_index = std::lower_bound(sort_fields[1].eval.ids + filter_index,
- sort_fields[1].eval.ids + sort_fields[1].eval.size, seq_id) -
- sort_fields[1].eval.ids;
+ auto const& count = sort_fields[1].eval_expressions.size();
+ if (filter_indexes.empty()) {
+ filter_indexes = std::vector<uint32_t>(count, 0);
+ }
- if (found_index != sort_fields[1].eval.size && sort_fields[1].eval.ids[found_index] == seq_id) {
- filter_index = found_index + 1;
- found = true;
+ bool found = false;
+ uint32_t index = 0;
+ auto const& eval = sort_fields[1].eval;
+ for (; index < count; index++) {
+ auto& filter_index = filter_indexes[index];
+ auto const& eval_ids = eval.eval_ids_vec[index];
+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];
+ if (filter_index == 0 || filter_index < eval_ids_count) {
+ // Returns iterator to the first element that is >= to value or last if no such element is found.
+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -
+ eval_ids;
+
+ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {
+ filter_index++;
+ found = true;
+ break;
+ }
}
-
- filter_index = found_index;
}
- scores[1] = int64_t(found);
+ scores[1] = found ? eval.scores[index] : 0;
} else if(field_values[1] == &vector_distance_sentinel_value) {
scores[1] = float_to_int64_t(vector_distance);
} else {
@@ -4491,22 +4511,32 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,
}
}
} else if(field_values[2] == &eval_sentinel_value) {
- // Returns iterator to the first element that is >= to value or last if no such element is found.
- bool found = false;
- if (filter_index == 0 || filter_index < sort_fields[2].eval.size) {
- size_t found_index = std::lower_bound(sort_fields[2].eval.ids + filter_index,
- sort_fields[2].eval.ids + sort_fields[2].eval.size, seq_id) -
- sort_fields[2].eval.ids;
+ auto const& count = sort_fields[2].eval_expressions.size();
+ if (filter_indexes.empty()) {
+ filter_indexes = std::vector<uint32_t>(count, 0);
+ }
- if (found_index != sort_fields[2].eval.size && sort_fields[2].eval.ids[found_index] == seq_id) {
- filter_index = found_index + 1;
- found = true;
+ bool found = false;
+ uint32_t index = 0;
+ auto const& eval = sort_fields[2].eval;
+ for (; index < count; index++) {
+ auto& filter_index = filter_indexes[index];
+ auto const& eval_ids = eval.eval_ids_vec[index];
+ auto const& eval_ids_count = eval.eval_ids_count_vec[index];
+ if (filter_index == 0 || filter_index < eval_ids_count) {
+ // Returns iterator to the first element that is >= to value or last if no such element is found.
+ filter_index = std::lower_bound(eval_ids + filter_index, eval_ids + eval_ids_count, seq_id) -
+ eval_ids;
+
+ if (filter_index < eval_ids_count && eval_ids[filter_index] == seq_id) {
+ filter_index++;
+ found = true;
+ break;
+ }
}
-
- filter_index = found_index;
}
- scores[2] = int64_t(found);
+ scores[2] = found ? eval.scores[index] : 0;
} else if(field_values[2] == &vector_distance_sentinel_value) {
scores[2] = float_to_int64_t(vector_distance);
} else {
@@ -4654,7 +4684,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::
all_result_ids_len = filter_result_iterator->to_filter_id_array(all_result_ids);
filter_result_iterator->reset();
- size_t filter_index = 0;
+ std::vector<uint32_t> filter_indexes;
// populate topster
for(size_t i = 0; i < std::min<size_t>(10000, all_result_ids_len); i++) {
auto seq_id = filter_result_iterator->seq_id;
@@ -4666,7 +4696,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::
int64_t match_score_index = -1;
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
- seq_id, references, filter_index, match_score, scores,
+ seq_id, references, filter_indexes, match_score, scores,
match_score_index, 0, collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
@@ -4810,7 +4840,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::v
}
bool field_is_array = search_schema.at(the_fields[field_id].name).is_array();
- size_t filter_index = 0;
+ std::vector<uint32_t> filter_indexes;
for(size_t i = 0; i < raw_infix_ids_length; i++) {
auto seq_id = raw_infix_ids[i];
@@ -4828,7 +4858,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::v
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values,
geopoint_indices, seq_id, references,
- filter_index, 100, scores, match_score_index,
+ filter_indexes, 100, scores, match_score_index,
0, collection_name);
if (!compute_sort_scores_op.ok()) {
return compute_sort_scores_op;
@@ -5196,7 +5226,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
search_stop_us = parent_search_stop_ms;
search_cutoff = parent_search_cutoff;
- size_t filter_index = 0;
+ std::vector<uint32_t> filter_indexes;
for(size_t i = 0; i < batch_result->count; i++) {
const uint32_t seq_id = batch_result->docs[i];
@@ -5214,7 +5244,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
int64_t match_score_index = -1;
auto compute_sort_scores_op = compute_sort_scores(sort_fields, sort_order, field_values, geopoint_indices,
- seq_id, references, filter_index, 100, scores,
+ seq_id, references, filter_indexes, 100, scores,
match_score_index, 0, collection_name);
if (!compute_sort_scores_op.ok()) {
compute_sort_score_status = new Option<bool>(compute_sort_scores_op.code(), compute_sort_scores_op.error());
@@ -5325,12 +5355,20 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint
field_values[i] = &seq_id_sentinel_value;
} else if (sort_fields_std[i].name == sort_field_const::eval) {
field_values[i] = &eval_sentinel_value;
- auto filter_result_iterator = filter_result_iterator_t("", this, sort_fields_std[i].eval.filter_tree_root);
- auto filter_init_op = filter_result_iterator.init_status();
- if (!filter_init_op.ok()) {
- return;
+ auto& eval_exp = sort_fields_std[i].eval;
+ auto count = sort_fields_std[i].eval_expressions.size();
+ for (uint32_t j = 0; j < count; j++) {
+ auto filter_result_iterator = filter_result_iterator_t("", this, &eval_exp.filter_trees[j]);
+ auto filter_init_op = filter_result_iterator.init_status();
+ if (!filter_init_op.ok()) {
+ return;
+ }
+ uint32_t* eval_ids = nullptr;
+ auto eval_ids_count = filter_result_iterator.to_filter_id_array(eval_ids);
+
+ eval_exp.eval_ids_vec.push_back(eval_ids);
+ eval_exp.eval_ids_count_vec.push_back(eval_ids_count);
}
- sort_fields_std[i].eval.size = filter_result_iterator.to_filter_id_array(sort_fields_std[i].eval.ids);
} else if(sort_fields_std[i].name == sort_field_const::vector_distance) {
field_values[i] = &vector_distance_sentinel_value;
} else if (search_schema.count(sort_fields_std[i].name) != 0 && search_schema.at(sort_fields_std[i].name).sort) {
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index e0f382fb..421ca276 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -1177,11 +1177,27 @@ TEST_F(CollectionManagerTest, ParseSortByClause) {
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("_eval(brand:nike && foo:bar):DESC,points:desc ", sort_fields);
ASSERT_TRUE(sort_by_parsed);
- ASSERT_EQ("_eval(brand:nike && foo:bar)", sort_fields[0].name);
+ ASSERT_EQ("_eval", sort_fields[0].name);
+ ASSERT_FALSE(sort_fields[0].eval_expressions.empty());
+ ASSERT_EQ("brand:nike && foo:bar", sort_fields[0].eval_expressions[0]);
+ ASSERT_EQ(1, sort_fields[0].eval.scores.size());
+ ASSERT_EQ(1, sort_fields[0].eval.scores[0]);
ASSERT_EQ("DESC", sort_fields[0].order);
ASSERT_EQ("points", sort_fields[1].name);
ASSERT_EQ("DESC", sort_fields[1].order);
+ sort_fields.clear();
+ sort_by_parsed = CollectionManager::parse_sort_by_str("_eval([(brand:nike || brand:air):3, (brand:adidas):2]):DESC", sort_fields);
+ ASSERT_TRUE(sort_by_parsed);
+ ASSERT_EQ("_eval", sort_fields[0].name);
+ ASSERT_EQ(2, sort_fields[0].eval_expressions.size());
+ ASSERT_EQ("brand:nike || brand:air", sort_fields[0].eval_expressions[0]);
+ ASSERT_EQ("brand:adidas", sort_fields[0].eval_expressions[1]);
+ ASSERT_EQ(2, sort_fields[0].eval.scores.size());
+ ASSERT_EQ(3, sort_fields[0].eval.scores[0]);
+ ASSERT_EQ(2, sort_fields[0].eval.scores[1]);
+ ASSERT_EQ("DESC", sort_fields[0].order);
+
sort_fields.clear();
sort_by_parsed = CollectionManager::parse_sort_by_str("points:desc,loc(24.56,10.45):ASC,"
"$Customers(product_price:DESC)", sort_fields);
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 4ae097d0..3b729c26 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -2007,7 +2007,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
}
std::vector<sort_by> sort_fields = {
- sort_by("_eval(brand:nike)", "DESC"),
+ sort_by({"brand:nike"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2021,7 +2021,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
// compound query
sort_fields = {
- sort_by("_eval(brand:nike && points:0)", "DESC"),
+ sort_by({"brand:nike && points:0"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2033,9 +2033,22 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());
}
+ sort_fields = {
+ sort_by({"brand:nike", "points:1"}, {2, 1}, "DESC"),
+ sort_by("points", "DESC"),
+ };
+
+ results = coll1->search("*", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();
+ ASSERT_EQ(5, results["hits"].size());
+
+ expected_ids = {"3", "0", "1", "4", "2"};
+ for(size_t i = 0; i < expected_ids.size(); i++) {
+ ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());
+ }
+
// when no results are found for eval query
sort_fields = {
- sort_by("_eval(brand:foobar)", "DESC"),
+ sort_by({"brand:foobar"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2049,7 +2062,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
// bad syntax for eval query
sort_fields = {
- sort_by("_eval(brandnike || points:0)", "DESC"),
+ sort_by({"brandnike || points:0"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2058,13 +2071,20 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
ASSERT_EQ("Error parsing eval expression in sort_by clause.", res_op.error());
// when eval condition is empty
- sort_fields = {
- sort_by("_eval()", "DESC"),
- sort_by("points", "DESC"),
+
+ std::map<std::string, std::string> req_params = {
+ {"collection", "coll1"},
+ {"q", "*"},
+ {"query_by", "title"},
+ {"sort_by", "_eval():desc"},
};
- res_op = coll1->search("*", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10);
- ASSERT_FALSE(res_op.ok());
- ASSERT_EQ("The eval expression in sort_by is empty.", res_op.error());
+ nlohmann::json embedded_params;
+ std::string json_res;
+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+ auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("The eval expression in sort_by is empty.", search_op.error());
// more bad syntax!
sort_fields = {
@@ -2078,8 +2098,8 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
// don't allow multiple sorting eval expressions
sort_fields = {
- sort_by("_eval(brand: nike || points:0)", "DESC"),
- sort_by("_eval(brand: nike || points:0)", "DESC"),
+ sort_by({"brand: nike || points:0"}, {1}, "DESC"),
+ sort_by({"brand: nike || points:0"}, {1}, "DESC"),
};
res_op = coll1->search("*", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10);
@@ -2114,7 +2134,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {
}
std::vector<sort_by> sort_fields = {
- sort_by("_eval(brand:nike)", "DESC"),
+ sort_by({"brand:nike"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2128,7 +2148,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {
// compound query
sort_fields = {
- sort_by("_eval(brand:nike && points:0)", "DESC"),
+ sort_by({"brand:nike && points:0"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2158,7 +2178,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {
// when no results are found for eval query
sort_fields = {
- sort_by("_eval(brand:foobar)", "DESC"),
+ sort_by({"brand:foobar"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2172,7 +2192,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {
// bad syntax for eval query
sort_fields = {
- sort_by("_eval(brandnike || points:0)", "DESC"),
+ sort_by({"brandnike || points:0"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2237,7 +2257,7 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSecondThirdParams) {
std::vector<sort_by> sort_fields = {
sort_by("val", "DESC"),
- sort_by("_eval(brand:nike)", "DESC"),
+ sort_by({"brand:nike"}, {1}, "DESC"),
sort_by("points", "DESC"),
};
@@ -2253,7 +2273,32 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSecondThirdParams) {
sort_fields = {
sort_by("val", "DESC"),
sort_by("val", "DESC"),
- sort_by("_eval(brand:nike)", "DESC"),
+ sort_by({"brand:nike"}, {1}, "DESC"),
+ };
+
+ results = coll1->search("title", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();
+ ASSERT_EQ(5, results["hits"].size());
+ for(size_t i = 0; i < expected_ids.size(); i++) {
+ ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());
+ }
+
+ sort_fields = {
+ sort_by("val", "DESC"),
+ sort_by({"brand:adidas", "brand:nike"}, {2, 1}, "DESC"),
+ sort_by("points", "DESC"),
+ };
+ expected_ids = {"4", "2", "1","3", "0"};
+
+ results = coll1->search("title", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();
+ ASSERT_EQ(5, results["hits"].size());
+ for(size_t i = 0; i < expected_ids.size(); i++) {
+ ASSERT_EQ(expected_ids[i], results["hits"][i]["document"]["id"].get<std::string>());
+ }
+
+ sort_fields = {
+ sort_by("val", "DESC"),
+ sort_by("val", "DESC"),
+ sort_by({"brand:adidas", "brand:nike"}, {2, 1}, "DESC"),
};
results = coll1->search("title", {"title"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 10).get();