Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow missing values sort config on numerical fields.

Kishore Nalland4cfe81a9027
diff --git a/include/field.h b/include/field.h
index 0f87263b..c18179af 100644
--- a/include/field.h
+++ b/include/field.h
@@ -454,7 +454,7 @@ struct sort_by {
enum missing_values_t {
first,
last,
- ordered,
+ normal,
};
std::string name;
@@ -472,7 +472,7 @@ struct sort_by {
sort_by(const std::string & name, const std::string & order):
name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),
- missing_values(ordered) {
+ missing_values(normal) {
}
@@ -480,7 +480,7 @@ struct sort_by {
uint32_t exclude_radius, uint32_t geo_precision) :
name(name), order(order), text_match_buckets(text_match_buckets),
geopoint(geopoint), exclude_radius(exclude_radius), geo_precision(geo_precision),
- missing_values(ordered) {
+ missing_values(normal) {
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 45398fd7..e22bdde1 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -547,7 +547,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
std::string error = "Bad syntax for sorting field `" + actual_field_name + "`";
- if(field_it->second.type == field_types::STRING) {
+ if(!field_it->second.is_geopoint()) {
// check for null value order
const std::string& sort_params_str = sort_field_std.name.substr(paran_start + 1,
sort_field_std.name.size() -
@@ -572,7 +572,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
}
}
- else if(field_it->second.is_geopoint()) {
+ else {
const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);
// e.g. geopoint_field(lat1, lng1, exclude_radius: 10 miles)
@@ -658,10 +658,6 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
sort_field_std.geopoint = lat_lng;
}
- else {
- return Option<bool>(400, error);
- }
-
sort_field_std.name = actual_field_name;
}
}
diff --git a/src/index.cpp b/src/index.cpp
index ee113163..65e6d9e9 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3279,6 +3279,14 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
} else {
auto it = field_values[0]->find(seq_id);
scores[0] = (it == field_values[0]->end()) ? default_score : it->second;
+
+ if(scores[0] == INT64_MIN && sort_fields[0].missing_values == sort_by::missing_values_t::first) {
+ // By default, missing numerical value are always going to be sorted to be at the end
+ // because: -INT64_MIN == INT64_MIN. To account for missing values config, we will have to change
+ // the default for missing value based on whether it's asc or desc sort.
+ bool is_asc = (sort_order[0] == -1);
+ scores[0] = is_asc ? (INT64_MIN + 1) : INT64_MAX;
+ }
}
if (sort_order[0] == -1) {
@@ -3310,6 +3318,10 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
} else {
auto it = field_values[1]->find(seq_id);
scores[1] = (it == field_values[1]->end()) ? default_score : it->second;
+ if(scores[1] == INT64_MIN && sort_fields[1].missing_values == sort_by::missing_values_t::first) {
+ bool is_asc = (sort_order[1] == -1);
+ scores[1] = is_asc ? (INT64_MIN + 1) : INT64_MAX;
+ }
}
if (sort_order[1] == -1) {
@@ -3341,6 +3353,10 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i
} else {
auto it = field_values[2]->find(seq_id);
scores[2] = (it == field_values[2]->end()) ? default_score : it->second;
+ if(scores[2] == INT64_MIN && sort_fields[2].missing_values == sort_by::missing_values_t::first) {
+ bool is_asc = (sort_order[2] == -1);
+ scores[2] = is_asc ? (INT64_MIN + 1) : INT64_MAX;
+ }
}
if (sort_order[2] == -1) {
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 4672f9de..f9175b84 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -1330,14 +1330,14 @@ TEST_F(CollectionSortingTest, SortByStringEmptyValuesConfigFirstField) {
// natural order
sort_fields = {
- sort_by("title(missing_values: ordered)", "asc"),
+ sort_by("title(missing_values: normal)", "asc"),
};
results = coll1->search("*", {"title"}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
sort_fields = {
- sort_by("title(missing_values: ordered)", "desc"),
+ sort_by("title(missing_values: normal)", "desc"),
};
results = coll1->search("*", {"title"}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
ASSERT_EQ(4, results["hits"].size());
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 77ffd43a..20fa5dce 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -260,3 +260,173 @@ TEST_F(CollectionSpecificMoreTest, VerbatimMatchNotOnPartialTokenMatch) {
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
}
+
+TEST_F(CollectionSpecificMoreTest, SortByStringEmptyValuesConfigFirstField) {
+ std::vector<field> fields = {field("points", field_types::INT32, false, true),
+ field("points2", field_types::INT32, false, true),
+ field("points3", field_types::INT32, false, true)};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ for(size_t i = 0; i < 4; i++) {
+ nlohmann::json doc;
+ if(i == 2) {
+ doc["points"] = nullptr;
+ } else {
+ doc["points"] = i;
+ }
+ doc["points2"] = 100;
+ doc["points3"] = 100;
+ coll1->add(doc.dump());
+ }
+
+ // without any order config: missing integers always end up last
+ sort_fields = {sort_by("points", "asc"),};
+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points", "desc"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ // ascending
+ sort_fields = {sort_by("points(missing_values: first)", "ASC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points(missing_values: last)", "ASC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ // descending
+ sort_fields = {sort_by("points(missing_values: first)", "DESC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points(missing_values: last)", "DESC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ // bad syntax
+ sort_fields = {sort_by("points(foo: bar)", "desc"),};
+ auto res_op = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true});
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Bad syntax for sorting field `points`", res_op.error());
+
+ sort_fields = {sort_by("points(missing_values: bar)", "desc"),};
+ res_op = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true});
+ ASSERT_FALSE(res_op.ok());
+ ASSERT_EQ("Bad syntax for sorting field `points`", res_op.error());
+}
+
+TEST_F(CollectionSpecificMoreTest, SortByStringEmptyValuesConfigSecondField) {
+ std::vector<field> fields = {field("points", field_types::INT32, false, true),
+ field("points2", field_types::INT32, false, true),
+ field("points3", field_types::INT32, false, true)};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ for(size_t i = 0; i < 4; i++) {
+ nlohmann::json doc;
+ if(i == 2) {
+ doc["points"] = nullptr;
+ } else {
+ doc["points"] = i;
+ }
+ doc["points2"] = 100;
+ doc["points3"] = 100;
+ coll1->add(doc.dump());
+ }
+
+ // without any order config: missing integers always end up last
+ sort_fields = {sort_by("points2", "asc"),sort_by("points", "asc")};
+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points2", "asc"),sort_by("points", "desc"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ // ascending
+ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: first)", "ASC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: last)", "ASC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ // descending
+ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: first)", "DESC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: last)", "DESC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+}
+
+TEST_F(CollectionSpecificMoreTest, SortByStringEmptyValuesConfigThirdField) {
+ std::vector<field> fields = {field("points", field_types::INT32, false, true),
+ field("points2", field_types::INT32, false, true),
+ field("points3", field_types::INT32, false, true)};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ for(size_t i = 0; i < 4; i++) {
+ nlohmann::json doc;
+ if(i == 2) {
+ doc["points"] = nullptr;
+ } else {
+ doc["points"] = i;
+ }
+ doc["points2"] = 100;
+ doc["points3"] = 100;
+ coll1->add(doc.dump());
+ }
+
+ // without any order config: missing integers always end up last
+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points", "asc")};
+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points", "desc"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ // ascending
+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: first)", "ASC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: last)", "ASC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+
+ // descending
+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: first)", "DESC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: last)", "DESC"),};
+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
+}