Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow missing values sort config on numerical fields.
Kishore Nalland4cfe81a9027
diff --git a/include/field.h b/include/field.hindex 0f87263b..c18179af 100644--- a/include/field.h+++ b/include/field.h@@ -454,7 +454,7 @@ struct sort_by {enum missing_values_t {first,last,- ordered,+ normal,};std::string name;@@ -472,7 +472,7 @@ struct sort_by {sort_by(const std::string & name, const std::string & order):name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0),- missing_values(ordered) {+ missing_values(normal) {}@@ -480,7 +480,7 @@ struct sort_by {uint32_t exclude_radius, uint32_t geo_precision) :name(name), order(order), text_match_buckets(text_match_buckets),geopoint(geopoint), exclude_radius(exclude_radius), geo_precision(geo_precision),- missing_values(ordered) {+ missing_values(normal) {}diff --git a/src/collection.cpp b/src/collection.cppindex 45398fd7..e22bdde1 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -547,7 +547,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<std::string error = "Bad syntax for sorting field `" + actual_field_name + "`";- if(field_it->second.type == field_types::STRING) {+ if(!field_it->second.is_geopoint()) {// check for null value orderconst std::string& sort_params_str = sort_field_std.name.substr(paran_start + 1,sort_field_std.name.size() -@@ -572,7 +572,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<}}- else if(field_it->second.is_geopoint()) {+ else {const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);// e.g. geopoint_field(lat1, lng1, exclude_radius: 10 miles)@@ -658,10 +658,6 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_field_std.geopoint = lat_lng;}- else {- return Option<bool>(400, error);- }-sort_field_std.name = actual_field_name;}}diff --git a/src/index.cpp b/src/index.cppindex ee113163..65e6d9e9 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3279,6 +3279,14 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i} else {auto it = field_values[0]->find(seq_id);scores[0] = (it == field_values[0]->end()) ? default_score : it->second;++ if(scores[0] == INT64_MIN && sort_fields[0].missing_values == sort_by::missing_values_t::first) {+ // By default, missing numerical value are always going to be sorted to be at the end+ // because: -INT64_MIN == INT64_MIN. To account for missing values config, we will have to change+ // the default for missing value based on whether it's asc or desc sort.+ bool is_asc = (sort_order[0] == -1);+ scores[0] = is_asc ? (INT64_MIN + 1) : INT64_MAX;+ }}if (sort_order[0] == -1) {@@ -3310,6 +3318,10 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i} else {auto it = field_values[1]->find(seq_id);scores[1] = (it == field_values[1]->end()) ? default_score : it->second;+ if(scores[1] == INT64_MIN && sort_fields[1].missing_values == sort_by::missing_values_t::first) {+ bool is_asc = (sort_order[1] == -1);+ scores[1] = is_asc ? (INT64_MIN + 1) : INT64_MAX;+ }}if (sort_order[1] == -1) {@@ -3341,6 +3353,10 @@ void Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const i} else {auto it = field_values[2]->find(seq_id);scores[2] = (it == field_values[2]->end()) ? default_score : it->second;+ if(scores[2] == INT64_MIN && sort_fields[2].missing_values == sort_by::missing_values_t::first) {+ bool is_asc = (sort_order[2] == -1);+ scores[2] = is_asc ? (INT64_MIN + 1) : INT64_MAX;+ }}if (sort_order[2] == -1) {diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 4672f9de..f9175b84 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -1330,14 +1330,14 @@ TEST_F(CollectionSortingTest, SortByStringEmptyValuesConfigFirstField) {// natural ordersort_fields = {- sort_by("title(missing_values: ordered)", "asc"),+ sort_by("title(missing_values: normal)", "asc"),};results = coll1->search("*", {"title"}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();ASSERT_EQ(4, results["hits"].size());ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());sort_fields = {- sort_by("title(missing_values: ordered)", "desc"),+ sort_by("title(missing_values: normal)", "desc"),};results = coll1->search("*", {"title"}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();ASSERT_EQ(4, results["hits"].size());diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 77ffd43a..20fa5dce 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -260,3 +260,173 @@ TEST_F(CollectionSpecificMoreTest, VerbatimMatchNotOnPartialTokenMatch) {ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());}++TEST_F(CollectionSpecificMoreTest, SortByStringEmptyValuesConfigFirstField) {+ std::vector<field> fields = {field("points", field_types::INT32, false, true),+ field("points2", field_types::INT32, false, true),+ field("points3", field_types::INT32, false, true)};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ for(size_t i = 0; i < 4; i++) {+ nlohmann::json doc;+ if(i == 2) {+ doc["points"] = nullptr;+ } else {+ doc["points"] = i;+ }+ doc["points2"] = 100;+ doc["points3"] = 100;+ coll1->add(doc.dump());+ }++ // without any order config: missing integers always end up last+ sort_fields = {sort_by("points", "asc"),};+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points", "desc"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ // ascending+ sort_fields = {sort_by("points(missing_values: first)", "ASC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points(missing_values: last)", "ASC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ // descending+ sort_fields = {sort_by("points(missing_values: first)", "DESC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points(missing_values: last)", "DESC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ // bad syntax+ sort_fields = {sort_by("points(foo: bar)", "desc"),};+ auto res_op = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true});+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Bad syntax for sorting field `points`", res_op.error());++ sort_fields = {sort_by("points(missing_values: bar)", "desc"),};+ res_op = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true});+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Bad syntax for sorting field `points`", res_op.error());+}++TEST_F(CollectionSpecificMoreTest, SortByStringEmptyValuesConfigSecondField) {+ std::vector<field> fields = {field("points", field_types::INT32, false, true),+ field("points2", field_types::INT32, false, true),+ field("points3", field_types::INT32, false, true)};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ for(size_t i = 0; i < 4; i++) {+ nlohmann::json doc;+ if(i == 2) {+ doc["points"] = nullptr;+ } else {+ doc["points"] = i;+ }+ doc["points2"] = 100;+ doc["points3"] = 100;+ coll1->add(doc.dump());+ }++ // without any order config: missing integers always end up last+ sort_fields = {sort_by("points2", "asc"),sort_by("points", "asc")};+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points2", "asc"),sort_by("points", "desc"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ // ascending+ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: first)", "ASC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: last)", "ASC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ // descending+ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: first)", "DESC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points2", "asc"),sort_by("points(missing_values: last)", "DESC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());+}++TEST_F(CollectionSpecificMoreTest, SortByStringEmptyValuesConfigThirdField) {+ std::vector<field> fields = {field("points", field_types::INT32, false, true),+ field("points2", field_types::INT32, false, true),+ field("points3", field_types::INT32, false, true)};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ for(size_t i = 0; i < 4; i++) {+ nlohmann::json doc;+ if(i == 2) {+ doc["points"] = nullptr;+ } else {+ doc["points"] = i;+ }+ doc["points2"] = 100;+ doc["points3"] = 100;+ coll1->add(doc.dump());+ }++ // without any order config: missing integers always end up last+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points", "asc")};+ auto results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points", "desc"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ // ascending+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: first)", "ASC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: last)", "ASC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());++ // descending+ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: first)", "DESC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ sort_fields = {sort_by("points2", "asc"),sort_by("points3", "asc"),sort_by("points(missing_values: last)", "DESC"),};+ results = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, MAX_SCORE, {true}).get();+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());+}