Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow bucketing of text match score.
Kishore Nallan774d7dbce550
diff --git a/include/collection.h b/include/collection.hindex 137f9218..8fe4d616 100644--- a/include/collection.h+++ b/include/collection.h@@ -127,6 +127,8 @@ private:const uint8_t CURATED_RECORD_IDENTIFIER = 100;+ const size_t DEFAULT_TOPSTER_SIZE = 250;+struct highlight_t {std::string field;std::vector<std::string> snippets;@@ -241,6 +243,9 @@ private:static std::vector<char> to_char_array(const std::vector<std::string>& strs);+ Option<bool> validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,+ std::vector<sort_by>& sort_fields_std) const;+public:enum {MAX_ARRAY_MATCHES = 5};diff --git a/include/field.h b/include/field.hindex 4948264b..0102e7b1 100644--- a/include/field.h+++ b/include/field.h@@ -562,25 +562,30 @@ struct sort_by {std::string name;std::string order;+ // for text_match score bucketing+ uint32_t text_match_buckets;+// geo related fieldsint64_t geopoint;uint32_t exclude_radius;uint32_t geo_precision;sort_by(const std::string & name, const std::string & order):- name(name), order(order), geopoint(0), exclude_radius(0), geo_precision(0) {+ name(name), order(order), text_match_buckets(0), geopoint(0), exclude_radius(0), geo_precision(0) {}- sort_by(const std::string &name, const std::string &order, int64_t geopoint,+ sort_by(const std::string &name, const std::string &order, uint32_t text_match_buckets, int64_t geopoint,uint32_t exclude_radius, uint32_t geo_precision) :- name(name), order(order), geopoint(geopoint), exclude_radius(exclude_radius), geo_precision(geo_precision) {+ name(name), order(order), text_match_buckets(text_match_buckets),+ geopoint(geopoint), exclude_radius(exclude_radius), geo_precision(geo_precision) {}sort_by& operator=(const sort_by& other) {name = other.name;order = other.order;+ text_match_buckets = other.text_match_buckets;geopoint = other.geopoint;exclude_radius = other.exclude_radius;geo_precision = other.geo_precision;diff --git a/src/collection.cpp b/src/collection.cppindex 6886edab..5ea4eea8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -473,6 +473,169 @@ void Collection::curate_results(string& actual_query, bool enable_overrides, boo}}+Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,+ std::vector<sort_by>& sort_fields_std) const {++ for(const sort_by& _sort_field: sort_fields) {+ sort_by sort_field_std(_sort_field.name, _sort_field.order);++ if(sort_field_std.name.back() == ')') {+ // check if this is a geo field or text match field+ size_t paran_start = 0;+ while(paran_start < sort_field_std.name.size() && sort_field_std.name[paran_start] != '(') {+ paran_start++;+ }++ const std::string& actual_field_name = sort_field_std.name.substr(0, paran_start);++ if(actual_field_name == sort_field_const::text_match) {+ std::vector<std::string> match_parts;+ const std::string& match_config = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);+ StringUtils::split(match_config, match_parts, ":");+ if(match_parts.size() != 2 || match_parts[0] != "buckets") {+ return Option<bool>(400, "Invalid sorting parameter passed for _text_match.");+ }++ if(!StringUtils::is_uint32_t(match_parts[1])) {+ return Option<bool>(400, "Invalid value passed for _text_match `buckets` configuration.");+ }++ sort_field_std.name = actual_field_name;+ sort_field_std.text_match_buckets = std::stoll(match_parts[1]);++ } else {+ if(sort_schema.count(actual_field_name) == 0) {+ std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";+ return Option<bool>(404, error);+ }++ const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);++ // e.g. geopoint_field(lat1, lng1, exclude_radius: 10 miles)++ std::vector<std::string> geo_parts;+ StringUtils::split(geo_coordstr, geo_parts, ",");++ std::string error = "Bad syntax for geopoint sorting field `" + actual_field_name + "`";++ if(geo_parts.size() != 2 && geo_parts.size() != 3) {+ return Option<bool>(400, error);+ }++ if(!StringUtils::is_float(geo_parts[0]) || !StringUtils::is_float(geo_parts[1])) {+ return Option<bool>(400, error);+ }++ if(geo_parts.size() == 3) {+ // try to parse the exclude radius option+ bool is_exclude_option = false;++ if(StringUtils::begins_with(geo_parts[2], sort_field_const::exclude_radius)) {+ is_exclude_option = true;+ } else if(StringUtils::begins_with(geo_parts[2], sort_field_const::precision)) {+ is_exclude_option = false;+ } else {+ return Option<bool>(400, error);+ }++ std::vector<std::string> param_parts;+ StringUtils::split(geo_parts[2], param_parts, ":");++ if(param_parts.size() != 2) {+ return Option<bool>(400, error);+ }++ std::vector<std::string> param_value_parts;+ StringUtils::split(param_parts[1], param_value_parts, " ");++ if(param_value_parts.size() != 2) {+ return Option<bool>(400, error);+ }++ if(!StringUtils::is_float(param_value_parts[0])) {+ return Option<bool>(400, error);+ }++ int32_t value_meters;++ if(param_value_parts[1] == "km") {+ value_meters = std::stof(param_value_parts[0]) * 1000;+ } else if(param_value_parts[1] == "mi") {+ value_meters = std::stof(param_value_parts[0]) * 1609.34;+ } else {+ return Option<bool>(400, "Sort field's parameter "+ "unit must be either `km` or `mi`.");+ }++ if(value_meters <= 0) {+ return Option<bool>(400, "Sort field's parameter must be a positive number.");+ }++ if(is_exclude_option) {+ sort_field_std.exclude_radius = value_meters;+ } else {+ sort_field_std.geo_precision = value_meters;+ }+ }++ double lat = std::stod(geo_parts[0]);+ double lng = std::stod(geo_parts[1]);+ int64_t lat_lng = GeoPoint::pack_lat_lng(lat, lng);+ sort_field_std.name = actual_field_name;+ sort_field_std.geopoint = lat_lng;+ }+ }++ if(sort_field_std.name != sort_field_const::text_match && sort_schema.count(sort_field_std.name) == 0) {+ std::string error = "Could not find a field named `" + sort_field_std.name + "` in the schema for sorting.";+ return Option<bool>(404, error);+ }++ StringUtils::toupper(sort_field_std.order);++ if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {+ std::string error = "Order for field` " + sort_field_std.name + "` should be either ASC or DESC.";+ return Option<bool>(400, error);+ }++ sort_fields_std.emplace_back(sort_field_std);+ }++ /*+ 1. Empty: [match_score, dsf] upstream+ 2. ONE : [usf, match_score]+ 3. TWO : [usf1, usf2, match_score]+ 4. THREE: do nothing+ */+ if(sort_fields_std.empty()) {+ sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);+ if(!default_sorting_field.empty()) {+ sort_fields_std.emplace_back(default_sorting_field, sort_field_const::desc);+ } else {+ sort_fields_std.emplace_back(sort_field_const::seq_id, sort_field_const::desc);+ }+ }++ bool found_match_score = false;+ for(const auto & sort_field : sort_fields_std) {+ if(sort_field.name == sort_field_const::text_match) {+ found_match_score = true;+ break;+ }+ }++ if(!found_match_score && sort_fields.size() < 3) {+ sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);+ }++ if(sort_fields_std.size() > 3) {+ std::string message = "Only upto 3 sort_by fields can be specified.";+ return Option<bool>(422, message);+ }++ return Option<bool>(true);+}+Option<nlohmann::json> Collection::search(const std::string & raw_query, const std::vector<std::string>& search_fields,const std::string & simple_filter_query, const std::vector<std::string>& facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,@@ -655,147 +818,26 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s// validate sort fields and standardizestd::vector<sort_by> sort_fields_std;+ auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields, sort_fields_std);+ if(!sort_validation_op.ok()) {+ return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());+ }- for(const sort_by& _sort_field: sort_fields) {- sort_by sort_field_std(_sort_field.name, _sort_field.order);-- if(sort_field_std.name.back() == ')') {- // check if this is a geo field- size_t paran_start = 0;- while(paran_start < sort_field_std.name.size() && sort_field_std.name[paran_start] != '(') {- paran_start++;- }-- const std::string& actual_field_name = sort_field_std.name.substr(0, paran_start);-- if(sort_schema.count(actual_field_name) == 0) {- std::string error = "Could not find a field named `" + actual_field_name + "` in the schema for sorting.";- return Option<nlohmann::json>(404, error);- }-- const std::string& geo_coordstr = sort_field_std.name.substr(paran_start+1, sort_field_std.name.size() - paran_start - 2);-- // e.g. geopoint_field(lat1, lng1, exclude_radius: 10 miles)-- std::vector<std::string> geo_parts;- StringUtils::split(geo_coordstr, geo_parts, ",");-- std::string error = "Bad syntax for geopoint sorting field `" + actual_field_name + "`";-- if(geo_parts.size() != 2 && geo_parts.size() != 3) {- return Option<nlohmann::json>(400, error);- }-- if(!StringUtils::is_float(geo_parts[0]) || !StringUtils::is_float(geo_parts[1])) {- return Option<nlohmann::json>(400, error);- }-- if(geo_parts.size() == 3) {- // try to parse the exclude radius option- bool is_exclude_option = false;-- if(StringUtils::begins_with(geo_parts[2], sort_field_const::exclude_radius)) {- is_exclude_option = true;- } else if(StringUtils::begins_with(geo_parts[2], sort_field_const::precision)) {- is_exclude_option = false;- } else {- return Option<nlohmann::json>(400, error);- }-- std::vector<std::string> param_parts;- StringUtils::split(geo_parts[2], param_parts, ":");-- if(param_parts.size() != 2) {- return Option<nlohmann::json>(400, error);- }-- std::vector<std::string> param_value_parts;- StringUtils::split(param_parts[1], param_value_parts, " ");-- if(param_value_parts.size() != 2) {- return Option<nlohmann::json>(400, error);- }-- if(!StringUtils::is_float(param_value_parts[0])) {- return Option<nlohmann::json>(400, error);- }-- int32_t value_meters;-- if(param_value_parts[1] == "km") {- value_meters = std::stof(param_value_parts[0]) * 1000;- } else if(param_value_parts[1] == "mi") {- value_meters = std::stof(param_value_parts[0]) * 1609.34;- } else {- return Option<nlohmann::json>(400, "Sort field's parameter "- "unit must be either `km` or `mi`.");- }-- if(value_meters <= 0) {- return Option<nlohmann::json>(400, "Sort field's parameter must be a positive number.");- }+ // apply bucketing on text match score+ int match_score_index = -1;+ for(size_t i = 0; i < sort_fields_std.size(); i++) {+ if(sort_fields_std[i].name == sort_field_const::text_match && sort_fields_std[i].text_match_buckets != 0) {+ match_score_index = i;- if(is_exclude_option) {- sort_field_std.exclude_radius = value_meters;- } else {- sort_field_std.geo_precision = value_meters;- }+ if(sort_fields_std[i].text_match_buckets > 1) {+ // we will disable prioritize exact match because it's incompatible with bucketing+ prioritize_exact_match = false;}- double lat = std::stod(geo_parts[0]);- double lng = std::stod(geo_parts[1]);- int64_t lat_lng = GeoPoint::pack_lat_lng(lat, lng);- sort_field_std.name = actual_field_name;- sort_field_std.geopoint = lat_lng;- }-- if(sort_field_std.name != sort_field_const::text_match && sort_schema.count(sort_field_std.name) == 0) {- std::string error = "Could not find a field named `" + sort_field_std.name + "` in the schema for sorting.";- return Option<nlohmann::json>(404, error);- }-- StringUtils::toupper(sort_field_std.order);-- if(sort_field_std.order != sort_field_const::asc && sort_field_std.order != sort_field_const::desc) {- std::string error = "Order for field` " + sort_field_std.name + "` should be either ASC or DESC.";- return Option<nlohmann::json>(400, error);- }-- sort_fields_std.emplace_back(sort_field_std);- }-- /*- 1. Empty: [match_score, dsf] upstream- 2. ONE : [usf, match_score]- 3. TWO : [usf1, usf2, match_score]- 4. THREE: do nothing- */- if(sort_fields_std.empty()) {- sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);- if(!default_sorting_field.empty()) {- sort_fields_std.emplace_back(default_sorting_field, sort_field_const::desc);- } else {- sort_fields_std.emplace_back(sort_field_const::seq_id, sort_field_const::desc);- }- }-- bool found_match_score = false;- for(const auto & sort_field : sort_fields_std) {- if(sort_field.name == sort_field_const::text_match) {- found_match_score = true;break;}}- if(!found_match_score && sort_fields.size() < 3) {- sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);- }-- if(sort_fields_std.size() > 3) {- std::string message = "Only upto 3 sort_by fields can be specified.";- return Option<nlohmann::json>(422, message);- }-// check for valid paginationif(page < 1) {std::string message = "Page must be an integer of value greater than 0.";@@ -813,7 +855,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sreturn Option<nlohmann::json>(422, message);}- size_t max_hits = 250;+ size_t max_hits = DEFAULT_TOPSTER_SIZE;// ensure that `max_hits` never exceeds number of documents in collectionif(search_fields.size() <= 1 || raw_query == "*") {@@ -965,8 +1007,28 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const stotal_found = search_params->all_result_ids_len;}- // All fields are sorted descending- std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv_group);+ if(match_score_index >= 0 && sort_fields_std[match_score_index].text_match_buckets > 1) {+ size_t num_buckets = sort_fields_std[match_score_index].text_match_buckets;++ const size_t max_kvs_bucketed = std::min<size_t>(DEFAULT_TOPSTER_SIZE, raw_result_kvs.size());+ std::vector<int64_t> result_scores(max_kvs_bucketed);++ // only first `max_kvs_bucketed` elements are bucketed to prevent pagination issues past 250 records+ for(size_t i = 0; i < max_kvs_bucketed; i++) {+ result_scores[i] = raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index];+ size_t block_index = (i / num_buckets) * num_buckets;+ int64_t text_match = raw_result_kvs[block_index][0]->scores[raw_result_kvs[block_index][0]->match_score_index];+ raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index] = text_match;+ }++ // sort again based on bucketed match score+ std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv_group);++ // restore original scores+ for(size_t i = 0; i < max_kvs_bucketed; i++) {+ raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index] = result_scores[i];+ }+ }// Sort based on position in overridden liststd::sort(diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 4a9bf745..b50aebfa 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -1146,3 +1146,124 @@ TEST_F(CollectionSortingTest, SortByTitle) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionSortingTest, TextMatchBucketRanking) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("description", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "Mark Antony";+ doc1["description"] = "Marriage Counsellor";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["title"] = "Mark Spencer";+ doc2["description"] = "Sales Expert";+ doc2["points"] = 200;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ sort_fields = {+ sort_by("_text_match(buckets: 10)", "DESC"),+ sort_by("points", "DESC"),+ };++ auto results = coll1->search("mark", {"title", "description"},+ "", {}, sort_fields, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3, 1}, 1000, true).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ // bucketing by 1 produces original text match+ sort_fields = {+ sort_by("_text_match(buckets: 1)", "DESC"),+ sort_by("points", "DESC"),+ };++ results = coll1->search("mark", {"title", "description"},+ "", {}, sort_fields, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3, 1}, 1000, true).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ // likewise with bucket 0+ sort_fields = {+ sort_by("_text_match(buckets: 0)", "DESC"),+ sort_by("points", "DESC"),+ };++ results = coll1->search("mark", {"title", "description"},+ "", {}, sort_fields, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3, 1}, 1000, true).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ // don't allow bad parameter name+ sort_fields[0] = sort_by("_text_match(foobar: 0)", "DESC");++ auto res_op = coll1->search("mark", {"title", "description"},+ "", {}, sort_fields, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3, 1}, 1000, true);++ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Invalid sorting parameter passed for _text_match.", res_op.error());++ // handle bad syntax+ sort_fields[0] = sort_by("_text_match(foobar:", "DESC");+ res_op = coll1->search("mark", {"title", "description"},+ "", {}, sort_fields, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3, 1}, 1000, true);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `_text_match(foobar:` in the schema for sorting.", res_op.error());++ // handle bad value+ sort_fields[0] = sort_by("_text_match(buckets: x)", "DESC");+ res_op = coll1->search("mark", {"title", "description"},+ "", {}, sort_fields, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3, 1}, 1000, true);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Invalid value passed for _text_match `buckets` configuration.", res_op.error());++ // handle negative value+ sort_fields[0] = sort_by("_text_match(buckets: -1)", "DESC");+ res_op = coll1->search("mark", {"title", "description"},+ "", {}, sort_fields, {2, 2}, 10,+ 1, FREQUENCY, {true, true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {3, 1}, 1000, true);+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Invalid value passed for _text_match `buckets` configuration.", res_op.error());++ collectionManager.drop_collection("coll1");+}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex a64d1840..8d8026a2 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -2245,42 +2245,6 @@ TEST_F(CollectionSpecificTest, HandleLargeWeights) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionSpecificTest, DISABLED_ExactMatchOnAFieldIgnoresOtherFieldScores) {- std::vector<field> fields = {field("title", field_types::STRING, false),- field("description", field_types::STRING, false),- field("points", field_types::INT32, false),};-- Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();-- nlohmann::json doc1;- doc1["id"] = "0";- doc1["title"] = "Mark Antony";- doc1["description"] = "Marriage Counsellor";- doc1["points"] = 100;-- nlohmann::json doc2;- doc2["id"] = "1";- doc2["title"] = "Mark Spencer";- doc2["description"] = "Sales Expert";- doc2["points"] = 200;-- ASSERT_TRUE(coll1->add(doc1.dump()).ok());- ASSERT_TRUE(coll1->add(doc2.dump()).ok());-- auto results = coll1->search("mark", {"title", "description"},- "", {}, {}, {2, 2}, 10,- 1, FREQUENCY, {true, true},- 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {3, 1}, 1000, true).get();-- ASSERT_EQ(2, results["hits"].size());- ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());- ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());-- collectionManager.drop_collection("coll1");-}-TEST_F(CollectionSpecificTest, VerbatimMatchShouldNotOverpowerHigherWeightedField) {std::vector<field> fields = {field("title", field_types::STRING, false),field("description", field_types::STRING, false),