Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Sort results on custom order.
kishorenceed10d554d55
diff --git a/include/field.h b/include/field.hindex a19c2dda..03b88467 100644--- a/include/field.h+++ b/include/field.h@@ -113,6 +113,7 @@ namespace sort_field_const {static const std::string order = "order";static const std::string asc = "ASC";static const std::string desc = "DESC";+ static const std::string match_score = "_match_score";}struct sort_by {diff --git a/include/topster.h b/include/topster.hindex 21d938d3..39fdd9dc 100644--- a/include/topster.h+++ b/include/topster.h@@ -14,8 +14,7 @@ struct KV {uint16_t array_index;uint64_t key;uint64_t match_score;- number_t primary_attr;- number_t secondary_attr;+ number_t scores[3]; // match score + 2 custom attributes};/*@@ -39,8 +38,6 @@ struct Topster {data[i].query_index = 0;data[i].key = 0;data[i].match_score = 0;- data[i].primary_attr = number_t();- data[i].secondary_attr = number_t();kvs[i] = &data[i];}}@@ -61,9 +58,9 @@ struct Topster {}void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,- const number_t &primary_attr, const number_t &secondary_attr) {+ const number_t scores[3]) {if (size >= MAX_SIZE) {- if(!is_greater(kvs[0], match_score, primary_attr, secondary_attr)) {+ if(!is_greater(kvs[0], scores)) {// when incoming value is less than the smallest in the heap, ignorereturn;}@@ -90,8 +87,9 @@ struct Topster {kvs[start]->query_index = query_index;kvs[start]->array_index = start;kvs[start]->match_score = match_score;- kvs[start]->primary_attr = primary_attr;- kvs[start]->secondary_attr = secondary_attr;+ kvs[start]->scores[0] = scores[0];+ kvs[start]->scores[1] = scores[1];+ kvs[start]->scores[2] = scores[2];// sift down to maintain heap propertywhile ((2*start+1) < MAX_SIZE) {@@ -129,8 +127,9 @@ struct Topster {kvs[start]->query_index = query_index;kvs[start]->array_index = start;kvs[start]->match_score = match_score;- kvs[start]->primary_attr = primary_attr;- kvs[start]->secondary_attr = secondary_attr;+ kvs[start]->scores[0] = scores[0];+ kvs[start]->scores[1] = scores[1];+ kvs[start]->scores[2] = scores[2];keys[key] = kvs[start];@@ -170,19 +169,19 @@ struct Topster {}}- static bool is_greater(const struct KV* i, uint64_t match_score, number_t primary_attr, number_t secondary_attr) {- return std::tie(match_score, primary_attr, secondary_attr) >- std::tie(i->match_score, i->primary_attr, i->secondary_attr);+ static bool is_greater(const struct KV* i, const number_t scores[3]) {+ return std::tie(scores[0], scores[1], scores[2]) >+ std::tie(i->scores[0], i->scores[1], i->scores[2]);}static bool is_greater_kv(const struct KV* i, const struct KV* j) {- return std::tie(i->match_score, i->primary_attr, i->secondary_attr, i->key) >- std::tie(j->match_score, j->primary_attr, j->secondary_attr, j->key);+ return std::tie(i->scores[0], i->scores[1], i->scores[2], i->key) >+ std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);}static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {- return std::tie(i.match_score, i.primary_attr, i.secondary_attr, i.key) >- std::tie(j.match_score, j.primary_attr, j.secondary_attr, j.key);+ return std::tie(i.scores[0], i.scores[1], i.scores[2], i.key) >+ std::tie(j.scores[0], j.scores[1], j.scores[2], j.key);}// topster must be sorted before iterated upon to remove dead array entriesdiff --git a/src/collection.cpp b/src/collection.cppindex e01d9485..52b298f6 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -508,7 +508,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::vector<sort_by> sort_fields_std;for(const sort_by & _sort_field: sort_fields) {- if(sort_schema.count(_sort_field.name) == 0) {+ if(_sort_field.name != sort_field_const::match_score && sort_schema.count(_sort_field.name) == 0) {std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema for sorting.";return Option<nlohmann::json>(404, error);}@@ -524,10 +524,29 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::sort_fields_std.emplace_back(_sort_field.name, sort_order);}+ /*+ 1. Empty: [match_score, dsf] upstream+ 2. ONE : [usf, match_score]+ 3. TWO : [usf1, usf2, match_score]+ 4. THREE: do nothing+ */if(sort_fields_std.empty()) {+ sort_fields_std.emplace_back(sort_field_const::match_score, sort_field_const::desc);sort_fields_std.emplace_back(default_sorting_field, sort_field_const::desc);}+ bool found_match_score = false;+ for(const auto & sort_field : sort_fields) {+ if(sort_field.name == sort_field_const::match_score) {+ found_match_score = true;+ break;+ }+ }++ if(!found_match_score && sort_fields.size() < 3) {+ sort_fields_std.emplace_back(sort_field_const::match_score, sort_field_const::desc);+ }+// check for valid paginationif(page < 1) {std::string message = "Page must be an integer of value greater than 0.";diff --git a/src/index.cpp b/src/index.cppindex d1abe316..3dc358b0 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -465,8 +465,8 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &tokenuint64_t hash = 0;if(a_field.is_float()) {- float f = atof(token.c_str());- hash = *reinterpret_cast<uint64_t*>(&f); // store as int without loss of precision+ float f = std::stof(token);+ reinterpret_cast<float&>(hash) = f; // store as int without loss of precision} else if(a_field.is_integer() || a_field.is_bool()) {hash = atol(token.c_str());} else {@@ -608,7 +608,7 @@ void Index::compute_facet_stats(facet &a_facet, int64_t raw_value, const std::sta_facet.stats.fvsum += val;a_facet.stats.fvcount++;} else if(field_type == field_types::FLOAT || field_type == field_types::FLOAT_ARRAY) {- float val = *reinterpret_cast<float*>(&raw_value);+ float val = reinterpret_cast<float&>(raw_value);if(val < a_facet.stats.fvmin) {a_facet.stats.fvmin = val;}@@ -1120,8 +1120,12 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f}}- curated_topster.add(seq_id, field_id, searched_queries.size(), match_score,- number_t(int64_t(1)), number_t(int64_t(1)));+ number_t scores[3];+ scores[0] = number_t(int64_t(match_score));+ scores[1] = number_t(int64_t(1));+ scores[2] = number_t(int64_t(1));++ curated_topster.add(seq_id, field_id, searched_queries.size(), match_score, scores);searched_queries.push_back(override_query);}@@ -1416,8 +1420,8 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect}}-void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,- const uint32_t total_cost, Topster & topster,+void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,+ const uint8_t & field_id, const uint32_t total_cost, Topster & topster,const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, const size_t result_size) const {@@ -1429,44 +1433,22 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16leaf_to_indices.emplace(token_leaf, indices);}- spp::sparse_hash_map<uint32_t, number_t> * primary_rank_scores = nullptr;- spp::sparse_hash_map<uint32_t, number_t> * secondary_rank_scores = nullptr;-// Used for asc/desc ordering. NOTE: Topster keeps biggest keys (i.e. it's desc in nature)number_t primary_rank_factor;number_t secondary_rank_factor;- if(sort_fields.size() > 0) {- // assumed that rank field exists in the index - checked earlier in the chain- primary_rank_scores = sort_index.at(sort_fields[0].name);-- // initialize primary_rank_factor- field sort_field = sort_schema.at(sort_fields[0].name);- if(sort_field.is_single_integer() || sort_field.is_single_bool()) {- primary_rank_factor = ((int64_t) 1);- } else {- primary_rank_factor = ((float) 1);- }+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively+ spp::sparse_hash_map<uint32_t, number_t>* field_values[3];- if(sort_fields[0].order == sort_field_const::asc) {- primary_rank_factor = -primary_rank_factor;+ for(size_t i = 0; i < sort_fields.size(); i++) {+ sort_order[i] = 1;+ if(sort_fields[i].order == sort_field_const::asc) {+ sort_order[i] = -1;}- }- if(sort_fields.size() > 1) {- secondary_rank_scores = sort_index.at(sort_fields[1].name);-- // initialize secondary_rank_factor- field sort_field = sort_schema.at(sort_fields[1].name);- if(sort_field.is_single_integer() || sort_field.is_single_bool()) {- secondary_rank_factor = ((int64_t) 1);- } else {- secondary_rank_factor = ((float) 1);- }-- if(sort_fields[1].order == sort_field_const::asc) {- secondary_rank_factor = -secondary_rank_factor;- }+ field_values[i] = (sort_fields[i].name != sort_field_const::match_score) ?+ sort_index.at(sort_fields[i].name) :+ nullptr;}//auto begin = std::chrono::high_resolution_clock::now();@@ -1509,22 +1491,46 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}const int64_t default_score = 0;- number_t primary_rank_score = number_t(default_score);- number_t secondary_rank_score = number_t(default_score);+ number_t scores[3];- if(primary_rank_scores) {- auto it = primary_rank_scores->find(seq_id);- primary_rank_score = (it == primary_rank_scores->end()) ? number_t(default_score) : number_t(it->second);+ // avoiding loop+ if(sort_fields.size() > 0) {+ if (field_values[0] != nullptr) {+ auto it = field_values[0]->find(seq_id);+ scores[0] = (it == field_values[0]->end()) ? number_t(default_score) : number_t(it->second);+ } else {+ scores[0] = number_t(int64_t(match_score));+ }+ if (sort_order[0] == -1) {+ scores[0] = -scores[0];+ }}- if(secondary_rank_scores) {- auto it = secondary_rank_scores->find(seq_id);- secondary_rank_score = (it == secondary_rank_scores->end()) ? number_t(default_score) : number_t(it->second);+ if(sort_fields.size() > 1) {+ if (field_values[1] != nullptr) {+ auto it = field_values[1]->find(seq_id);+ scores[1] = (it == field_values[1]->end()) ? number_t(default_score) : number_t(it->second);+ } else {+ scores[1] = number_t(int64_t(match_score));+ }+ if (sort_order[1] == -1) {+ scores[1] = -scores[1];+ }+ }++ if(sort_fields.size() > 2) {+ if(field_values[2] != nullptr) {+ auto it = field_values[2]->find(seq_id);+ scores[2] = (it == field_values[2]->end()) ? number_t(default_score) : number_t(it->second);+ } else {+ scores[2] = number_t(int64_t(match_score));+ }+ if(sort_order[2] == -1) {+ scores[2] = -scores[2];+ }}- const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;- const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;- topster.add(seq_id, field_id, query_index, match_score, primary_rank_value, secondary_rank_value);+ topster.add(seq_id, field_id, query_index, match_score, scores);}//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 2c70ef6c..17f0179f 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -30,7 +30,7 @@ protected:};query_fields = {"title"};- sort_fields = { sort_by("points", "DESC") };+ sort_fields = { sort_by("_match_score", "DESC"), sort_by("points", "DESC") };collection = collectionManager.get_collection("collection");if(collection == nullptr) {@@ -149,7 +149,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {"What is the power requirement of a <mark>rocket</mark> <mark>launch</mark> these days?");// Check ASC sort order- std::vector<sort_by> sort_fields_asc = { sort_by("points", "ASC") };+ std::vector<sort_by> sort_fields_asc = { sort_by("_match_score", "DESC"), sort_by("points", "ASC") };results = collection->search("rocket launch", query_fields, "", facets, sort_fields_asc, 0, 10).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(5, results["found"].get<uint32_t>());@@ -2802,7 +2802,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {field("age", field_types::INT32, false),};- std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };+ std::vector<sort_by> sort_fields = { sort_by("_match_score", "DESC"), sort_by("age", "DESC") };coll_large_text = collectionManager.get_collection("coll_large_text");if(coll_large_text == nullptr) {diff --git a/test/topster_test.cpp b/test/topster_test.cppindex 5704e8fb..e6f86f1d 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -30,8 +30,12 @@ TEST(TopsterTest, MaxIntValues) {};for(int i = 0; i < 14; i++) {- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score,- number_t(data[i].primary_attr), number_t(data[i].secondary_attr));+ number_t scores[3];+ scores[0] = number_t(int64_t(data[i].match_score));+ scores[1] = number_t(data[i].primary_attr);+ scores[2] = number_t(data[i].secondary_attr);++ topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);}topster.sort();@@ -77,8 +81,12 @@ TEST(TopsterTest, MaxFloatValues) {};for(int i = 0; i < 12; i++) {- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score,- number_t(data[i].primary_attr), number_t(data[i].secondary_attr));+ number_t scores[3];+ scores[0] = number_t(int64_t(data[i].match_score));+ scores[1] = number_t(data[i].primary_attr);+ scores[2] = number_t(data[i].secondary_attr);++ topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);}topster.sort();