Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Sort results on custom order.

kishorenceed10d554d55
diff --git a/include/field.h b/include/field.h
index a19c2dda..03b88467 100644
--- a/include/field.h
+++ b/include/field.h
@@ -113,6 +113,7 @@ namespace sort_field_const {
static const std::string order = "order";
static const std::string asc = "ASC";
static const std::string desc = "DESC";
+ static const std::string match_score = "_match_score";
}
struct sort_by {
diff --git a/include/topster.h b/include/topster.h
index 21d938d3..39fdd9dc 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -14,8 +14,7 @@ struct KV {
uint16_t array_index;
uint64_t key;
uint64_t match_score;
- number_t primary_attr;
- number_t secondary_attr;
+ number_t scores[3]; // match score + 2 custom attributes
};
/*
@@ -39,8 +38,6 @@ struct Topster {
data[i].query_index = 0;
data[i].key = 0;
data[i].match_score = 0;
- data[i].primary_attr = number_t();
- data[i].secondary_attr = number_t();
kvs[i] = &data[i];
}
}
@@ -61,9 +58,9 @@ struct Topster {
}
void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,
- const number_t &primary_attr, const number_t &secondary_attr) {
+ const number_t scores[3]) {
if (size >= MAX_SIZE) {
- if(!is_greater(kvs[0], match_score, primary_attr, secondary_attr)) {
+ if(!is_greater(kvs[0], scores)) {
// when incoming value is less than the smallest in the heap, ignore
return;
}
@@ -90,8 +87,9 @@ struct Topster {
kvs[start]->query_index = query_index;
kvs[start]->array_index = start;
kvs[start]->match_score = match_score;
- kvs[start]->primary_attr = primary_attr;
- kvs[start]->secondary_attr = secondary_attr;
+ kvs[start]->scores[0] = scores[0];
+ kvs[start]->scores[1] = scores[1];
+ kvs[start]->scores[2] = scores[2];
// sift down to maintain heap property
while ((2*start+1) < MAX_SIZE) {
@@ -129,8 +127,9 @@ struct Topster {
kvs[start]->query_index = query_index;
kvs[start]->array_index = start;
kvs[start]->match_score = match_score;
- kvs[start]->primary_attr = primary_attr;
- kvs[start]->secondary_attr = secondary_attr;
+ kvs[start]->scores[0] = scores[0];
+ kvs[start]->scores[1] = scores[1];
+ kvs[start]->scores[2] = scores[2];
keys[key] = kvs[start];
@@ -170,19 +169,19 @@ struct Topster {
}
}
- static bool is_greater(const struct KV* i, uint64_t match_score, number_t primary_attr, number_t secondary_attr) {
- return std::tie(match_score, primary_attr, secondary_attr) >
- std::tie(i->match_score, i->primary_attr, i->secondary_attr);
+ static bool is_greater(const struct KV* i, const number_t scores[3]) {
+ return std::tie(scores[0], scores[1], scores[2]) >
+ std::tie(i->scores[0], i->scores[1], i->scores[2]);
}
static bool is_greater_kv(const struct KV* i, const struct KV* j) {
- return std::tie(i->match_score, i->primary_attr, i->secondary_attr, i->key) >
- std::tie(j->match_score, j->primary_attr, j->secondary_attr, j->key);
+ return std::tie(i->scores[0], i->scores[1], i->scores[2], i->key) >
+ std::tie(j->scores[0], j->scores[1], j->scores[2], j->key);
}
static bool is_greater_kv_value(const struct KV & i, const struct KV & j) {
- return std::tie(i.match_score, i.primary_attr, i.secondary_attr, i.key) >
- std::tie(j.match_score, j.primary_attr, j.secondary_attr, j.key);
+ return std::tie(i.scores[0], i.scores[1], i.scores[2], i.key) >
+ std::tie(j.scores[0], j.scores[1], j.scores[2], j.key);
}
// topster must be sorted before iterated upon to remove dead array entries
diff --git a/src/collection.cpp b/src/collection.cpp
index e01d9485..52b298f6 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -508,7 +508,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::vector<sort_by> sort_fields_std;
for(const sort_by & _sort_field: sort_fields) {
- if(sort_schema.count(_sort_field.name) == 0) {
+ if(_sort_field.name != sort_field_const::match_score && sort_schema.count(_sort_field.name) == 0) {
std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema for sorting.";
return Option<nlohmann::json>(404, error);
}
@@ -524,10 +524,29 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
sort_fields_std.emplace_back(_sort_field.name, sort_order);
}
+ /*
+ 1. Empty: [match_score, dsf] upstream
+ 2. ONE : [usf, match_score]
+ 3. TWO : [usf1, usf2, match_score]
+ 4. THREE: do nothing
+ */
if(sort_fields_std.empty()) {
+ sort_fields_std.emplace_back(sort_field_const::match_score, sort_field_const::desc);
sort_fields_std.emplace_back(default_sorting_field, sort_field_const::desc);
}
+ bool found_match_score = false;
+ for(const auto & sort_field : sort_fields) {
+ if(sort_field.name == sort_field_const::match_score) {
+ found_match_score = true;
+ break;
+ }
+ }
+
+ if(!found_match_score && sort_fields.size() < 3) {
+ sort_fields_std.emplace_back(sort_field_const::match_score, sort_field_const::desc);
+ }
+
// check for valid pagination
if(page < 1) {
std::string message = "Page must be an integer of value greater than 0.";
diff --git a/src/index.cpp b/src/index.cpp
index d1abe316..3dc358b0 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -465,8 +465,8 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &token
uint64_t hash = 0;
if(a_field.is_float()) {
- float f = atof(token.c_str());
- hash = *reinterpret_cast<uint64_t*>(&f); // store as int without loss of precision
+ float f = std::stof(token);
+ reinterpret_cast<float&>(hash) = f; // store as int without loss of precision
} else if(a_field.is_integer() || a_field.is_bool()) {
hash = atol(token.c_str());
} else {
@@ -608,7 +608,7 @@ void Index::compute_facet_stats(facet &a_facet, int64_t raw_value, const std::st
a_facet.stats.fvsum += val;
a_facet.stats.fvcount++;
} else if(field_type == field_types::FLOAT || field_type == field_types::FLOAT_ARRAY) {
- float val = *reinterpret_cast<float*>(&raw_value);
+ float val = reinterpret_cast<float&>(raw_value);
if(val < a_facet.stats.fvmin) {
a_facet.stats.fvmin = val;
}
@@ -1120,8 +1120,12 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f
}
}
- curated_topster.add(seq_id, field_id, searched_queries.size(), match_score,
- number_t(int64_t(1)), number_t(int64_t(1)));
+ number_t scores[3];
+ scores[0] = number_t(int64_t(match_score));
+ scores[1] = number_t(int64_t(1));
+ scores[2] = number_t(int64_t(1));
+
+ curated_topster.add(seq_id, field_id, searched_queries.size(), match_score, scores);
searched_queries.push_back(override_query);
}
@@ -1416,8 +1420,8 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
}
}
-void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,
- const uint32_t total_cost, Topster & topster,
+void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,
+ const uint8_t & field_id, const uint32_t total_cost, Topster & topster,
const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, const size_t result_size) const {
@@ -1429,44 +1433,22 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
leaf_to_indices.emplace(token_leaf, indices);
}
- spp::sparse_hash_map<uint32_t, number_t> * primary_rank_scores = nullptr;
- spp::sparse_hash_map<uint32_t, number_t> * secondary_rank_scores = nullptr;
-
// Used for asc/desc ordering. NOTE: Topster keeps biggest keys (i.e. it's desc in nature)
number_t primary_rank_factor;
number_t secondary_rank_factor;
- if(sort_fields.size() > 0) {
- // assumed that rank field exists in the index - checked earlier in the chain
- primary_rank_scores = sort_index.at(sort_fields[0].name);
-
- // initialize primary_rank_factor
- field sort_field = sort_schema.at(sort_fields[0].name);
- if(sort_field.is_single_integer() || sort_field.is_single_bool()) {
- primary_rank_factor = ((int64_t) 1);
- } else {
- primary_rank_factor = ((float) 1);
- }
+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
+ spp::sparse_hash_map<uint32_t, number_t>* field_values[3];
- if(sort_fields[0].order == sort_field_const::asc) {
- primary_rank_factor = -primary_rank_factor;
+ for(size_t i = 0; i < sort_fields.size(); i++) {
+ sort_order[i] = 1;
+ if(sort_fields[i].order == sort_field_const::asc) {
+ sort_order[i] = -1;
}
- }
- if(sort_fields.size() > 1) {
- secondary_rank_scores = sort_index.at(sort_fields[1].name);
-
- // initialize secondary_rank_factor
- field sort_field = sort_schema.at(sort_fields[1].name);
- if(sort_field.is_single_integer() || sort_field.is_single_bool()) {
- secondary_rank_factor = ((int64_t) 1);
- } else {
- secondary_rank_factor = ((float) 1);
- }
-
- if(sort_fields[1].order == sort_field_const::asc) {
- secondary_rank_factor = -secondary_rank_factor;
- }
+ field_values[i] = (sort_fields[i].name != sort_field_const::match_score) ?
+ sort_index.at(sort_fields[i].name) :
+ nullptr;
}
//auto begin = std::chrono::high_resolution_clock::now();
@@ -1509,22 +1491,46 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
const int64_t default_score = 0;
- number_t primary_rank_score = number_t(default_score);
- number_t secondary_rank_score = number_t(default_score);
+ number_t scores[3];
- if(primary_rank_scores) {
- auto it = primary_rank_scores->find(seq_id);
- primary_rank_score = (it == primary_rank_scores->end()) ? number_t(default_score) : number_t(it->second);
+ // avoiding loop
+ if(sort_fields.size() > 0) {
+ if (field_values[0] != nullptr) {
+ auto it = field_values[0]->find(seq_id);
+ scores[0] = (it == field_values[0]->end()) ? number_t(default_score) : number_t(it->second);
+ } else {
+ scores[0] = number_t(int64_t(match_score));
+ }
+ if (sort_order[0] == -1) {
+ scores[0] = -scores[0];
+ }
}
- if(secondary_rank_scores) {
- auto it = secondary_rank_scores->find(seq_id);
- secondary_rank_score = (it == secondary_rank_scores->end()) ? number_t(default_score) : number_t(it->second);
+ if(sort_fields.size() > 1) {
+ if (field_values[1] != nullptr) {
+ auto it = field_values[1]->find(seq_id);
+ scores[1] = (it == field_values[1]->end()) ? number_t(default_score) : number_t(it->second);
+ } else {
+ scores[1] = number_t(int64_t(match_score));
+ }
+ if (sort_order[1] == -1) {
+ scores[1] = -scores[1];
+ }
+ }
+
+ if(sort_fields.size() > 2) {
+ if(field_values[2] != nullptr) {
+ auto it = field_values[2]->find(seq_id);
+ scores[2] = (it == field_values[2]->end()) ? number_t(default_score) : number_t(it->second);
+ } else {
+ scores[2] = number_t(int64_t(match_score));
+ }
+ if(sort_order[2] == -1) {
+ scores[2] = -scores[2];
+ }
}
- const number_t & primary_rank_value = primary_rank_score * primary_rank_factor;
- const number_t & secondary_rank_value = secondary_rank_score * secondary_rank_factor;
- topster.add(seq_id, field_id, query_index, match_score, primary_rank_value, secondary_rank_value);
+ topster.add(seq_id, field_id, query_index, match_score, scores);
}
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 2c70ef6c..17f0179f 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -30,7 +30,7 @@ protected:
};
query_fields = {"title"};
- sort_fields = { sort_by("points", "DESC") };
+ sort_fields = { sort_by("_match_score", "DESC"), sort_by("points", "DESC") };
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
@@ -149,7 +149,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
"What is the power requirement of a <mark>rocket</mark> <mark>launch</mark> these days?");
// Check ASC sort order
- std::vector<sort_by> sort_fields_asc = { sort_by("points", "ASC") };
+ std::vector<sort_by> sort_fields_asc = { sort_by("_match_score", "DESC"), sort_by("points", "ASC") };
results = collection->search("rocket launch", query_fields, "", facets, sort_fields_asc, 0, 10).get();
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ(5, results["found"].get<uint32_t>());
@@ -2802,7 +2802,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {
field("age", field_types::INT32, false),
};
- std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
+ std::vector<sort_by> sort_fields = { sort_by("_match_score", "DESC"), sort_by("age", "DESC") };
coll_large_text = collectionManager.get_collection("coll_large_text");
if(coll_large_text == nullptr) {
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index 5704e8fb..e6f86f1d 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -30,8 +30,12 @@ TEST(TopsterTest, MaxIntValues) {
};
for(int i = 0; i < 14; i++) {
- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score,
- number_t(data[i].primary_attr), number_t(data[i].secondary_attr));
+ number_t scores[3];
+ scores[0] = number_t(int64_t(data[i].match_score));
+ scores[1] = number_t(data[i].primary_attr);
+ scores[2] = number_t(data[i].secondary_attr);
+
+ topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);
}
topster.sort();
@@ -77,8 +81,12 @@ TEST(TopsterTest, MaxFloatValues) {
};
for(int i = 0; i < 12; i++) {
- topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score,
- number_t(data[i].primary_attr), number_t(data[i].secondary_attr));
+ number_t scores[3];
+ scores[0] = number_t(int64_t(data[i].match_score));
+ scores[1] = number_t(data[i].primary_attr);
+ scores[2] = number_t(data[i].secondary_attr);
+
+ topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);
}
topster.sort();