Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Ditch use of number_t for sorting.

kishorenc5e1c5f209361
diff --git a/include/index.h b/include/index.h
index 79d3879c..95d14736 100644
--- a/include/index.h
+++ b/include/index.h
@@ -129,7 +129,7 @@ private:
spp::sparse_hash_map<uint32_t, std::vector<std::vector<uint64_t>>> facet_index_v2;
- spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;
+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;
StringUtils string_utils;
@@ -197,6 +197,7 @@ private:
void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);
+
public:
Index() = delete;
@@ -276,5 +277,7 @@ public:
const art_leaf *token_leaf, uint32_t doc_index);
int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;
+
+ static int64_t float_to_in64_t(float n);
};
diff --git a/include/number.h b/include/number.h
index 5367815b..a3c32f9c 100644
--- a/include/number.h
+++ b/include/number.h
@@ -2,6 +2,7 @@
#include <sparsepp.h>
+/*
struct number_t {
bool is_float;
int64_t intval;
@@ -69,4 +70,4 @@ struct number_t {
return *this;
}
-};
\ No newline at end of file
+};*/
diff --git a/include/topster.h b/include/topster.h
index 39fdd9dc..a356de50 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -14,7 +14,7 @@ struct KV {
uint16_t array_index;
uint64_t key;
uint64_t match_score;
- number_t scores[3]; // match score + 2 custom attributes
+ int64_t scores[3]; // match score + 2 custom attributes
};
/*
@@ -58,7 +58,7 @@ struct Topster {
}
void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,
- const number_t scores[3]) {
+ const int64_t scores[3]) {
if (size >= MAX_SIZE) {
if(!is_greater(kvs[0], scores)) {
// when incoming value is less than the smallest in the heap, ignore
@@ -169,7 +169,7 @@ struct Topster {
}
}
- static bool is_greater(const struct KV* i, const number_t scores[3]) {
+ static bool is_greater(const struct KV* i, const int64_t scores[3]) {
return std::tie(scores[0], scores[1], scores[2]) >
std::tie(i->scores[0], i->scores[1], i->scores[2]);
}
diff --git a/src/index.cpp b/src/index.cpp
index 3dc358b0..8347444b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -28,7 +28,7 @@ Index::Index(const std::string name, const std::unordered_map<std::string, field
}
for(const auto & pair: sort_schema) {
- spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();
sort_index.emplace(pair.first, doc_to_score);
}
@@ -74,6 +74,16 @@ int32_t Index::get_points_from_doc(const nlohmann::json &document, const std::st
return points;
}
+int64_t Index::float_to_in64_t(float f) {
+ // https://stackoverflow.com/questions/60530255/convert-float-to-int64-t-while-preserving-ordering
+ int32_t i;
+ memcpy(&i, &f, sizeof i);
+ if (i < 0) {
+ i ^= INT32_MAX;
+ }
+ return i;
+}
+
Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,
const std::string & default_sorting_field) {
int32_t points = get_points_from_doc(document, default_sorting_field);
@@ -177,12 +187,13 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
// add numerical values automatically into sort index
if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||
field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL) {
- spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);
+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(field_pair.first);
if(field_pair.second.is_integer() ) {
doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());
} else if(field_pair.second.is_float()) {
- doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());
+ int64_t ifloat = float_to_in64_t(document[field_pair.first].get<float>());
+ doc_to_score->emplace(seq_id, ifloat);
} else if(field_pair.second.is_bool()) {
doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());
}
@@ -1120,10 +1131,10 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f
}
}
- number_t scores[3];
- scores[0] = number_t(int64_t(match_score));
- scores[1] = number_t(int64_t(1));
- scores[2] = number_t(int64_t(1));
+ int64_t scores[3];
+ scores[0] = int64_t(match_score);
+ scores[1] = int64_t(1);
+ scores[2] = int64_t(1);
curated_topster.add(seq_id, field_id, searched_queries.size(), match_score, scores);
@@ -1433,12 +1444,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
leaf_to_indices.emplace(token_leaf, indices);
}
- // Used for asc/desc ordering. NOTE: Topster keeps biggest keys (i.e. it's desc in nature)
- number_t primary_rank_factor;
- number_t secondary_rank_factor;
-
int sort_order[3]; // 1 or -1 based on DESC or ASC respectively
- spp::sparse_hash_map<uint32_t, number_t>* field_values[3];
+ spp::sparse_hash_map<uint32_t, int64_t>* field_values[3];
for(size_t i = 0; i < sort_fields.size(); i++) {
sort_order[i] = 1;
@@ -1491,39 +1498,41 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
const int64_t default_score = 0;
- number_t scores[3];
+ int64_t scores[3];
// avoiding loop
if(sort_fields.size() > 0) {
if (field_values[0] != nullptr) {
auto it = field_values[0]->find(seq_id);
- scores[0] = (it == field_values[0]->end()) ? number_t(default_score) : number_t(it->second);
+ scores[0] = (it == field_values[0]->end()) ? default_score : it->second;
} else {
- scores[0] = number_t(int64_t(match_score));
+ scores[0] = int64_t(match_score);
}
if (sort_order[0] == -1) {
scores[0] = -scores[0];
}
+ scores[1] = 0;
}
if(sort_fields.size() > 1) {
if (field_values[1] != nullptr) {
auto it = field_values[1]->find(seq_id);
- scores[1] = (it == field_values[1]->end()) ? number_t(default_score) : number_t(it->second);
+ scores[1] = (it == field_values[1]->end()) ? default_score : it->second;
} else {
- scores[1] = number_t(int64_t(match_score));
+ scores[1] = int64_t(match_score);
}
if (sort_order[1] == -1) {
scores[1] = -scores[1];
}
+ scores[2] = 0;
}
if(sort_fields.size() > 2) {
if(field_values[2] != nullptr) {
auto it = field_values[2]->find(seq_id);
- scores[2] = (it == field_values[2]->end()) ? number_t(default_score) : number_t(it->second);
+ scores[2] = (it == field_values[2]->end()) ? default_score : it->second;
} else {
- scores[2] = number_t(int64_t(match_score));
+ scores[2] = int64_t(match_score);
}
if(sort_order[2] == -1) {
scores[2] = -scores[2];
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 17f0179f..6183564c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1506,7 +1506,7 @@ TEST_F(CollectionTest, SortOnFloatFields) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["document"]["id"];
std::string id = ids.at(i);
- EXPECT_STREQ(id.c_str(), result_id.c_str());
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
}
std::vector<sort_by> sort_fields_asc = { sort_by("score", "ASC"), sort_by("average", "ASC") };
diff --git a/test/topster_test.cpp b/test/topster_test.cpp
index e6f86f1d..1e9438a8 100644
--- a/test/topster_test.cpp
+++ b/test/topster_test.cpp
@@ -1,4 +1,5 @@
#include <gtest/gtest.h>
+#include <index.h>
#include "topster.h"
#include "match_score.h"
@@ -30,10 +31,10 @@ TEST(TopsterTest, MaxIntValues) {
};
for(int i = 0; i < 14; i++) {
- number_t scores[3];
- scores[0] = number_t(int64_t(data[i].match_score));
- scores[1] = number_t(data[i].primary_attr);
- scores[2] = number_t(data[i].secondary_attr);
+ int64_t scores[3];
+ scores[0] = int64_t(data[i].match_score);
+ scores[1] = data[i].primary_attr;
+ scores[2] = data[i].secondary_attr;
topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);
}
@@ -81,10 +82,10 @@ TEST(TopsterTest, MaxFloatValues) {
};
for(int i = 0; i < 12; i++) {
- number_t scores[3];
- scores[0] = number_t(int64_t(data[i].match_score));
- scores[1] = number_t(data[i].primary_attr);
- scores[2] = number_t(data[i].secondary_attr);
+ int64_t scores[3];
+ scores[0] = int64_t(data[i].match_score);
+ scores[1] = Index::float_to_in64_t(data[i].primary_attr);
+ scores[2] = data[i].secondary_attr;
topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);
}