Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Ditch use of number_t for sorting.
kishorenc5e1c5f209361
diff --git a/include/index.h b/include/index.hindex 79d3879c..95d14736 100644--- a/include/index.h+++ b/include/index.h@@ -129,7 +129,7 @@ private:spp::sparse_hash_map<uint32_t, std::vector<std::vector<uint64_t>>> facet_index_v2;- spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, number_t>*> sort_index;+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;StringUtils string_utils;@@ -197,6 +197,7 @@ private:void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);+public:Index() = delete;@@ -276,5 +277,7 @@ public:const art_leaf *token_leaf, uint32_t doc_index);int get_bounded_typo_cost(const size_t max_cost, const size_t token_len) const;++ static int64_t float_to_in64_t(float n);};diff --git a/include/number.h b/include/number.hindex 5367815b..a3c32f9c 100644--- a/include/number.h+++ b/include/number.h@@ -2,6 +2,7 @@#include <sparsepp.h>+/*struct number_t {bool is_float;int64_t intval;@@ -69,4 +70,4 @@ struct number_t {return *this;}-};\ No newline at end of file+};*/diff --git a/include/topster.h b/include/topster.hindex 39fdd9dc..a356de50 100644--- a/include/topster.h+++ b/include/topster.h@@ -14,7 +14,7 @@ struct KV {uint16_t array_index;uint64_t key;uint64_t match_score;- number_t scores[3]; // match score + 2 custom attributes+ int64_t scores[3]; // match score + 2 custom attributes};/*@@ -58,7 +58,7 @@ struct Topster {}void add(const uint64_t &key, const uint8_t &field_id, const uint16_t &query_index, const uint64_t &match_score,- const number_t scores[3]) {+ const int64_t scores[3]) {if (size >= MAX_SIZE) {if(!is_greater(kvs[0], scores)) {// when incoming value is less than the smallest in the heap, ignore@@ -169,7 +169,7 @@ struct Topster {}}- static bool is_greater(const struct KV* i, const number_t scores[3]) {+ static bool is_greater(const struct KV* i, const int64_t scores[3]) {return std::tie(scores[0], scores[1], scores[2]) >std::tie(i->scores[0], i->scores[1], i->scores[2]);}diff --git a/src/index.cpp b/src/index.cppindex 3dc358b0..8347444b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -28,7 +28,7 @@ Index::Index(const std::string name, const std::unordered_map<std::string, field}for(const auto & pair: sort_schema) {- spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();sort_index.emplace(pair.first, doc_to_score);}@@ -74,6 +74,16 @@ int32_t Index::get_points_from_doc(const nlohmann::json &document, const std::streturn points;}+int64_t Index::float_to_in64_t(float f) {+ // https://stackoverflow.com/questions/60530255/convert-float-to-int64-t-while-preserving-ordering+ int32_t i;+ memcpy(&i, &f, sizeof i);+ if (i < 0) {+ i ^= INT32_MAX;+ }+ return i;+}+Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t seq_id,const std::string & default_sorting_field) {int32_t points = get_points_from_doc(document, default_sorting_field);@@ -177,12 +187,13 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t// add numerical values automatically into sort indexif(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL) {- spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(field_pair.first);if(field_pair.second.is_integer() ) {doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());} else if(field_pair.second.is_float()) {- doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());+ int64_t ifloat = float_to_in64_t(document[field_pair.first].get<float>());+ doc_to_score->emplace(seq_id, ifloat);} else if(field_pair.second.is_bool()) {doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());}@@ -1120,10 +1131,10 @@ void Index::collate_curated_ids(const std::string & query, const std::string & f}}- number_t scores[3];- scores[0] = number_t(int64_t(match_score));- scores[1] = number_t(int64_t(1));- scores[2] = number_t(int64_t(1));+ int64_t scores[3];+ scores[0] = int64_t(match_score);+ scores[1] = int64_t(1);+ scores[2] = int64_t(1);curated_topster.add(seq_id, field_id, searched_queries.size(), match_score, scores);@@ -1433,12 +1444,8 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16leaf_to_indices.emplace(token_leaf, indices);}- // Used for asc/desc ordering. NOTE: Topster keeps biggest keys (i.e. it's desc in nature)- number_t primary_rank_factor;- number_t secondary_rank_factor;-int sort_order[3]; // 1 or -1 based on DESC or ASC respectively- spp::sparse_hash_map<uint32_t, number_t>* field_values[3];+ spp::sparse_hash_map<uint32_t, int64_t>* field_values[3];for(size_t i = 0; i < sort_fields.size(); i++) {sort_order[i] = 1;@@ -1491,39 +1498,41 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}const int64_t default_score = 0;- number_t scores[3];+ int64_t scores[3];// avoiding loopif(sort_fields.size() > 0) {if (field_values[0] != nullptr) {auto it = field_values[0]->find(seq_id);- scores[0] = (it == field_values[0]->end()) ? number_t(default_score) : number_t(it->second);+ scores[0] = (it == field_values[0]->end()) ? default_score : it->second;} else {- scores[0] = number_t(int64_t(match_score));+ scores[0] = int64_t(match_score);}if (sort_order[0] == -1) {scores[0] = -scores[0];}+ scores[1] = 0;}if(sort_fields.size() > 1) {if (field_values[1] != nullptr) {auto it = field_values[1]->find(seq_id);- scores[1] = (it == field_values[1]->end()) ? number_t(default_score) : number_t(it->second);+ scores[1] = (it == field_values[1]->end()) ? default_score : it->second;} else {- scores[1] = number_t(int64_t(match_score));+ scores[1] = int64_t(match_score);}if (sort_order[1] == -1) {scores[1] = -scores[1];}+ scores[2] = 0;}if(sort_fields.size() > 2) {if(field_values[2] != nullptr) {auto it = field_values[2]->find(seq_id);- scores[2] = (it == field_values[2]->end()) ? number_t(default_score) : number_t(it->second);+ scores[2] = (it == field_values[2]->end()) ? default_score : it->second;} else {- scores[2] = number_t(int64_t(match_score));+ scores[2] = int64_t(match_score);}if(sort_order[2] == -1) {scores[2] = -scores[2];diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 17f0179f..6183564c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1506,7 +1506,7 @@ TEST_F(CollectionTest, SortOnFloatFields) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["document"]["id"];std::string id = ids.at(i);- EXPECT_STREQ(id.c_str(), result_id.c_str());+ ASSERT_STREQ(id.c_str(), result_id.c_str());}std::vector<sort_by> sort_fields_asc = { sort_by("score", "ASC"), sort_by("average", "ASC") };diff --git a/test/topster_test.cpp b/test/topster_test.cppindex e6f86f1d..1e9438a8 100644--- a/test/topster_test.cpp+++ b/test/topster_test.cpp@@ -1,4 +1,5 @@#include <gtest/gtest.h>+#include <index.h>#include "topster.h"#include "match_score.h"@@ -30,10 +31,10 @@ TEST(TopsterTest, MaxIntValues) {};for(int i = 0; i < 14; i++) {- number_t scores[3];- scores[0] = number_t(int64_t(data[i].match_score));- scores[1] = number_t(data[i].primary_attr);- scores[2] = number_t(data[i].secondary_attr);+ int64_t scores[3];+ scores[0] = int64_t(data[i].match_score);+ scores[1] = data[i].primary_attr;+ scores[2] = data[i].secondary_attr;topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);}@@ -81,10 +82,10 @@ TEST(TopsterTest, MaxFloatValues) {};for(int i = 0; i < 12; i++) {- number_t scores[3];- scores[0] = number_t(int64_t(data[i].match_score));- scores[1] = number_t(data[i].primary_attr);- scores[2] = number_t(data[i].secondary_attr);+ int64_t scores[3];+ scores[0] = int64_t(data[i].match_score);+ scores[1] = Index::float_to_in64_t(data[i].primary_attr);+ scores[2] = data[i].secondary_attr;topster.add(data[i].key, data[i].field_id, data[i].query_index, data[i].match_score, scores);}