Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Compute facet stats for numerical fields.
kishorenc8a24d047ec0a
diff --git a/include/collection.h b/include/collection.hindex 907693a2..6711dcee 100644--- a/include/collection.h+++ b/include/collection.h@@ -263,5 +263,9 @@ public:static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";static constexpr const char* SEQ_ID_PREFIX = "$SI";static constexpr const char* DOC_ID_PREFIX = "$DI";++ void compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount, double &fvsum,+ const facet_count_t &facet_count, const nlohmann::json &document,+ std::string &value);};diff --git a/src/collection.cpp b/src/collection.cppindex b0e27be8..6c8bfe25 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -745,6 +745,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::vector<facet_value_t> facet_values;+ double fvmin = std::numeric_limits<double>::max(),+ fvmax = std::numeric_limits<double>::min(),+ fvcount = 0,+ fvsum = 0;+for(size_t i = 0; i < max_facets; i++) {// remap facet value hash with actual stringauto & kv = facet_hash_counts[i];@@ -761,46 +766,12 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}std::string value;-- if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {- value = document[a_facet.field_name];- } else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {- value = document[a_facet.field_name][facet_count.array_pos];- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {- value = std::to_string(document[a_facet.field_name].get<int32_t>());- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<int32_t>());- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {- value = std::to_string(document[a_facet.field_name].get<int64_t>());- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<int64_t>());- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {- value = std::to_string(document[a_facet.field_name].get<float>());- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<float>());- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {- value = std::to_string(document[a_facet.field_name].get<bool>());- value = (value == "1") ? "true" : "false";- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());- value = (value == "1") ? "true" : "false";- }+ compute_facet_results(a_facet, fvmin, fvmax, fvcount, fvsum, facet_count, document, value);std::vector<std::string> tokens;StringUtils::split(value, tokens, " ");std::stringstream highlightedss;- /*std::cout << "array_pos: " << facet_count.array_pos << std::endl;-- for(auto xx: facet_count.query_token_pos) {- std::cout << xx.first << " -> " << xx.second.pos << " , " << xx.second.cost << std::endl;- }-- std::cout << "doc id: " << facet_count.doc_id << " i: " << i << std::endl;- std::cout << "doc: " << document << std::endl;*/-// invert query_pos -> token_posspp::sparse_hash_map<uint32_t, uint32_t> token_query_pos;for(auto qtoken_pos: facet_count.query_token_pos) {@@ -837,6 +808,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::facet_result["counts"].push_back(facet_value_count);}+ // add facet value stats+ facet_result["stats"] = nlohmann::json::object();+ if(fvcount != 0) {+ facet_result["stats"]["min"] = fvmin;+ facet_result["stats"]["max"] = fvmax;+ facet_result["stats"]["sum"] = fvsum;+ facet_result["stats"]["avg"] = (fvsum / fvcount);+ }+result["facet_counts"].push_back(facet_result);}@@ -846,6 +826,90 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::return result;}+void Collection::compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount,+ double &fvsum, const facet_count_t &facet_count,+ const nlohmann::json &document, std::string &value) {+ if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {+ value = document[a_facet.field_name];+ } else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {+ value = document[a_facet.field_name][facet_count.array_pos];+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {+ int32_t raw_val = document[a_facet.field_name].get<int32_t>();+ value = std::to_string(raw_val);+ if(raw_val < fvmin) {+ fvmin = raw_val;+ }+ if(raw_val > fvmax) {+ fvmax = raw_val;+ }+ fvsum += raw_val;+ fvcount++;+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {+ int32_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int32_t>();+ value = std::to_string(raw_val);+ if(raw_val < fvmin) {+ fvmin = raw_val;+ }+ if(raw_val > fvmax) {+ fvmax = raw_val;+ }+ fvsum += raw_val;+ fvcount++;+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {+ int64_t raw_val = document[a_facet.field_name].get<int64_t>();+ value = std::to_string(raw_val);+ if(raw_val < fvmin) {+ fvmin = raw_val;+ }+ if(raw_val > fvmax) {+ fvmax = raw_val;+ }+ fvsum += raw_val;+ fvcount++;+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {+ int64_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int64_t>();+ value = std::to_string(raw_val);+ if(raw_val < fvmin) {+ fvmin = raw_val;+ }+ if(raw_val > fvmax) {+ fvmax = raw_val;+ }+ fvsum += raw_val;+ fvcount++;+ } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {+ float raw_val = document[a_facet.field_name].get<float>();+ value = std::to_string(raw_val);+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros+ if(raw_val < fvmin) {+ fvmin = raw_val;+ }+ if(raw_val > fvmax) {+ fvmax = raw_val;+ }+ fvsum += raw_val;+ fvcount++;+ } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {+ float raw_val = document[a_facet.field_name][facet_count.array_pos].get<float>();+ value = std::to_string(raw_val);+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros+ if(raw_val < fvmin) {+ fvmin = raw_val;+ }+ if(raw_val > fvmax) {+ fvmax = raw_val;+ }+ fvsum += raw_val;+ fvcount++;+ } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {+ value = std::to_string(document[a_facet.field_name].get<bool>());+ value = (value == "1") ? "true" : "false";+ } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {+ value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());+ value = (value == "1") ? "true" : "false";+ }+}+void Collection::highlight_result(const field &search_field,const std::vector<std::vector<art_leaf *>> &searched_queries,const KV & field_order_kv, const nlohmann::json & document,diff --git a/test/collection_test.cpp b/test/collection_test.cppindex d9692ab4..e81087dd 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1823,9 +1823,10 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());- ASSERT_EQ(2, results["facet_counts"][0].size());+ ASSERT_EQ(3, results["facet_counts"][0].size());ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);ASSERT_EQ(4, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ(0, results["facet_counts"][0]["stats"].size());ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);@@ -1974,6 +1975,18 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_STREQ("21", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ // facet on a float field without query to check on stats+ results = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, 500, "").get();++ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());+ ASSERT_FLOAT_EQ(4.880199885368347, results["facet_counts"][0]["stats"]["avg"].get<double>());+ ASSERT_FLOAT_EQ(0.0, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(9.99899959564209, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(24.400999426841736, results["facet_counts"][0]["stats"]["sum"].get<double>());+// facet query on a float fieldresults = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, 0, 10, 1, FREQUENCY,false, Index::DROP_TOKENS_THRESHOLD,@@ -1987,7 +2000,14 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_STREQ("7.812", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_STREQ("<mark>7</mark>.812", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());- // facet query on a arrary integer field+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());+ ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["avg"].get<double>());+ ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(7.812, results["facet_counts"][0]["stats"]["sum"].get<double>());+++ // facet query on an array integer fieldresults = coll_array_fields->search("*", query_fields, "", {"timestamps"}, sort_fields, 0, 10, 1, FREQUENCY,false, Index::DROP_TOKENS_THRESHOLD,@@ -2002,6 +2022,12 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_STREQ("1421890022", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_STREQ("<mark>142189002</mark>2", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["avg"].get<double>());+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["sum"].get<double>());+// facet query that does not match any indexed valueresults = coll_array_fields->search("*", query_fields, "", {facets}, sort_fields, 0, 10, 1, FREQUENCY,false, Index::DROP_TOKENS_THRESHOLD,@@ -2209,6 +2235,7 @@ TEST_F(CollectionTest, FacetCountsBool) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ(0, results["facet_counts"][0]["stats"].size());ASSERT_STREQ("in_stock", results["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);