Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Compute facet stats for numerical fields.

kishorenc8a24d047ec0a
diff --git a/include/collection.h b/include/collection.h
index 907693a2..6711dcee 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -263,5 +263,9 @@ public:
static constexpr const char* COLLECTION_OVERRIDE_PREFIX = "$CO";
static constexpr const char* SEQ_ID_PREFIX = "$SI";
static constexpr const char* DOC_ID_PREFIX = "$DI";
+
+ void compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount, double &fvsum,
+ const facet_count_t &facet_count, const nlohmann::json &document,
+ std::string &value);
};
diff --git a/src/collection.cpp b/src/collection.cpp
index b0e27be8..6c8bfe25 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -745,6 +745,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::vector<facet_value_t> facet_values;
+ double fvmin = std::numeric_limits<double>::max(),
+ fvmax = std::numeric_limits<double>::min(),
+ fvcount = 0,
+ fvsum = 0;
+
for(size_t i = 0; i < max_facets; i++) {
// remap facet value hash with actual string
auto & kv = facet_hash_counts[i];
@@ -761,46 +766,12 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
std::string value;
-
- if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {
- value = document[a_facet.field_name];
- } else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {
- value = document[a_facet.field_name][facet_count.array_pos];
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {
- value = std::to_string(document[a_facet.field_name].get<int32_t>());
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {
- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<int32_t>());
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {
- value = std::to_string(document[a_facet.field_name].get<int64_t>());
- } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {
- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<int64_t>());
- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {
- value = std::to_string(document[a_facet.field_name].get<float>());
- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
- } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {
- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<float>());
- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {
- value = std::to_string(document[a_facet.field_name].get<bool>());
- value = (value == "1") ? "true" : "false";
- } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {
- value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());
- value = (value == "1") ? "true" : "false";
- }
+ compute_facet_results(a_facet, fvmin, fvmax, fvcount, fvsum, facet_count, document, value);
std::vector<std::string> tokens;
StringUtils::split(value, tokens, " ");
std::stringstream highlightedss;
- /*std::cout << "array_pos: " << facet_count.array_pos << std::endl;
-
- for(auto xx: facet_count.query_token_pos) {
- std::cout << xx.first << " -> " << xx.second.pos << " , " << xx.second.cost << std::endl;
- }
-
- std::cout << "doc id: " << facet_count.doc_id << " i: " << i << std::endl;
- std::cout << "doc: " << document << std::endl;*/
-
// invert query_pos -> token_pos
spp::sparse_hash_map<uint32_t, uint32_t> token_query_pos;
for(auto qtoken_pos: facet_count.query_token_pos) {
@@ -837,6 +808,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
facet_result["counts"].push_back(facet_value_count);
}
+ // add facet value stats
+ facet_result["stats"] = nlohmann::json::object();
+ if(fvcount != 0) {
+ facet_result["stats"]["min"] = fvmin;
+ facet_result["stats"]["max"] = fvmax;
+ facet_result["stats"]["sum"] = fvsum;
+ facet_result["stats"]["avg"] = (fvsum / fvcount);
+ }
+
result["facet_counts"].push_back(facet_result);
}
@@ -846,6 +826,90 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return result;
}
+void Collection::compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount,
+ double &fvsum, const facet_count_t &facet_count,
+ const nlohmann::json &document, std::string &value) {
+ if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {
+ value = document[a_facet.field_name];
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {
+ value = document[a_facet.field_name][facet_count.array_pos];
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {
+ int32_t raw_val = document[a_facet.field_name].get<int32_t>();
+ value = std::to_string(raw_val);
+ if(raw_val < fvmin) {
+ fvmin = raw_val;
+ }
+ if(raw_val > fvmax) {
+ fvmax = raw_val;
+ }
+ fvsum += raw_val;
+ fvcount++;
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {
+ int32_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int32_t>();
+ value = std::to_string(raw_val);
+ if(raw_val < fvmin) {
+ fvmin = raw_val;
+ }
+ if(raw_val > fvmax) {
+ fvmax = raw_val;
+ }
+ fvsum += raw_val;
+ fvcount++;
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {
+ int64_t raw_val = document[a_facet.field_name].get<int64_t>();
+ value = std::to_string(raw_val);
+ if(raw_val < fvmin) {
+ fvmin = raw_val;
+ }
+ if(raw_val > fvmax) {
+ fvmax = raw_val;
+ }
+ fvsum += raw_val;
+ fvcount++;
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {
+ int64_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int64_t>();
+ value = std::to_string(raw_val);
+ if(raw_val < fvmin) {
+ fvmin = raw_val;
+ }
+ if(raw_val > fvmax) {
+ fvmax = raw_val;
+ }
+ fvsum += raw_val;
+ fvcount++;
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {
+ float raw_val = document[a_facet.field_name].get<float>();
+ value = std::to_string(raw_val);
+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
+ if(raw_val < fvmin) {
+ fvmin = raw_val;
+ }
+ if(raw_val > fvmax) {
+ fvmax = raw_val;
+ }
+ fvsum += raw_val;
+ fvcount++;
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {
+ float raw_val = document[a_facet.field_name][facet_count.array_pos].get<float>();
+ value = std::to_string(raw_val);
+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
+ if(raw_val < fvmin) {
+ fvmin = raw_val;
+ }
+ if(raw_val > fvmax) {
+ fvmax = raw_val;
+ }
+ fvsum += raw_val;
+ fvcount++;
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {
+ value = std::to_string(document[a_facet.field_name].get<bool>());
+ value = (value == "1") ? "true" : "false";
+ } else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL_ARRAY) {
+ value = std::to_string(document[a_facet.field_name][facet_count.array_pos].get<bool>());
+ value = (value == "1") ? "true" : "false";
+ }
+}
+
void Collection::highlight_result(const field &search_field,
const std::vector<std::vector<art_leaf *>> &searched_queries,
const KV & field_order_kv, const nlohmann::json & document,
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index d9692ab4..e81087dd 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1823,9 +1823,10 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ(1, results["facet_counts"].size());
- ASSERT_EQ(2, results["facet_counts"][0].size());
+ ASSERT_EQ(3, results["facet_counts"][0].size());
ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
ASSERT_EQ(4, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ(0, results["facet_counts"][0]["stats"].size());
ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);
@@ -1974,6 +1975,18 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_STREQ("21", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ // facet on a float field without query to check on stats
+ results = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, 0, 10, 1, FREQUENCY,
+ false, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, 500, "").get();
+
+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
+ ASSERT_FLOAT_EQ(4.880199885368347, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(0.0, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(9.99899959564209, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(24.400999426841736, results["facet_counts"][0]["stats"]["sum"].get<double>());
+
// facet query on a float field
results = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, 0, 10, 1, FREQUENCY,
false, Index::DROP_TOKENS_THRESHOLD,
@@ -1987,7 +2000,14 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_STREQ("7.812", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_STREQ("<mark>7</mark>.812", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
- // facet query on a arrary integer field
+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
+ ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(7.812, results["facet_counts"][0]["stats"]["sum"].get<double>());
+
+
+ // facet query on an array integer field
results = coll_array_fields->search("*", query_fields, "", {"timestamps"}, sort_fields, 0, 10, 1, FREQUENCY,
false, Index::DROP_TOKENS_THRESHOLD,
@@ -2002,6 +2022,12 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_STREQ("1421890022", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
ASSERT_STREQ("<mark>142189002</mark>2", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["sum"].get<double>());
+
// facet query that does not match any indexed value
results = coll_array_fields->search("*", query_fields, "", {facets}, sort_fields, 0, 10, 1, FREQUENCY,
false, Index::DROP_TOKENS_THRESHOLD,
@@ -2209,6 +2235,7 @@ TEST_F(CollectionTest, FacetCountsBool) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ(0, results["facet_counts"][0]["stats"].size());
ASSERT_STREQ("in_stock", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);