Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Compute stats on all facet values regardless of facet query.

kishorenc31e9e4719a26
diff --git a/include/collection.h b/include/collection.h
index da3043ba..71291a47 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -262,8 +262,7 @@ public:
static constexpr const char* SEQ_ID_PREFIX = "$SI";
static constexpr const char* DOC_ID_PREFIX = "$DI";
- void compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount, double &fvsum,
- const facet_count_t &facet_count, const nlohmann::json &document,
+ void facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,
std::string &value);
};
diff --git a/include/field.h b/include/field.h
index 6a27ae80..a19c2dda 100644
--- a/include/field.h
+++ b/include/field.h
@@ -145,9 +145,17 @@ struct facet_count_t {
spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_pos;
};
+struct facet_stats_t {
+ double fvmin = std::numeric_limits<double>::max(),
+ fvmax = -std::numeric_limits<double>::min(),
+ fvcount = 0,
+ fvsum = 0;
+};
+
struct facet {
const std::string field_name;
std::map<uint64_t, facet_count_t> result_map;
+ facet_stats_t stats;
facet(const std::string & field_name): field_name(field_name) {
diff --git a/include/index.h b/include/index.h
index 1e019997..79d3879c 100644
--- a/include/index.h
+++ b/include/index.h
@@ -113,6 +113,8 @@ struct batch_index_result {
class Index {
private:
+ const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();
+
std::string name;
size_t num_documents;
@@ -163,10 +165,10 @@ private:
const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;
void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,
- int facet_id);
+ int facet_id, const field & a_field);
void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
- uint32_t seq_id, int facet_id);
+ uint32_t seq_id, int facet_id, const field & a_field);
void index_int32_field(const int32_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
@@ -191,6 +193,10 @@ private:
const std::vector<uint32_t> & included_ids,
Topster & curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);
+ uint64_t facet_token_hash(const field & a_field, const std::string &token);
+
+ void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);
+
public:
Index() = delete;
diff --git a/include/string_utils.h b/include/string_utils.h
index ff4c1057..8472f42a 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -220,8 +220,9 @@ struct StringUtils {
return seq_id;
}
- static int64_t hash_wy(const void* key, uint64_t len) {
+ static uint64_t hash_wy(const void* key, uint64_t len) {
uint64_t hash = wyhash(key, len, 0, _wyp);
- return hash != 0 ? hash : 1; // reserve 0 for use as a delimiter
+ // reserve max() for use as a delimiter
+ return hash != std::numeric_limits<uint64_t>::max() ? hash : (std::numeric_limits<uint64_t>::max()-1);
}
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 0b39d3e2..76295ef8 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -614,6 +614,13 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
acc_facet.result_map[facet_kv.first].query_token_pos = facet_kv.second.query_token_pos;
}
+
+ if(this_facet.stats.fvcount != 0) {
+ acc_facet.stats.fvcount += this_facet.stats.fvcount;
+ acc_facet.stats.fvsum += this_facet.stats.fvsum;
+ acc_facet.stats.fvmax = std::max(acc_facet.stats.fvmax, this_facet.stats.fvmax);
+ acc_facet.stats.fvmin = std::min(acc_facet.stats.fvmin, this_facet.stats.fvmin);
+ }
}
total_found += index->search_params.all_result_ids_len;
@@ -731,9 +738,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
facet_result["field_name"] = a_facet.field_name;
facet_result["counts"] = nlohmann::json::array();
- std::vector<std::pair<uint64_t, facet_count_t>> facet_hash_counts;
- for (const auto & itr : a_facet.result_map) {
- facet_hash_counts.emplace_back(itr);
+ std::vector<std::pair<int64_t, facet_count_t>> facet_hash_counts;
+ for (const auto & kv : a_facet.result_map) {
+ facet_hash_counts.emplace_back(kv);
}
// keep only top K facets
@@ -747,14 +754,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
std::vector<facet_value_t> facet_values;
- double fvmin = std::numeric_limits<double>::max(),
- fvmax = std::numeric_limits<double>::min(),
- fvcount = 0,
- fvsum = 0;
-
- for(size_t i = 0; i < max_facets; i++) {
+ for(size_t fi = 0; fi < max_facets; fi++) {
// remap facet value hash with actual string
- auto & kv = facet_hash_counts[i];
+ auto & kv = facet_hash_counts[fi];
auto & facet_count = kv.second;
// fetch actual facet value from representative doc id
@@ -768,7 +770,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
std::string value;
- compute_facet_results(a_facet, fvmin, fvmax, fvcount, fvsum, facet_count, document, value);
+ facet_value_to_string(a_facet, facet_count, document, value);
std::vector<std::string> tokens;
StringUtils::split(value, tokens, " ");
@@ -812,11 +814,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
// add facet value stats
facet_result["stats"] = nlohmann::json::object();
- if(fvcount != 0) {
- facet_result["stats"]["min"] = fvmin;
- facet_result["stats"]["max"] = fvmax;
- facet_result["stats"]["sum"] = fvsum;
- facet_result["stats"]["avg"] = (fvsum / fvcount);
+ if(a_facet.stats.fvcount != 0) {
+ facet_result["stats"]["min"] = a_facet.stats.fvmin;
+ facet_result["stats"]["max"] = a_facet.stats.fvmax;
+ facet_result["stats"]["sum"] = a_facet.stats.fvsum;
+ facet_result["stats"]["avg"] = (a_facet.stats.fvsum / a_facet.stats.fvcount);
}
result["facet_counts"].push_back(facet_result);
@@ -828,9 +830,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return result;
}
-void Collection::compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount,
- double &fvsum, const facet_count_t &facet_count,
+void Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,
const nlohmann::json &document, std::string &value) {
+
if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {
value = document[a_facet.field_name];
} else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {
@@ -838,71 +840,23 @@ void Collection::compute_facet_results(const facet &a_facet, double &fvmin, doub
} else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {
int32_t raw_val = document[a_facet.field_name].get<int32_t>();
value = std::to_string(raw_val);
- if(raw_val < fvmin) {
- fvmin = raw_val;
- }
- if(raw_val > fvmax) {
- fvmax = raw_val;
- }
- fvsum += raw_val;
- fvcount++;
} else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {
int32_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int32_t>();
value = std::to_string(raw_val);
- if(raw_val < fvmin) {
- fvmin = raw_val;
- }
- if(raw_val > fvmax) {
- fvmax = raw_val;
- }
- fvsum += raw_val;
- fvcount++;
} else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {
int64_t raw_val = document[a_facet.field_name].get<int64_t>();
value = std::to_string(raw_val);
- if(raw_val < fvmin) {
- fvmin = raw_val;
- }
- if(raw_val > fvmax) {
- fvmax = raw_val;
- }
- fvsum += raw_val;
- fvcount++;
} else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {
int64_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int64_t>();
value = std::to_string(raw_val);
- if(raw_val < fvmin) {
- fvmin = raw_val;
- }
- if(raw_val > fvmax) {
- fvmax = raw_val;
- }
- fvsum += raw_val;
- fvcount++;
} else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {
float raw_val = document[a_facet.field_name].get<float>();
value = std::to_string(raw_val);
value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
- if(raw_val < fvmin) {
- fvmin = raw_val;
- }
- if(raw_val > fvmax) {
- fvmax = raw_val;
- }
- fvsum += raw_val;
- fvcount++;
} else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {
float raw_val = document[a_facet.field_name][facet_count.array_pos].get<float>();
value = std::to_string(raw_val);
value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
- if(raw_val < fvmin) {
- fvmin = raw_val;
- }
- if(raw_val > fvmax) {
- fvmax = raw_val;
- }
- fvsum += raw_val;
- fvcount++;
} else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {
value = std::to_string(document[a_facet.field_name].get<bool>());
value = (value == "1") ? "true" : "false";
diff --git a/src/index.cpp b/src/index.cpp
index 93571f64..dafebf14 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -122,7 +122,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
strings.push_back(std::to_string(value));
}
}
- index_string_array_field(strings, points, t, seq_id, facet_id);
+ index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);
} else {
std::string text;
@@ -136,7 +136,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
text = std::to_string(document[field_name].get<bool>());
}
- index_string_field(text, points, t, seq_id, facet_id);
+ index_string_field(text, points, t, seq_id, facet_id, field_pair.second);
}
}
@@ -144,7 +144,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
if(field_pair.second.type == field_types::STRING) {
const std::string & text = document[field_name];
- index_string_field(text, points, t, seq_id, facet_id);
+ index_string_field(text, points, t, seq_id, facet_id, field_pair.second);
} else if(field_pair.second.type == field_types::INT32) {
uint32_t value = document[field_name];
index_int32_field(value, points, t, seq_id);
@@ -159,7 +159,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
index_bool_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
std::vector<std::string> strings = document[field_name];
- index_string_array_field(strings, points, t, seq_id, facet_id);
+ index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);
} else if(field_pair.second.type == field_types::INT32_ARRAY) {
std::vector<int32_t> values = document[field_name];
index_int32_array_field(values, points, t, seq_id);
@@ -460,8 +460,25 @@ void Index::index_float_field(const float value, uint32_t score, art_tree *t, ui
}
+uint64_t Index::facet_token_hash(const field & a_field, const std::string &token) {
+ // for integer/float use their native values
+ uint64_t hash = 0;
+
+ if(a_field.is_float()) {
+ float f = atof(token.c_str());
+ hash = *reinterpret_cast<uint64_t*>(&f); // store as int without loss of precision
+ } else if(a_field.is_integer() || a_field.is_bool()) {
+ hash = atol(token.c_str());
+ } else {
+ // string field
+ hash = StringUtils::hash_wy(token.c_str(), token.size());
+ }
+
+ return hash;
+}
+
void Index::index_string_field(const std::string & text, const uint32_t score, art_tree *t,
- uint32_t seq_id, int facet_id) {
+ uint32_t seq_id, int facet_id, const field & a_field) {
std::vector<std::string> tokens;
StringUtils::split(text, tokens, " ");
@@ -469,14 +486,17 @@ void Index::index_string_field(const std::string & text, const uint32_t score, a
for(uint32_t i=0; i<tokens.size(); i++) {
auto & token = tokens[i];
- string_utils.unicode_normalize(token);
- token_to_offsets[token].push_back(i);
+
+ if(a_field.is_string()) {
+ string_utils.unicode_normalize(token);
+ }
if(facet_id >= 0) {
- // facet field
- uint64_t hash = StringUtils::hash_wy(token.c_str(), token.size());
+ uint64_t hash = facet_token_hash(a_field, token);
facet_index_v2[seq_id][facet_id].push_back(hash);
}
+
+ token_to_offsets[token].push_back(i);
}
insert_doc(score, t, seq_id, token_to_offsets);
@@ -487,7 +507,7 @@ void Index::index_string_field(const std::string & text, const uint32_t score, a
}
void Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
- uint32_t seq_id, int facet_id) {
+ uint32_t seq_id, int facet_id, const field & a_field) {
std::unordered_map<std::string, std::vector<uint32_t>> token_positions;
for(size_t array_index = 0; array_index < strings.size(); array_index++) {
@@ -501,20 +521,23 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c
// iterate and append offset positions
for(size_t i=0; i<tokens.size(); i++) {
auto & token = tokens[i];
- string_utils.unicode_normalize(token);
- token_positions[token].push_back(i);
- token_set.insert(token);
+
+ if(a_field.is_string()) {
+ string_utils.unicode_normalize(token);
+ }
if(facet_id >= 0) {
- // facet field
- uint64_t hash = StringUtils::hash_wy(token.c_str(), token.size());
- //printf("indexing %.*s - %llu\n", token.size(), token.c_str(), hash);
+ uint64_t hash = facet_token_hash(a_field, token);
facet_index_v2[seq_id][facet_id].push_back(hash);
+ //printf("indexing %.*s - %llu\n", token.size(), token.c_str(), hash);
}
+
+ token_positions[token].push_back(i);
+ token_set.insert(token);
}
if(facet_id >= 0) {
- facet_index_v2[seq_id][facet_id].push_back(0); // as a delimiter
+ facet_index_v2[seq_id][facet_id].push_back(FACET_ARRAY_DELIMETER); // as a delimiter
}
// repeat last element to indicate end of offsets for this array index
@@ -563,6 +586,40 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin
}
}
+void Index::compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type) {
+ if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {
+ int32_t val = raw_value;
+ if (val < a_facet.stats.fvmin) {
+ a_facet.stats.fvmin = val;
+ }
+ if (val > a_facet.stats.fvmax) {
+ a_facet.stats.fvmax = val;
+ }
+ a_facet.stats.fvsum += val;
+ a_facet.stats.fvcount++;
+ } else if(field_type == field_types::INT64 || field_type == field_types::INT64_ARRAY) {
+ int64_t val = raw_value;
+ if(val < a_facet.stats.fvmin) {
+ a_facet.stats.fvmin = val;
+ }
+ if(val > a_facet.stats.fvmax) {
+ a_facet.stats.fvmax = val;
+ }
+ a_facet.stats.fvsum += val;
+ a_facet.stats.fvcount++;
+ } else if(field_type == field_types::FLOAT || field_type == field_types::FLOAT_ARRAY) {
+ float val = *reinterpret_cast<float*>(&raw_value);
+ if(val < a_facet.stats.fvmin) {
+ a_facet.stats.fvmin = val;
+ }
+ if(val > a_facet.stats.fvmax) {
+ a_facet.stats.fvmax = val;
+ }
+ a_facet.stats.fvsum += val;
+ a_facet.stats.fvcount++;
+ }
+}
+
void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
const uint32_t* result_ids, size_t results_size) {
@@ -576,12 +633,12 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
// assumed that facet fields have already been validated upstream
for(auto & a_facet: facets) {
- spp::sparse_hash_map<int64_t, token_pos_cost_t> fhash_qtoken_pos; // facet hash => token position in the query
+ spp::sparse_hash_map<uint64_t, token_pos_cost_t> fhash_qtoken_pos; // facet hash => token position in the query
bool use_facet_query = false;
+ const field & facet_field = facet_schema.at(a_facet.field_name);
if(a_facet.field_name == facet_query.field_name && !facet_query.query.empty()) {
use_facet_query = true;
- const field & facet_field = facet_schema.at(a_facet.field_name);
if(facet_field.is_bool()) {
if(facet_query.query == "true") {
@@ -598,7 +655,10 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
for(size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {
auto & q = query_tokens[qtoken_index];
- string_utils.unicode_normalize(q);
+ if(facet_field.is_string()) {
+ string_utils.unicode_normalize(q);
+ }
+
int bounded_cost = (q.size() < 3) ? 0 : 1;
bool prefix_search = (qtoken_index == (query_tokens.size()-1)); // only last token must be used as prefix
@@ -611,7 +671,9 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
for(size_t i = 0; i < leaves.size(); i++) {
const auto & leaf = leaves[i];
// calculate hash without terminating null char
- uint64_t hash = StringUtils::hash_wy(leaf->key, leaf->key_len-1);
+ std::string key_str((const char*)leaf->key, leaf->key_len-1);
+ uint64_t hash = facet_token_hash(facet_field, key_str);
+
token_pos_cost_t token_pos_cost = {qtoken_index, 0};
fhash_qtoken_pos.emplace(hash, token_pos_cost);
//printf("%.*s - %llu\n", leaf->key_len, leaf->key, hash);
@@ -637,11 +699,14 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
size_t field_token_index = -1;
for(size_t j = 0; j < fhashes.size(); j++) {
- if(fhashes[j] != 0) {
- uint64_t ftoken_hash = fhashes[j];
+ if(fhashes[j] != FACET_ARRAY_DELIMETER) {
+ int64_t ftoken_hash = fhashes[j];
fvaluestream << ftoken_hash;
field_token_index++;
+ // ftoken_hash is the raw value for numeric fields
+ compute_facet_stats(a_facet, ftoken_hash, facet_field.type);
+
if(!use_facet_query || fhash_qtoken_pos.find(ftoken_hash) != fhash_qtoken_pos.end()) {
// not using facet query or this particular facet value is found in facet filter
fvalue_found |= 1; // bitwise to ensure only one count for a multi-token facet value
@@ -664,10 +729,15 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
//std::cout << "j: " << j << std::endl;
// 0 indicates separator, while the second condition checks for non-array string
- if(fhashes[j] == 0 || (fhashes.back() != 0 && j == fhashes.size() - 1)) {
+ if(fhashes[j] == FACET_ARRAY_DELIMETER || (fhashes.back() != FACET_ARRAY_DELIMETER && j == fhashes.size() - 1)) {
if(!use_facet_query || fvalue_found != 0) {
const std::string & fvalue_str = fvaluestream.str();
- uint64_t fhash = StringUtils::hash_wy(fvalue_str.c_str(), fvalue_str.size());
+ uint64_t fhash = 0;
+ if(facet_field.is_string()) {
+ fhash = facet_token_hash(facet_field, fvalue_str);
+ } else {
+ fhash = std::atoi(fvalue_str.c_str());
+ }
if(a_facet.result_map.count(fhash) == 0) {
a_facet.result_map[fhash] = facet_count_t{0, doc_seq_id, 0,
@@ -711,6 +781,8 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>
}
for(auto & a_facet: facets) {
+ const field & facet_field = facet_schema.at(a_facet.field_name);
+
// assumed that facet fields have already been validated upstream
for(const uint32_t doc_seq_id: ids) {
if(facet_index_v2.count(doc_seq_id) != 0) {
@@ -721,14 +793,15 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>
std::stringstream fvaluestream; // for hashing the entire facet value (multiple tokens)
for(size_t j = 0; j < fhashes.size(); j++) {
- if(fhashes[j] != 0) {
- uint64_t ftoken_hash = fhashes[j];
+ if(fhashes[j] != FACET_ARRAY_DELIMETER) {
+ int64_t ftoken_hash = fhashes[j];
fvaluestream << ftoken_hash;
}
- if(fhashes[j] == 0 || (fhashes.back() != 0 && j == fhashes.size() - 1)) {
+ if(fhashes[j] == FACET_ARRAY_DELIMETER || (fhashes.back() != FACET_ARRAY_DELIMETER &&
+ j == fhashes.size() - 1)) {
const std::string & fvalue_str = fvaluestream.str();
- uint64_t fhash = StringUtils::hash_wy(fvalue_str.c_str(), fvalue_str.size());
+ uint64_t fhash = facet_token_hash(facet_field, fvalue_str);
if(a_facet.result_map.count(fhash) != 0) {
a_facet.result_map[fhash].count -= 1;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 89f234c0..981ebbd9 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -1540,6 +1540,72 @@ TEST_F(CollectionTest, SortOnFloatFields) {
collectionManager.drop_collection("coll_float_fields");
}
+TEST_F(CollectionTest, FacetStatOnFloatFields) {
+ Collection *coll_float_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/float_documents.jsonl");
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("score", field_types::FLOAT, false),
+ field("average", field_types::FLOAT, true)
+ };
+
+ std::vector<sort_by> sort_fields_desc = { sort_by("average", "DESC") };
+
+ coll_float_fields = collectionManager.get_collection("coll_float_fields");
+ if(coll_float_fields == nullptr) {
+ coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "average").get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_float_fields->add(json_line);
+ }
+
+ infile.close();
+
+ query_fields = {"title"};
+ std::vector<std::string> facets;
+ auto res_op = coll_float_fields->search("Jeremy", query_fields, "", {"average"}, sort_fields_desc, 0, 10,
+ 1, FREQUENCY, false);
+
+ auto results = res_op.get();
+
+ ASSERT_EQ(7, results["hits"].size());
+
+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
+ ASSERT_FLOAT_EQ(-21.3799991607666, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(277.8160007725237, results["facet_counts"][0]["stats"]["sum"].get<double>());
+ ASSERT_FLOAT_EQ(39.68800011036053, results["facet_counts"][0]["stats"]["avg"].get<double>());
+
+ results = coll_float_fields->search("*", query_fields, "average:>100", {"average"}, sort_fields_desc, 0, 10,
+ 1, FREQUENCY, false).get();
+
+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["sum"].get<double>());
+
+ // facet filter, though should not be considered when calculating facet stats
+
+ results = coll_float_fields->search("*", query_fields, "", {"average"}, sort_fields_desc, 0, 10, 1, FREQUENCY,
+ false, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(),
+ 10, 500, "average: 11").get();
+
+ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
+ ASSERT_FLOAT_EQ(39.68800011036053, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(-21.3799991607666, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(277.8160007725237, results["facet_counts"][0]["stats"]["sum"].get<double>());
+
+ collectionManager.drop_collection("coll_float_fields");
+}
+
TEST_F(CollectionTest, QueryBoolFields) {
Collection *coll_bool;
@@ -1968,19 +2034,18 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_STREQ("age", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("24", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("21", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("21", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("24", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
// facet on a float field without query to check on stats
results = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, 0, 10, 1, FREQUENCY,
false, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, 500, "").get();
-
ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
ASSERT_FLOAT_EQ(4.880199885368347, results["facet_counts"][0]["stats"]["avg"].get<double>());
ASSERT_FLOAT_EQ(0.0, results["facet_counts"][0]["stats"]["min"].get<double>());
@@ -2001,10 +2066,10 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_STREQ("<mark>7</mark>.812", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
- ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["avg"].get<double>());
- ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["min"].get<double>());
- ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["max"].get<double>());
- ASSERT_FLOAT_EQ(7.812, results["facet_counts"][0]["stats"]["sum"].get<double>());
+ ASSERT_FLOAT_EQ(4.880199885368347, results["facet_counts"][0]["stats"]["avg"].get<double>());
+ ASSERT_FLOAT_EQ(0.0, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(9.99899959564209, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(24.400999426841736, results["facet_counts"][0]["stats"]["sum"].get<double>());
// facet query on an array integer field
@@ -2023,10 +2088,10 @@ TEST_F(CollectionTest, FacetCounts) {
ASSERT_STREQ("<mark>142189002</mark>2", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());
- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["avg"].get<double>());
- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["min"].get<double>());
- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["max"].get<double>());
- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["sum"].get<double>());
+ ASSERT_FLOAT_EQ(348974822.0, results["facet_counts"][0]["stats"]["min"].get<double>());
+ ASSERT_FLOAT_EQ(1453426022.0, results["facet_counts"][0]["stats"]["max"].get<double>());
+ ASSERT_FLOAT_EQ(13275854664.0, results["facet_counts"][0]["stats"]["sum"].get<double>());
+ ASSERT_FLOAT_EQ(1106321222.0, results["facet_counts"][0]["stats"]["avg"].get<double>());
// facet query that does not match any indexed value
results = coll_array_fields->search("*", query_fields, "", {facets}, sort_fields, 0, 10, 1, FREQUENCY,
@@ -2113,12 +2178,12 @@ TEST_F(CollectionTest, FacetCountsHighlighting) {
ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);
- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
coll1->remove("100");