Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Compute stats on all facet values regardless of facet query.
kishorenc31e9e4719a26
diff --git a/include/collection.h b/include/collection.hindex da3043ba..71291a47 100644--- a/include/collection.h+++ b/include/collection.h@@ -262,8 +262,7 @@ public:static constexpr const char* SEQ_ID_PREFIX = "$SI";static constexpr const char* DOC_ID_PREFIX = "$DI";- void compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount, double &fvsum,- const facet_count_t &facet_count, const nlohmann::json &document,+ void facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,std::string &value);};diff --git a/include/field.h b/include/field.hindex 6a27ae80..a19c2dda 100644--- a/include/field.h+++ b/include/field.h@@ -145,9 +145,17 @@ struct facet_count_t {spp::sparse_hash_map<uint32_t, token_pos_cost_t> query_token_pos;};+struct facet_stats_t {+ double fvmin = std::numeric_limits<double>::max(),+ fvmax = -std::numeric_limits<double>::min(),+ fvcount = 0,+ fvsum = 0;+};+struct facet {const std::string field_name;std::map<uint64_t, facet_count_t> result_map;+ facet_stats_t stats;facet(const std::string & field_name): field_name(field_name) {diff --git a/include/index.h b/include/index.hindex 1e019997..79d3879c 100644--- a/include/index.h+++ b/include/index.h@@ -113,6 +113,8 @@ struct batch_index_result {class Index {private:+ const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();+std::string name;size_t num_documents;@@ -163,10 +165,10 @@ private:const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,- int facet_id);+ int facet_id, const field & a_field);void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,- uint32_t seq_id, int facet_id);+ uint32_t seq_id, int facet_id, const field & a_field);void index_int32_field(const int32_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;@@ -191,6 +193,10 @@ private:const std::vector<uint32_t> & included_ids,Topster & curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);+ uint64_t facet_token_hash(const field & a_field, const std::string &token);++ void compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type);+public:Index() = delete;diff --git a/include/string_utils.h b/include/string_utils.hindex ff4c1057..8472f42a 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -220,8 +220,9 @@ struct StringUtils {return seq_id;}- static int64_t hash_wy(const void* key, uint64_t len) {+ static uint64_t hash_wy(const void* key, uint64_t len) {uint64_t hash = wyhash(key, len, 0, _wyp);- return hash != 0 ? hash : 1; // reserve 0 for use as a delimiter+ // reserve max() for use as a delimiter+ return hash != std::numeric_limits<uint64_t>::max() ? hash : (std::numeric_limits<uint64_t>::max()-1);}};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 0b39d3e2..76295ef8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -614,6 +614,13 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;acc_facet.result_map[facet_kv.first].query_token_pos = facet_kv.second.query_token_pos;}++ if(this_facet.stats.fvcount != 0) {+ acc_facet.stats.fvcount += this_facet.stats.fvcount;+ acc_facet.stats.fvsum += this_facet.stats.fvsum;+ acc_facet.stats.fvmax = std::max(acc_facet.stats.fvmax, this_facet.stats.fvmax);+ acc_facet.stats.fvmin = std::min(acc_facet.stats.fvmin, this_facet.stats.fvmin);+ }}total_found += index->search_params.all_result_ids_len;@@ -731,9 +738,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::facet_result["field_name"] = a_facet.field_name;facet_result["counts"] = nlohmann::json::array();- std::vector<std::pair<uint64_t, facet_count_t>> facet_hash_counts;- for (const auto & itr : a_facet.result_map) {- facet_hash_counts.emplace_back(itr);+ std::vector<std::pair<int64_t, facet_count_t>> facet_hash_counts;+ for (const auto & kv : a_facet.result_map) {+ facet_hash_counts.emplace_back(kv);}// keep only top K facets@@ -747,14 +754,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::vector<facet_value_t> facet_values;- double fvmin = std::numeric_limits<double>::max(),- fvmax = std::numeric_limits<double>::min(),- fvcount = 0,- fvsum = 0;-- for(size_t i = 0; i < max_facets; i++) {+ for(size_t fi = 0; fi < max_facets; fi++) {// remap facet value hash with actual string- auto & kv = facet_hash_counts[i];+ auto & kv = facet_hash_counts[fi];auto & facet_count = kv.second;// fetch actual facet value from representative doc id@@ -768,7 +770,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}std::string value;- compute_facet_results(a_facet, fvmin, fvmax, fvcount, fvsum, facet_count, document, value);+ facet_value_to_string(a_facet, facet_count, document, value);std::vector<std::string> tokens;StringUtils::split(value, tokens, " ");@@ -812,11 +814,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::// add facet value statsfacet_result["stats"] = nlohmann::json::object();- if(fvcount != 0) {- facet_result["stats"]["min"] = fvmin;- facet_result["stats"]["max"] = fvmax;- facet_result["stats"]["sum"] = fvsum;- facet_result["stats"]["avg"] = (fvsum / fvcount);+ if(a_facet.stats.fvcount != 0) {+ facet_result["stats"]["min"] = a_facet.stats.fvmin;+ facet_result["stats"]["max"] = a_facet.stats.fvmax;+ facet_result["stats"]["sum"] = a_facet.stats.fvsum;+ facet_result["stats"]["avg"] = (a_facet.stats.fvsum / a_facet.stats.fvcount);}result["facet_counts"].push_back(facet_result);@@ -828,9 +830,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::return result;}-void Collection::compute_facet_results(const facet &a_facet, double &fvmin, double &fvmax, double &fvcount,- double &fvsum, const facet_count_t &facet_count,+void Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,const nlohmann::json &document, std::string &value) {+if(facet_schema.at(a_facet.field_name).type == field_types::STRING) {value = document[a_facet.field_name];} else if(facet_schema.at(a_facet.field_name).type == field_types::STRING_ARRAY) {@@ -838,71 +840,23 @@ void Collection::compute_facet_results(const facet &a_facet, double &fvmin, doub} else if(facet_schema.at(a_facet.field_name).type == field_types::INT32) {int32_t raw_val = document[a_facet.field_name].get<int32_t>();value = std::to_string(raw_val);- if(raw_val < fvmin) {- fvmin = raw_val;- }- if(raw_val > fvmax) {- fvmax = raw_val;- }- fvsum += raw_val;- fvcount++;} else if(facet_schema.at(a_facet.field_name).type == field_types::INT32_ARRAY) {int32_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int32_t>();value = std::to_string(raw_val);- if(raw_val < fvmin) {- fvmin = raw_val;- }- if(raw_val > fvmax) {- fvmax = raw_val;- }- fvsum += raw_val;- fvcount++;} else if(facet_schema.at(a_facet.field_name).type == field_types::INT64) {int64_t raw_val = document[a_facet.field_name].get<int64_t>();value = std::to_string(raw_val);- if(raw_val < fvmin) {- fvmin = raw_val;- }- if(raw_val > fvmax) {- fvmax = raw_val;- }- fvsum += raw_val;- fvcount++;} else if(facet_schema.at(a_facet.field_name).type == field_types::INT64_ARRAY) {int64_t raw_val = document[a_facet.field_name][facet_count.array_pos].get<int64_t>();value = std::to_string(raw_val);- if(raw_val < fvmin) {- fvmin = raw_val;- }- if(raw_val > fvmax) {- fvmax = raw_val;- }- fvsum += raw_val;- fvcount++;} else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT) {float raw_val = document[a_facet.field_name].get<float>();value = std::to_string(raw_val);value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros- if(raw_val < fvmin) {- fvmin = raw_val;- }- if(raw_val > fvmax) {- fvmax = raw_val;- }- fvsum += raw_val;- fvcount++;} else if(facet_schema.at(a_facet.field_name).type == field_types::FLOAT_ARRAY) {float raw_val = document[a_facet.field_name][facet_count.array_pos].get<float>();value = std::to_string(raw_val);value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros- if(raw_val < fvmin) {- fvmin = raw_val;- }- if(raw_val > fvmax) {- fvmax = raw_val;- }- fvsum += raw_val;- fvcount++;} else if(facet_schema.at(a_facet.field_name).type == field_types::BOOL) {value = std::to_string(document[a_facet.field_name].get<bool>());value = (value == "1") ? "true" : "false";diff --git a/src/index.cpp b/src/index.cppindex 93571f64..dafebf14 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -122,7 +122,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tstrings.push_back(std::to_string(value));}}- index_string_array_field(strings, points, t, seq_id, facet_id);+ index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);} else {std::string text;@@ -136,7 +136,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_ttext = std::to_string(document[field_name].get<bool>());}- index_string_field(text, points, t, seq_id, facet_id);+ index_string_field(text, points, t, seq_id, facet_id, field_pair.second);}}@@ -144,7 +144,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tif(field_pair.second.type == field_types::STRING) {const std::string & text = document[field_name];- index_string_field(text, points, t, seq_id, facet_id);+ index_string_field(text, points, t, seq_id, facet_id, field_pair.second);} else if(field_pair.second.type == field_types::INT32) {uint32_t value = document[field_name];index_int32_field(value, points, t, seq_id);@@ -159,7 +159,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_tindex_bool_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {std::vector<std::string> strings = document[field_name];- index_string_array_field(strings, points, t, seq_id, facet_id);+ index_string_array_field(strings, points, t, seq_id, facet_id, field_pair.second);} else if(field_pair.second.type == field_types::INT32_ARRAY) {std::vector<int32_t> values = document[field_name];index_int32_array_field(values, points, t, seq_id);@@ -460,8 +460,25 @@ void Index::index_float_field(const float value, uint32_t score, art_tree *t, ui}+uint64_t Index::facet_token_hash(const field & a_field, const std::string &token) {+ // for integer/float use their native values+ uint64_t hash = 0;++ if(a_field.is_float()) {+ float f = atof(token.c_str());+ hash = *reinterpret_cast<uint64_t*>(&f); // store as int without loss of precision+ } else if(a_field.is_integer() || a_field.is_bool()) {+ hash = atol(token.c_str());+ } else {+ // string field+ hash = StringUtils::hash_wy(token.c_str(), token.size());+ }++ return hash;+}+void Index::index_string_field(const std::string & text, const uint32_t score, art_tree *t,- uint32_t seq_id, int facet_id) {+ uint32_t seq_id, int facet_id, const field & a_field) {std::vector<std::string> tokens;StringUtils::split(text, tokens, " ");@@ -469,14 +486,17 @@ void Index::index_string_field(const std::string & text, const uint32_t score, afor(uint32_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];- string_utils.unicode_normalize(token);- token_to_offsets[token].push_back(i);++ if(a_field.is_string()) {+ string_utils.unicode_normalize(token);+ }if(facet_id >= 0) {- // facet field- uint64_t hash = StringUtils::hash_wy(token.c_str(), token.size());+ uint64_t hash = facet_token_hash(a_field, token);facet_index_v2[seq_id][facet_id].push_back(hash);}++ token_to_offsets[token].push_back(i);}insert_doc(score, t, seq_id, token_to_offsets);@@ -487,7 +507,7 @@ void Index::index_string_field(const std::string & text, const uint32_t score, a}void Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,- uint32_t seq_id, int facet_id) {+ uint32_t seq_id, int facet_id, const field & a_field) {std::unordered_map<std::string, std::vector<uint32_t>> token_positions;for(size_t array_index = 0; array_index < strings.size(); array_index++) {@@ -501,20 +521,23 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, c// iterate and append offset positionsfor(size_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];- string_utils.unicode_normalize(token);- token_positions[token].push_back(i);- token_set.insert(token);++ if(a_field.is_string()) {+ string_utils.unicode_normalize(token);+ }if(facet_id >= 0) {- // facet field- uint64_t hash = StringUtils::hash_wy(token.c_str(), token.size());- //printf("indexing %.*s - %llu\n", token.size(), token.c_str(), hash);+ uint64_t hash = facet_token_hash(a_field, token);facet_index_v2[seq_id][facet_id].push_back(hash);+ //printf("indexing %.*s - %llu\n", token.size(), token.c_str(), hash);}++ token_positions[token].push_back(i);+ token_set.insert(token);}if(facet_id >= 0) {- facet_index_v2[seq_id][facet_id].push_back(0); // as a delimiter+ facet_index_v2[seq_id][facet_id].push_back(FACET_ARRAY_DELIMETER); // as a delimiter}// repeat last element to indicate end of offsets for this array index@@ -563,6 +586,40 @@ void Index::index_float_array_field(const std::vector<float> & values, const uin}}+void Index::compute_facet_stats(facet &a_facet, int64_t raw_value, const std::string & field_type) {+ if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {+ int32_t val = raw_value;+ if (val < a_facet.stats.fvmin) {+ a_facet.stats.fvmin = val;+ }+ if (val > a_facet.stats.fvmax) {+ a_facet.stats.fvmax = val;+ }+ a_facet.stats.fvsum += val;+ a_facet.stats.fvcount++;+ } else if(field_type == field_types::INT64 || field_type == field_types::INT64_ARRAY) {+ int64_t val = raw_value;+ if(val < a_facet.stats.fvmin) {+ a_facet.stats.fvmin = val;+ }+ if(val > a_facet.stats.fvmax) {+ a_facet.stats.fvmax = val;+ }+ a_facet.stats.fvsum += val;+ a_facet.stats.fvcount++;+ } else if(field_type == field_types::FLOAT || field_type == field_types::FLOAT_ARRAY) {+ float val = *reinterpret_cast<float*>(&raw_value);+ if(val < a_facet.stats.fvmin) {+ a_facet.stats.fvmin = val;+ }+ if(val > a_facet.stats.fvmax) {+ a_facet.stats.fvmax = val;+ }+ a_facet.stats.fvsum += val;+ a_facet.stats.fvcount++;+ }+}+void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const uint32_t* result_ids, size_t results_size) {@@ -576,12 +633,12 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,// assumed that facet fields have already been validated upstreamfor(auto & a_facet: facets) {- spp::sparse_hash_map<int64_t, token_pos_cost_t> fhash_qtoken_pos; // facet hash => token position in the query+ spp::sparse_hash_map<uint64_t, token_pos_cost_t> fhash_qtoken_pos; // facet hash => token position in the querybool use_facet_query = false;+ const field & facet_field = facet_schema.at(a_facet.field_name);if(a_facet.field_name == facet_query.field_name && !facet_query.query.empty()) {use_facet_query = true;- const field & facet_field = facet_schema.at(a_facet.field_name);if(facet_field.is_bool()) {if(facet_query.query == "true") {@@ -598,7 +655,10 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,for(size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {auto & q = query_tokens[qtoken_index];- string_utils.unicode_normalize(q);+ if(facet_field.is_string()) {+ string_utils.unicode_normalize(q);+ }+int bounded_cost = (q.size() < 3) ? 0 : 1;bool prefix_search = (qtoken_index == (query_tokens.size()-1)); // only last token must be used as prefix@@ -611,7 +671,9 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,for(size_t i = 0; i < leaves.size(); i++) {const auto & leaf = leaves[i];// calculate hash without terminating null char- uint64_t hash = StringUtils::hash_wy(leaf->key, leaf->key_len-1);+ std::string key_str((const char*)leaf->key, leaf->key_len-1);+ uint64_t hash = facet_token_hash(facet_field, key_str);+token_pos_cost_t token_pos_cost = {qtoken_index, 0};fhash_qtoken_pos.emplace(hash, token_pos_cost);//printf("%.*s - %llu\n", leaf->key_len, leaf->key, hash);@@ -637,11 +699,14 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,size_t field_token_index = -1;for(size_t j = 0; j < fhashes.size(); j++) {- if(fhashes[j] != 0) {- uint64_t ftoken_hash = fhashes[j];+ if(fhashes[j] != FACET_ARRAY_DELIMETER) {+ int64_t ftoken_hash = fhashes[j];fvaluestream << ftoken_hash;field_token_index++;+ // ftoken_hash is the raw value for numeric fields+ compute_facet_stats(a_facet, ftoken_hash, facet_field.type);+if(!use_facet_query || fhash_qtoken_pos.find(ftoken_hash) != fhash_qtoken_pos.end()) {// not using facet query or this particular facet value is found in facet filterfvalue_found |= 1; // bitwise to ensure only one count for a multi-token facet value@@ -664,10 +729,15 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,//std::cout << "j: " << j << std::endl;// 0 indicates separator, while the second condition checks for non-array string- if(fhashes[j] == 0 || (fhashes.back() != 0 && j == fhashes.size() - 1)) {+ if(fhashes[j] == FACET_ARRAY_DELIMETER || (fhashes.back() != FACET_ARRAY_DELIMETER && j == fhashes.size() - 1)) {if(!use_facet_query || fvalue_found != 0) {const std::string & fvalue_str = fvaluestream.str();- uint64_t fhash = StringUtils::hash_wy(fvalue_str.c_str(), fvalue_str.size());+ uint64_t fhash = 0;+ if(facet_field.is_string()) {+ fhash = facet_token_hash(facet_field, fvalue_str);+ } else {+ fhash = std::atoi(fvalue_str.c_str());+ }if(a_facet.result_map.count(fhash) == 0) {a_facet.result_map[fhash] = facet_count_t{0, doc_seq_id, 0,@@ -711,6 +781,8 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>}for(auto & a_facet: facets) {+ const field & facet_field = facet_schema.at(a_facet.field_name);+// assumed that facet fields have already been validated upstreamfor(const uint32_t doc_seq_id: ids) {if(facet_index_v2.count(doc_seq_id) != 0) {@@ -721,14 +793,15 @@ void Index::drop_facets(std::vector<facet> & facets, const std::vector<uint32_t>std::stringstream fvaluestream; // for hashing the entire facet value (multiple tokens)for(size_t j = 0; j < fhashes.size(); j++) {- if(fhashes[j] != 0) {- uint64_t ftoken_hash = fhashes[j];+ if(fhashes[j] != FACET_ARRAY_DELIMETER) {+ int64_t ftoken_hash = fhashes[j];fvaluestream << ftoken_hash;}- if(fhashes[j] == 0 || (fhashes.back() != 0 && j == fhashes.size() - 1)) {+ if(fhashes[j] == FACET_ARRAY_DELIMETER || (fhashes.back() != FACET_ARRAY_DELIMETER &&+ j == fhashes.size() - 1)) {const std::string & fvalue_str = fvaluestream.str();- uint64_t fhash = StringUtils::hash_wy(fvalue_str.c_str(), fvalue_str.size());+ uint64_t fhash = facet_token_hash(facet_field, fvalue_str);if(a_facet.result_map.count(fhash) != 0) {a_facet.result_map[fhash].count -= 1;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 89f234c0..981ebbd9 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -1540,6 +1540,72 @@ TEST_F(CollectionTest, SortOnFloatFields) {collectionManager.drop_collection("coll_float_fields");}+TEST_F(CollectionTest, FacetStatOnFloatFields) {+ Collection *coll_float_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/float_documents.jsonl");+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("score", field_types::FLOAT, false),+ field("average", field_types::FLOAT, true)+ };++ std::vector<sort_by> sort_fields_desc = { sort_by("average", "DESC") };++ coll_float_fields = collectionManager.get_collection("coll_float_fields");+ if(coll_float_fields == nullptr) {+ coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "average").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_float_fields->add(json_line);+ }++ infile.close();++ query_fields = {"title"};+ std::vector<std::string> facets;+ auto res_op = coll_float_fields->search("Jeremy", query_fields, "", {"average"}, sort_fields_desc, 0, 10,+ 1, FREQUENCY, false);++ auto results = res_op.get();++ ASSERT_EQ(7, results["hits"].size());++ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());+ ASSERT_FLOAT_EQ(-21.3799991607666, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(277.8160007725237, results["facet_counts"][0]["stats"]["sum"].get<double>());+ ASSERT_FLOAT_EQ(39.68800011036053, results["facet_counts"][0]["stats"]["avg"].get<double>());++ results = coll_float_fields->search("*", query_fields, "average:>100", {"average"}, sort_fields_desc, 0, 10,+ 1, FREQUENCY, false).get();++ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["avg"].get<double>());+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["sum"].get<double>());++ // facet filter, though should not be considered when calculating facet stats++ results = coll_float_fields->search("*", query_fields, "", {"average"}, sort_fields_desc, 0, 10, 1, FREQUENCY,+ false, Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(),+ 10, 500, "average: 11").get();++ ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());+ ASSERT_FLOAT_EQ(39.68800011036053, results["facet_counts"][0]["stats"]["avg"].get<double>());+ ASSERT_FLOAT_EQ(-21.3799991607666, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(300, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(277.8160007725237, results["facet_counts"][0]["stats"]["sum"].get<double>());++ collectionManager.drop_collection("coll_float_fields");+}+TEST_F(CollectionTest, QueryBoolFields) {Collection *coll_bool;@@ -1968,19 +2034,18 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_STREQ("age", results["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("24", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("21", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("21", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("24", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());// facet on a float field without query to check on statsresults = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, 0, 10, 1, FREQUENCY,false, Index::DROP_TOKENS_THRESHOLD,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, 500, "").get();-ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());ASSERT_FLOAT_EQ(4.880199885368347, results["facet_counts"][0]["stats"]["avg"].get<double>());ASSERT_FLOAT_EQ(0.0, results["facet_counts"][0]["stats"]["min"].get<double>());@@ -2001,10 +2066,10 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_STREQ("<mark>7</mark>.812", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());- ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["avg"].get<double>());- ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["min"].get<double>());- ASSERT_FLOAT_EQ(7.811999797821045, results["facet_counts"][0]["stats"]["max"].get<double>());- ASSERT_FLOAT_EQ(7.812, results["facet_counts"][0]["stats"]["sum"].get<double>());+ ASSERT_FLOAT_EQ(4.880199885368347, results["facet_counts"][0]["stats"]["avg"].get<double>());+ ASSERT_FLOAT_EQ(0.0, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(9.99899959564209, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(24.400999426841736, results["facet_counts"][0]["stats"]["sum"].get<double>());// facet query on an array integer field@@ -2023,10 +2088,10 @@ TEST_F(CollectionTest, FacetCounts) {ASSERT_STREQ("<mark>142189002</mark>2", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(4, results["facet_counts"][0]["stats"].size());- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["avg"].get<double>());- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["min"].get<double>());- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["max"].get<double>());- ASSERT_FLOAT_EQ(1421890022.0, results["facet_counts"][0]["stats"]["sum"].get<double>());+ ASSERT_FLOAT_EQ(348974822.0, results["facet_counts"][0]["stats"]["min"].get<double>());+ ASSERT_FLOAT_EQ(1453426022.0, results["facet_counts"][0]["stats"]["max"].get<double>());+ ASSERT_FLOAT_EQ(13275854664.0, results["facet_counts"][0]["stats"]["sum"].get<double>());+ ASSERT_FLOAT_EQ(1106321222.0, results["facet_counts"][0]["stats"]["avg"].get<double>());// facet query that does not match any indexed valueresults = coll_array_fields->search("*", query_fields, "", {facets}, sort_fields, 0, 10, 1, FREQUENCY,@@ -2113,12 +2178,12 @@ TEST_F(CollectionTest, FacetCountsHighlighting) {ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][2]["count"]);- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("Cell Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phone Cases & Clips", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());coll1->remove("100");