Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Have separate facet aggregation structures for value/hash facets.
Kishore Nallana3a44acf2f3e
diff --git a/include/collection.h b/include/collection.hindex 692e4142..1f948175 100644--- a/include/collection.h+++ b/include/collection.h@@ -187,9 +187,8 @@ private:std::vector<field>& new_fields,bool enable_nested_fields);- static bool facet_count_compare(const std::pair<uint32_t, facet_count_t>& a,- const std::pair<uint32_t, facet_count_t>& b) {- return std::tie(a.second.count, a.first) > std::tie(b.second.count, b.first);+ static bool facet_count_compare(const facet_count_t& a, const facet_count_t& b) {+ return std::tie(a.count, a.fhash) > std::tie(b.count, b.fhash);}static bool facet_count_str_compare(const facet_value_t& a,diff --git a/include/field.h b/include/field.hindex c1cfe1aa..a051c7ee 100644--- a/include/field.h+++ b/include/field.h@@ -570,6 +570,10 @@ public:struct facet_count_t {uint32_t count = 0;+ // for value based faceting, actual value is stored here+ std::string fvalue;+ // for hash based faceting, hash value is stored here+ int64_t fhash;// used to fetch the actual document and value for representationuint32_t doc_id = 0;uint32_t array_pos = 0;@@ -584,9 +588,12 @@ struct facet_stats_t {struct facet {const std::string field_name;- spp::sparse_hash_map<std::string, facet_count_t> result_map;+ spp::sparse_hash_map<uint64_t, facet_count_t> result_map;+ spp::sparse_hash_map<std::string, facet_count_t> value_result_map;+// used for facet value query- spp::sparse_hash_map<std::string, std::vector<std::string>> hash_tokens;+ spp::sparse_hash_map<std::string, std::vector<std::string>> fvalue_tokens;+ spp::sparse_hash_map<uint64_t, std::vector<std::string>> hash_tokens;// used for faceting grouped resultsspp::sparse_hash_map<uint32_t, spp::sparse_hash_set<uint32_t>> hash_groups;@@ -594,7 +601,7 @@ struct facet {facet_stats_t stats;//dictionary of key=>pair(range_id, range_val)- std::map<std::string, std::string> facet_range_map;+ std::map<int64_t, std::string> facet_range_map;bool is_range_query;@@ -604,16 +611,14 @@ struct facet {bool is_intersected = false;- bool get_range(std::string key, std::pair<std::string, std::string>& range_pair)- {- if(facet_range_map.empty())- {+ bool get_range(int64_t key, std::pair<int64_t, std::string>& range_pair) {+ if(facet_range_map.empty()) {LOG (ERROR) << "Facet range is not defined!!!";}+auto it = facet_range_map.lower_bound(key);- if(it != facet_range_map.end())- {+ if(it != facet_range_map.end()) {range_pair.first = it->first;range_pair.second = it->second;return true;@@ -622,17 +627,16 @@ struct facet {return false;}- explicit facet(const std::string& field_name,- std::map<std::string, std::string> facet_range = {}, bool is_range_q = false)- :field_name(field_name){- facet_range_map = facet_range;- is_range_query = is_range_q;+ explicit facet(const std::string& field_name, std::map<int64_t, std::string> facet_range = {},+ bool is_range_q = false) :field_name(field_name), facet_range_map(facet_range),+ is_range_query(is_range_q) {}};struct facet_info_t {// facet hash => resolved tokens- std::unordered_map<std::string, std::vector<std::string>> hashes;+ std::unordered_map<uint64_t, std::vector<std::string>> hashes;+ std::vector<std::string> fvalue_searched_tokens;bool use_facet_query = false;bool should_compute_stats = false;bool use_value_index = false;diff --git a/src/collection.cpp b/src/collection.cppindex 99bfacf0..11eff55e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2001,9 +2001,11 @@ Option<nlohmann::json> Collection::search(std::string raw_query,result["facet_counts"] = nlohmann::json::array();// populate facets- for(facet & a_facet: facets) {+ for(facet& a_facet: facets) {// Don't return zero counts for a wildcard facet.- if (a_facet.is_wildcard_match && a_facet.result_map.size() == 0) {+ if (a_facet.is_wildcard_match &&+ (((a_facet.is_intersected && a_facet.value_result_map.empty())) ||+ (!a_facet.is_intersected && a_facet.result_map.empty()))) {continue;}@@ -2020,28 +2022,28 @@ Option<nlohmann::json> Collection::search(std::string raw_query,facet_result["counts"] = nlohmann::json::array();std::vector<facet_value_t> facet_values;- std::vector<std::pair<std::string, facet_count_t>> facet_counts;+ std::vector<facet_count_t> facet_counts;for (const auto & kv : a_facet.result_map) {- facet_counts.emplace_back(std::make_pair(kv.first, kv.second));+ facet_count_t v = kv.second;+ v.fhash = kv.first;+ facet_counts.emplace_back(v);+ }++ for (const auto& kv : a_facet.value_result_map) {+ facet_count_t v = kv.second;+ v.fvalue = kv.first;+ v.fhash = StringUtils::hash_wy(kv.first.c_str(), kv.first.size());+ facet_counts.emplace_back(v);}auto max_facets = std::min(max_facet_values, facet_counts.size());auto nthElement = max_facets == facet_counts.size() ? max_facets - 1 : max_facets;- std::nth_element(facet_counts.begin(), facet_counts.begin() + nthElement,- facet_counts.end(), [&](const auto& kv1, const auto& kv2) {- size_t a_count = kv1.second.count;- size_t b_count = kv2.second.count;-- size_t a_value_size = UINT64_MAX - kv1.first.size();- size_t b_value_size = UINT64_MAX - kv2.first.size();-- return std::tie(a_count, a_value_size) > std::tie(b_count, b_value_size);- });+ std::nth_element(facet_counts.begin(), facet_counts.begin() + nthElement, facet_counts.end(),+ Collection::facet_count_compare);if(a_facet.is_range_query){- for(auto kv : a_facet.result_map){-+ for(const auto& kv : a_facet.result_map){auto facet_range_iter = a_facet.facet_range_map.find(kv.first);if(facet_range_iter != a_facet.facet_range_map.end()){auto & facet_count = kv.second;@@ -2059,13 +2061,11 @@ Option<nlohmann::json> Collection::search(std::string raw_query,for(size_t fi = 0; fi < max_facets; fi++) {// remap facet value hash with actual string- auto & kv = facet_counts[fi];- auto & facet_count = kv.second;-+ auto & facet_count = facet_counts[fi];std::string value;if(a_facet.is_intersected) {- value = kv.first;+ value = facet_count.fvalue;//LOG(INFO) << "used intersection";} else {// fetch actual facet value from representative doc id@@ -2089,7 +2089,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}std::unordered_map<std::string, size_t> ftoken_pos;- std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];+ std::vector<string>& ftokens = a_facet.is_intersected ? a_facet.fvalue_tokens[facet_count.fvalue] :+ a_facet.hash_tokens[facet_count.fhash];//LOG(INFO) << "working on hash_tokens for hash " << kv.first << " with size " << ftokens.size();for(size_t ti = 0; ti < ftokens.size(); ti++) {if(the_field.is_bool()) {@@ -4874,7 +4875,7 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vectorreturn Option<bool>(400, error);}- std::vector<std::tuple<std::string, std::string, std::string>> tupVec;+ std::vector<std::tuple<int64_t, int64_t, std::string>> tupVec;auto& range_map = a_facet.facet_range_map;for(const auto& range : result){@@ -4889,26 +4890,28 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vectorauto pos2 = range.find(",");auto pos3 = range.find("]");- std::string lower_range, upper_range;+ int64_t lower_range, upper_range;auto lower_range_start = pos1 + 2;auto lower_range_len = pos2 - lower_range_start;auto upper_range_start = pos2 + 1;auto upper_range_len = pos3 - upper_range_start;if(a_field.is_integer()) {- lower_range = range.substr(lower_range_start, lower_range_len);- StringUtils::trim(lower_range);- upper_range = range.substr(upper_range_start, upper_range_len);- StringUtils::trim(upper_range);+ std::string lower_range_str = range.substr(lower_range_start, lower_range_len);+ StringUtils::trim(lower_range_str);+ lower_range = std::stoll(lower_range_str);+ std::string upper_range_str = range.substr(upper_range_start, upper_range_len);+ StringUtils::trim(upper_range_str);+ upper_range = std::stoll(upper_range_str);} else {float val = std::stof(range.substr(pos1 + 2, pos2));- lower_range = std::to_string(Index::float_to_int64_t(val));+ lower_range = Index::float_to_int64_t(val);val = std::stof(range.substr(pos2 + 1, pos3));- upper_range = std::to_string(Index::float_to_int64_t(val));+ upper_range = Index::float_to_int64_t(val);}- tupVec.emplace_back(std::make_tuple(lower_range, upper_range, range_val));+ tupVec.emplace_back(lower_range, upper_range, range_val);}//sort the range values so that we can check continuitydiff --git a/src/index.cpp b/src/index.cppindex 49f38451..091c5dd2 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1295,32 +1295,29 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,//range facet processingif(a_facet.is_range_query) {const auto doc_val = kv.first;- std::pair<std::string, std::string> range_pair {};- if(a_facet.get_range(doc_val, range_pair)) {+ std::pair<int64_t , std::string> range_pair {};+ if(a_facet.get_range(std::stoll(doc_val), range_pair)) {const auto& range_id = range_pair.first;facet_count_t& facet_count = a_facet.result_map[range_id];facet_count.count = kv.second;}} else {if(use_facet_query) {- const auto fquery_hashes_it = fquery_hashes.find(facet_field.name);- if(fquery_hashes_it != fquery_hashes.end()) {- const auto& searched_tokens = fquery_hashes_it->second;- auto facet_str = kv.first;- transform(facet_str.begin(), facet_str.end(), facet_str.begin(), ::tolower);-- for(const auto& val : searched_tokens) {- if(facet_str.find(val) != std::string::npos) {- facet_count_t& facet_count = a_facet.result_map[kv.first];- facet_count.count = kv.second;-- a_facet.hash_tokens[kv.first] = searched_tokens;- }+ const auto& searched_tokens = facet_infos[findex].fvalue_searched_tokens;+ auto facet_str = kv.first;+ transform(facet_str.begin(), facet_str.end(), facet_str.begin(), ::tolower);++ for(const auto& val : searched_tokens) {+ if(facet_str.find(val) != std::string::npos) {+ facet_count_t& facet_count = a_facet.value_result_map[kv.first];+ facet_count.count = kv.second;++ a_facet.fvalue_tokens[kv.first] = searched_tokens;}}} else {- facet_count_t& facet_count = a_facet.result_map[kv.first];+ facet_count_t& facet_count = a_facet.value_result_map[kv.first];facet_count.count = kv.second;}}@@ -1389,18 +1386,17 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,compute_facet_stats(a_facet, fhash, facet_field.type);}- std::string fhash_str = std::to_string(fhash);if(a_facet.is_range_query) {int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);- std::pair<std::string, std::string> range_pair {};- if(a_facet.get_range(std::to_string(doc_val), range_pair)) {+ std::pair<int64_t , std::string> range_pair {};+ if(a_facet.get_range(doc_val, range_pair)) {const auto& range_id = range_pair.first;facet_count_t& facet_count = a_facet.result_map[range_id];facet_count.count += 1;}- } else if(!use_facet_query || fquery_hashes.find(fhash_str) != fquery_hashes.end()) {- facet_count_t& facet_count = a_facet.result_map[fhash_str];+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {+ facet_count_t& facet_count = a_facet.result_map[fhash];//LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;facet_count.doc_id = doc_seq_id;facet_count.array_pos = j;@@ -1411,7 +1407,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}if(use_facet_query) {//LOG (INFO) << "adding hash tokens for hash " << fhash;- a_facet.hash_tokens[fhash_str] = fquery_hashes.at(fhash_str);+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);}}}@@ -2891,7 +2887,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor(auto & facet_kv: this_facet.result_map) {uint32_t fhash = 0;if(group_limit) {- fhash = std::stoul(facet_kv.first);+ fhash = facet_kv.first;// we have to add all group setsacc_facet.hash_groups[fhash].insert(this_facet.hash_groups[fhash].begin(),@@ -2915,6 +2911,24 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consacc_facet.hash_tokens[facet_kv.first] = this_facet.hash_tokens[facet_kv.first];}+ for(auto& facet_kv: this_facet.value_result_map) {+ size_t count = 0;+ if(acc_facet.value_result_map.count(facet_kv.first) == 0) {+ // not found, so set it+ count = facet_kv.second.count;+ } else {+ count = acc_facet.value_result_map[facet_kv.first].count + facet_kv.second.count;+ }++ acc_facet.value_result_map[facet_kv.first].count = count;++ acc_facet.value_result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;+ acc_facet.value_result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;+ acc_facet.is_intersected = this_facet.is_intersected;++ acc_facet.fvalue_tokens[facet_kv.first] = this_facet.fvalue_tokens[facet_kv.first];+ }+if(this_facet.stats.fvcount != 0) {acc_facet.stats.fvcount += this_facet.stats.fvcount;acc_facet.stats.fvsum += this_facet.stats.fvsum;@@ -2927,9 +2941,15 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor(auto & acc_facet: facets) {for(auto& facet_kv: acc_facet.result_map) {if(group_limit) {- facet_kv.second.count = acc_facet.hash_groups[std::stoul(facet_kv.first)].size();+ facet_kv.second.count = acc_facet.hash_groups[facet_kv.first].size();+ }++ if(estimate_facets) {+ facet_kv.second.count = size_t(double(facet_kv.second.count) * (100.0f / facet_sample_percent));}+ }+ for(auto& facet_kv: acc_facet.value_result_map) {if(estimate_facets) {facet_kv.second.count = size_t(double(facet_kv.second.count) * (100.0f / facet_sample_percent));}@@ -4482,7 +4502,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&// since `field_result_ids` contains documents matched across all queries// value based indexfor(const auto& val : searched_tokens) {- facet_infos[findex].hashes[facet_field.name].emplace_back(val);+ facet_infos[findex].fvalue_searched_tokens.emplace_back(val);}}}@@ -4519,7 +4539,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(size_t array_index: array_indices) {if(array_index < facet_hashes.size()) {- std::string hash = std::to_string(facet_hashes[array_index]);+ uint32_t hash = facet_hashes[array_index];/*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "<< array_index;*/@@ -4531,7 +4551,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&}}} else {- std::string hash = std::to_string(facet_hashes[0]);+ uint32_t hash = facet_hashes[0];if(facet_infos[findex].hashes.count(hash) == 0) {//LOG(INFO) << "adding searched_tokens for hash " << hash;facet_infos[findex].hashes.emplace(hash, searched_tokens);