Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
hybrid approach preserving both structures
krunal1313595a9b493353
diff --git a/include/facet_index.h b/include/facet_index.hindex fecea8e7..29243f8d 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -2,7 +2,6 @@#include "ids_t.h"#include "tsl/htrie_map.h"-#include <list>class facet_index_t {private:@@ -16,13 +15,18 @@ private:uint32_t count;};+ struct facet_index_struct {+ void* id_list_ptr;+ uint32_t index;+ };+struct facet_index_counter {- tsl::htrie_map<char, void*> facet_index_map;- std::list<count_list> counter_list;-+ tsl::htrie_map<char, facet_index_struct> facet_index_map;+ std::vector<count_list> counter_list;+~facet_index_counter() {for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {- ids_t::destroy_list(it.value());+ ids_t::destroy_list(it.value().id_list_ptr);}facet_index_map.clear();@@ -32,23 +36,23 @@ private:};std::map<std::string, facet_index_counter> facet_field_map;+ uint32_t count_index = 0;public:facet_index_t() = default;~facet_index_t();- void insert(const std::string& field, const std::string& value, uint32_t id);+ uint32_t insert(const std::string& field, const std::string& value, uint32_t id);void erase(const std::string& field);bool contains(const std::string& field);- size_t size();+ size_t get_facet_count(const std::string& field);int intersect(const std::string& val, const uint32_t* result_ids, int result_id_len,int max_facet_count, std::map<std::string, uint32_t>& found);- int get_facet(const std::string& field, const std::vector<std::string>& searched_tokens,- std::vector<std::string>& facets);+ std::string get_facet_by_count_index(const std::string& field, uint32_t count_index);};\ No newline at end of filediff --git a/include/field.h b/include/field.hindex 1fa358ae..b42f6e0f 100644--- a/include/field.h+++ b/include/field.h@@ -637,11 +637,10 @@ struct facet {const std::string field_name;spp::sparse_hash_map<std::string, facet_count_t> result_map;// used for facet value query- //spp::sparse_hash_map<uint64_t, std::vector<std::string>> hash_tokens;- spp::sparse_hash_map<std::string, std::vector<std::string>> facet_tokens;+ spp::sparse_hash_map<uint32_t, std::vector<std::string>> hash_tokens;// used for faceting grouped results- //spp::sparse_hash_map<uint64_t, spp::sparse_hash_set<uint64_t>> hash_groups;+ spp::sparse_hash_map<uint32_t, spp::sparse_hash_set<uint32_t>> hash_groups;facet_stats_t stats;@@ -652,6 +651,8 @@ struct facet {bool sampled = false;+ bool is_intersected = false;+bool get_range(std::string key, std::pair<int64_t, std::string>& range_pair){if(facet_range_map.empty())@@ -680,9 +681,8 @@ struct facet {struct facet_info_t {// facet hash => resolved tokens- //std::unordered_map<uint64_t, std::vector<std::string>> hashes;+ std::unordered_map<uint64_t, std::vector<std::string>> hashes;//facet name => resolved tokens- std::unordered_map<std::string, std::vector<std::string>> facet_tokens;bool use_facet_query = false;bool should_compute_stats = false;field facet_field{"", "", false};@@ -701,11 +701,12 @@ struct facet_value_t {struct facet_hash_values_t {uint32_t length = 0;- uint64_t* hashes = nullptr;+ //uint32_t* hashes = nullptr;+ std::vector<uint32_t> hashes;facet_hash_values_t() {length = 0;- hashes = nullptr;+ //hashes = nullptr;}facet_hash_values_t(facet_hash_values_t&& hash_values) noexcept {@@ -713,17 +714,20 @@ struct facet_hash_values_t {hashes = hash_values.hashes;hash_values.length = 0;- hash_values.hashes = nullptr;+ //hash_values.hashes = nullptr;+ hash_values.hashes.clear();}facet_hash_values_t& operator=(facet_hash_values_t&& other) noexcept {if (this != &other) {- delete[] hashes;+ //delete[] hashes;+ hashes.clear();hashes = other.hashes;length = other.length;- other.hashes = nullptr;+ //other.hashes = nullptr;+ other.hashes.clear();other.length = 0;}@@ -731,8 +735,9 @@ struct facet_hash_values_t {}~facet_hash_values_t() {- delete [] hashes;- hashes = nullptr;+ //delete [] hashes;+ //hashes = nullptr;+ hashes.clear();}uint64_t size() const {@@ -740,6 +745,7 @@ struct facet_hash_values_t {}uint64_t back() const {- return hashes[length - 1];+ //return hashes[length - 1];+ return hashes.back();}};diff --git a/include/index.h b/include/index.hindex cf8f76c4..953dcf6d 100644--- a/include/index.h+++ b/include/index.h@@ -33,10 +33,10 @@#include "facet_index.h"static constexpr size_t ARRAY_FACET_DIM = 4;-//using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;-//using single_val_facet_map_t = spp::sparse_hash_map<uint32_t, uint64_t>;-//using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;-//using array_mapped_single_val_facet_t = std::array<single_val_facet_map_t*, ARRAY_FACET_DIM>;+using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;+using single_val_facet_map_t = spp::sparse_hash_map<uint32_t, uint32_t>;+using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;+using array_mapped_single_val_facet_t = std::array<single_val_facet_map_t*, ARRAY_FACET_DIM>;static constexpr size_t ARRAY_INFIX_DIM = 4;using array_mapped_infix_t = std::vector<tsl::htrie_set<char>*>;@@ -190,7 +190,7 @@ struct search_args {struct offsets_facet_hashes_t {std::unordered_map<std::string, std::vector<uint32_t>> offsets;- std::vector<uint64_t> facet_hashes;+ //std::vector<uint32_t> facet_hashes;};struct index_record {@@ -309,11 +309,12 @@ private:spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;// facet_field => (seq_id => values)- //spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;+ spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;+facet_index_t* facet_index_v4 = nullptr;// facet_field => (seq_id => hash)- //spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;+ spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;// sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;@@ -491,15 +492,15 @@ private:static void tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes);+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,+ std::vector<uint32_t>& facet_hashes*/);static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes);+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,+ std::vector<uint32_t>& facet_hashes*/);void collate_included_ids(const std::vector<token_t>& q_included_tokens,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,@@ -507,7 +508,7 @@ private:static uint64_t facet_token_hash(const field & a_field, const std::string &token);- static void compute_facet_stats(facet &a_facet, std::string raw_value, const std::string & field_type);+ static void compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type);static void handle_doc_ops(const tsl::htrie_map<char, field>& search_schema,nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc);diff --git a/src/collection.cpp b/src/collection.cppindex eb4ae625..b4e5849d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1920,22 +1920,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,facet_result["counts"] = nlohmann::json::array();std::vector<facet_value_t> facet_values;- // std::vector<std::pair<uint64_t, facet_count_t>> facet_hash_counts;-- // for (const auto & kv : a_facet.result_map) {- // facet_hash_counts.emplace_back(kv);- // }- auto the_field = search_schema.at(a_facet.field_name);- // keep only top K facets- //auto max_facets = std::min(max_facet_values, facet_hash_counts.size());- auto max_facets = std::min(max_facet_values, a_facet.result_map.size());- // std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,- // facet_hash_counts.end(), Collection::facet_count_compare);- //LOG (INFO) << "found facet size " << a_facet.result_map.size();- for(auto& kv : a_facet.result_map) {-- if(a_facet.is_range_query){+ if(a_facet.is_range_query){+ for(auto kv : a_facet.result_map){+auto facet_range_iter = a_facet.facet_range_map.find(kv.first);if(facet_range_iter != a_facet.facet_range_map.end()){auto & facet_count = kv.second;@@ -1945,39 +1933,78 @@ Option<nlohmann::json> Collection::search(std::string raw_query,else{LOG (ERROR) << "range_id not found in result map.";}- } else {- //facet_value_t facet_value = { kv.first, std::string(), kv.second.count};- //facet_values.emplace_back(facet_value);+ }+ }++ if(a_facet.is_intersected) {+ std::vector<std::pair<std::string, facet_count_t>> facet_counts;++ for (const auto & kv : a_facet.result_map) {+ facet_counts.emplace_back(std::make_pair(kv.first, kv.second));+ }++ auto max_facets = std::min(max_facet_values, facet_counts.size());+ std::sort(facet_counts.begin(), facet_counts.end(),+ [&](const auto& p1, const auto& p2) {+ return p1.second.count > p2.second.count;+ });++ for(int i = 0; i < max_facets; ++i) {+ if(a_facet.is_range_query) {+ break;+ }+ const auto& kv = facet_counts[i];+ facet_value_t facet_value = { kv.first, kv.first, kv.second.count};+ facet_values.emplace_back(facet_value);+ }+ } else {++ std::vector<std::pair<uint32_t, facet_count_t>> facet_hash_counts;++ for (const auto & kv : a_facet.result_map) {+ facet_hash_counts.emplace_back(std::make_pair(std::stoul(kv.first), kv.second));+ }++ auto the_field = search_schema.at(a_facet.field_name);+ // keep only top K facets+ auto max_facets = std::min(max_facet_values, facet_hash_counts.size());++ std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,+ facet_hash_counts.end(), Collection::facet_count_compare);+ for(size_t fi = 0; fi < max_facets; fi++) {++ if(a_facet.is_range_query){+ break;+ }// remap facet value hash with actual string- // auto & kv = facet_hash_counts[fi];- // auto & facet_count = kv.second;- // // fetch actual facet value from representative doc id- // const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);- // nlohmann::json document;- // const Option<bool> & document_op = get_document_from_store(seq_id_key, document);- // if(!document_op.ok()) {- // LOG(ERROR) << "Facet fetch error. " << document_op.error();- // continue;- // }- //std::string value;- // bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);- // if(!facet_found) {- // continue;- // }- std::string value = kv.first;+ auto & kv = facet_hash_counts[fi];+ auto & facet_count = kv.second;+ // fetch actual facet value from representative doc id+ const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);+ nlohmann::json document;+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);+ if(!document_op.ok()) {+ LOG(ERROR) << "Facet fetch error. " << document_op.error();+ continue;+ }+ std::string value;+ bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);+ if(!facet_found) {+ continue;+ }std::unordered_map<std::string, size_t> ftoken_pos;- //std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];- std::vector<string>& ftokens = a_facet.facet_tokens[kv.first];+ std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];+ //LOG(INFO) << "working on hash_tokens for hash " << kv.first << " with size " << ftokens.size();for(size_t ti = 0; ti < ftokens.size(); ti++) {- // if(the_field.is_bool()) {- // if(ftokens[ti] == "1") {- // ftokens[ti] = "true";- // } else {- // ftokens[ti] = "false";- // }- // }+ if(the_field.is_bool()) {+ if(ftokens[ti] == "1") {+ ftokens[ti] = "true";+ } else {+ ftokens[ti] = "false";+ }+ }const std::string& resolved_token = ftokens[ti];ftoken_pos[resolved_token] = ti;}@@ -2006,6 +2033,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}}}+ //LOG(INFO) << "token_offsets size " << token_offsets.size();auto offset_it = token_offsets.begin();size_t i = 0;std::stringstream highlightedss;@@ -2035,7 +2063,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,highlightedss << value[i];i++;}- facet_value_t facet_value = {value, highlightedss.str(), kv.second.count};+ facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};facet_values.emplace_back(facet_value);}}@@ -4617,7 +4645,7 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vectorreturn Option<bool>(400, error);}- std::vector<std::tuple<int64_t, int64_t, std::string>> tupVec;+ std::vector<std::tuple<std::string, std::string, std::string>> tupVec;auto& range_map = a_facet.facet_range_map;for(const auto& range : result){@@ -4626,7 +4654,6 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vectorstd::string error = "Facet range value is not valid.";return Option<bool>(400, error);}-auto pos1 = range.find(":");std::string range_val = range.substr(0, pos1);@@ -4654,8 +4681,8 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vectorfor(const auto& tup : tupVec){- const std::string& lower_range = std::to_string(std::get<0>(tup));- const std::string& upper_range = std::to_string(std::get<1>(tup));+ const auto& lower_range = std::get<0>(tup);+ const auto& upper_range = std::get<1>(tup);const std::string& range_val = std::get<2>(tup);//check if ranges are continous or notif((!range_map.empty()) && (range_map.find(lower_range)== range_map.end())){diff --git a/src/facet_index.cpp b/src/facet_index.cppindex 4a0e147f..0421653a 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -1,23 +1,30 @@#include "facet_index.h"+#include <tokenizer.h>+#include "string_utils.h"-void facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {-+uint32_t facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {auto& facet_index_map = facet_field_map[field].facet_index_map;-+ uint32_t index;const auto sv = value.substr(0, 100);if(facet_index_map.count(sv) == 0) {- facet_index_map.emplace(sv, SET_COMPACT_IDS(compact_id_list_t::create(1, {id})));+ index = ++count_index;++ facet_index_struct fis {};+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));+ fis.index = index;+ facet_index_map.emplace(sv, fis);}else {- auto ids = facet_index_map[sv];+ auto ids = facet_index_map[sv].id_list_ptr;if (!ids_t::contains(ids, id)) {ids_t::upsert(ids, id);- facet_index_map[sv] = ids;+ facet_index_map[sv].id_list_ptr = ids;}+ index = facet_index_map[sv].index;}-- const auto facet_count = ids_t::num_ids(facet_index_map.at(sv));+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);//LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;auto& counter_list = facet_field_map[field].counter_list;+ count_list node(sv, facet_count);if(counter_list.empty()) {counter_list.emplace_back(count_list(sv, facet_count));@@ -49,6 +56,8 @@ void facet_index_t::insert(const std::string& field, const std::string& value, ucounter_list.emplace_back(node);}}++ return index;}bool facet_index_t::contains(const std::string& field) {@@ -68,8 +77,14 @@ void facet_index_t::erase(const std::string& field) {}}-size_t facet_index_t::size() {- return facet_field_map.size();+size_t facet_index_t::get_facet_count(const std::string& field) {+ const auto it = facet_field_map.find(field);++ if(it == facet_field_map.end()) {+ return 0;+ }++ return it->second.counter_list.size();}//returns the count of matching seq_ids from result array@@ -90,12 +105,11 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_id// << " , counter_list size " << counter_list.size();std::vector<uint32_t> id_list;- const auto max_facets = std::min((int)counter_list.size(), max_facet_count);for(const auto& counter_list_it : counter_list) {- //LOG (INFO) << "checking ids in facet_value " << counter_list_it->facet_value- // << " having total count " << counter_list_it->count;+ // LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value+ // << " having total count " << counter_list_it.count;- auto ids = facet_index_map.at(counter_list_it.facet_value);+ auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;ids_t::uncompress(ids, id_list);const auto ids_len = id_list.size();int count = 0;@@ -108,8 +122,7 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_idif(count) {found[counter_list_it.facet_value] = count;-- if(found.size() == max_facets) {+ if(found.size() == 2*max_facet_count) {break;}}@@ -120,34 +133,22 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_idreturn found.size();}-int facet_index_t::get_facet(const std::string& field, const std::vector<std::string>& searched_tokens,- std::vector<std::string>& facets) {+std::string facet_index_t::get_facet_by_count_index(const std::string& field, uint32_t count_index) {const auto& facet_field_it = facet_field_map.find(field);if(facet_field_it == facet_field_map.end()) {- return 0;+ return "";}-+ std::string result = "";auto facet_index_map = facet_field_it->second.facet_index_map;- for(const auto& token : searched_tokens) {- auto token_string = token;- std::transform(token_string.begin(), token_string.end(), token_string.begin(), ::tolower);-- for(auto facet_index_map_it = facet_index_map.begin();- facet_index_map_it != facet_index_map.end(); ++facet_index_map_it) {-- auto facet_string = facet_index_map_it.key();- std::transform(facet_string.begin(), facet_string.end(), facet_string.begin(), ::tolower);-- if(facet_string.find(token_string) != std::string::npos) {- facets.emplace_back(facet_index_map_it.key());- }+ for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {+ if(it.value().index == count_index) {+ result = it.key();}}-- return facets.size();+ return result;}facet_index_t::~facet_index_t() {diff --git a/src/index.cpp b/src/index.cppindex 67943ff7..1f12a6ad 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -103,7 +103,7 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*}if(a_field.facet) {- //initialize_facet_indexes(a_field);+ initialize_facet_indexes(a_field);}// initialize for non-string facet fields@@ -186,23 +186,23 @@ Index::~Index() {str_sort_index.clear();- //for(auto& field_name_facet_map_array: facet_index_v3) {- // for(auto& facet_map: field_name_facet_map_array.second) {- // delete facet_map;- // facet_map = nullptr;- // }- //}//+ for(auto& field_name_facet_map_array: facet_index_v3) {+ for(auto& facet_map: field_name_facet_map_array.second) {+ delete facet_map;+ facet_map = nullptr;+ }+ }- //facet_index_v3.clear();+ facet_index_v3.clear();- ////for(auto& field_name_facet_map_array: single_val_facet_index_v3) {- // for(auto& facet_map: field_name_facet_map_array.second) {- //// delete facet_map;- // facet_map = nullptr;- // }- //}+ for(auto& field_name_facet_map_array: single_val_facet_index_v3) {+ for(auto& facet_map: field_name_facet_map_array.second) {+ delete facet_map;+ facet_map = nullptr;+ }+ }- //single_val_facet_index_v3.clear();+ single_val_facet_index_v3.clear();delete facet_index_v4;delete seq_ids;@@ -296,7 +296,7 @@ void Index::compute_token_offsets_facets(index_record& record,tokenize_string_array_with_facets(strings, is_facet, the_field,local_symbols_to_index, local_token_separators,- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);} else {std::string text;@@ -312,7 +312,7 @@ void Index::compute_token_offsets_facets(index_record& record,tokenize_string_with_facets(text, is_facet, the_field,local_symbols_to_index, local_token_separators,- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);}}@@ -321,15 +321,15 @@ void Index::compute_token_offsets_facets(index_record& record,if(the_field.type == field_types::STRING) {tokenize_string_with_facets(document[field_name], is_facet, the_field,local_symbols_to_index, local_token_separators,- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);} else {tokenize_string_array_with_facets(document[field_name], is_facet, the_field,local_symbols_to_index, local_token_separators,- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);}}- if(!offset_facet_hashes.offsets.empty() || !offset_facet_hashes.facet_hashes.empty()) {+ if(!offset_facet_hashes.offsets.empty() /*|| !offset_facet_hashes.facet_hashes.empty()*/) {record.field_index.emplace(field_name, std::move(offset_facet_hashes));}}@@ -652,98 +652,102 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>if(field_index_it == record.field_index.end()) {continue;}++ std::string value;if(afield.facet) {- //if(afield.is_array()) {- // facet_hash_values_t fhashvalues;- // fhashvalues.length = field_index_it->second.facet_hashes.size();- // fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];-- // for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {- // fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];- // }-- // auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- // if(facet_dim_index == nullptr) {- // LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- // } else {- // facet_dim_index->emplace(seq_id, std::move(fhashvalues));- // }- //} else {- // uint64_t fhash;- // fhash = field_index_it->second.facet_hashes[0];-- // auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- // if(facet_dim_index == nullptr) {- // LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- // } else {- // facet_dim_index->emplace(seq_id, std::move(fhash));- // }- //}- std::string value;- if(afield.type == field_types::INT32) {- int32_t raw_val = document[afield.name].get<int32_t>();- value = std::to_string(raw_val);- facet_index_v4->insert(afield.name, value, seq_id);- }- else if(afield.type == field_types::INT32_ARRAY) {- for(int i = 0; i < document[afield.name].size(); ++i) {- int32_t raw_val = document[afield.name][i].get<int32_t>();- value = std::to_string(raw_val);- facet_index_v4->insert(afield.name, value, seq_id);+ if(afield.is_array()) {+ facet_hash_values_t fhashvalues;++ if(afield.type == field_types::INT32_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ int32_t raw_val = document[afield.name][i].get<int32_t>();+ value = std::to_string(raw_val);+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }}- }- else if(afield.type == field_types::INT64) {- int64_t raw_val = document[afield.name].get<int64_t>();- value = std::to_string(raw_val);- facet_index_v4->insert(afield.name, value, seq_id);- }- else if(afield.type == field_types::INT64_ARRAY) {- for(int i = 0; i < document[afield.name].size(); ++i) {- int64_t raw_val = document[afield.name][i].get<int64_t>();+ else if(afield.type == field_types::INT64_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ int64_t raw_val = document[afield.name][i].get<int64_t>();+ value = std::to_string(raw_val);+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }+ }+ else if(afield.type == field_types::STRING_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ value = document[afield.name][i];+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }+ }+ else if(afield.type == field_types::FLOAT_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ float raw_val = document[afield.name][i].get<float>();+ value = StringUtils::float_to_str(raw_val);+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }+ }+ else if(afield.type == field_types::BOOL_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ value = std::to_string(document[afield.name][i].get<bool>());+ //value = (value == "1") ? "true" : "false";+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }+ }+ fhashvalues.length = fhashvalues.hashes.size();+ //LOG(INFO) << "fhashvalues.length " << fhashvalues.length;+ //fhashvalues.length = field_index_it->second.facet_hashes.size();+ //fhashvalues.hashes = new uint32_t[field_index_it->second.facet_hashes.size()];++ // for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {+ // fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];+ // }++ auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;+ } else {+ facet_dim_index->emplace(seq_id, std::move(fhashvalues));+ }+ } else {+ uint32_t fhash;++ if(afield.type == field_types::INT32) {+ int32_t raw_val = document[afield.name].get<int32_t>();value = std::to_string(raw_val);- facet_index_v4->insert(afield.name, value, seq_id);+ fhash = facet_index_v4->insert(afield.name, value, seq_id);}- }- else if(afield.type == field_types::STRING) {- value = document[afield.name];- facet_index_v4->insert(afield.name, value, seq_id);- }- else if(afield.type == field_types::STRING_ARRAY) {- for(int i = 0; i < document[afield.name].size(); ++i) {- value = document[afield.name][i];- facet_index_v4->insert(afield.name, value, seq_id);+ else if(afield.type == field_types::INT64) {+ int64_t raw_val = document[afield.name].get<int64_t>();+ value = std::to_string(raw_val);+ fhash = facet_index_v4->insert(afield.name, value, seq_id);}- }- else if(afield.type == field_types::FLOAT) {- float raw_val = document[afield.name].get<float>();- value = StringUtils::float_to_str(raw_val);- if(value != "0") {- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros+ else if(afield.type == field_types::STRING) {+ value = document[afield.name];+ fhash = facet_index_v4->insert(afield.name, value, seq_id);}- facet_index_v4->insert(afield.name, value, seq_id);- }- else if(afield.type == field_types::FLOAT) {- for(int i = 0; i < document[afield.name].size(); ++i) {- float raw_val = document[afield.name][i].get<float>();+ else if(afield.type == field_types::FLOAT) {+ float raw_val = document[afield.name].get<float>();value = StringUtils::float_to_str(raw_val);- if(value != "0") {- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros- }- facet_index_v4->insert(afield.name, value, seq_id);+ fhash = facet_index_v4->insert(afield.name, value, seq_id);}- }- else if(afield.type == field_types::BOOL) {- value = std::to_string(document[afield.name].get<bool>());- value = (value == "1") ? "true" : "false";- facet_index_v4->insert(afield.name, value, seq_id);- }- else if(afield.type == field_types::BOOL) {- for(int i = 0; i < document[afield.name].size(); ++i) {- value = std::to_string(document[afield.name][i].get<bool>());- value = (value == "1") ? "true" : "false";- facet_index_v4->insert(afield.name, value, seq_id);+ else if(afield.type == field_types::BOOL) {+ value = std::to_string(document[afield.name].get<bool>());+ //value = (value == "1") ? "true" : "false";+ fhash = facet_index_v4->insert(afield.name, value, seq_id);}+ //fhash = field_index_it->second.facet_hashes[0];++ auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;+ } else {+ facet_dim_index->emplace(seq_id, std::move(fhash));+ }}}@@ -1055,8 +1059,8 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &tokenvoid Index::tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes) {+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,+ std::vector<uint32_t>& facet_hashes*/) {Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);std::string token;@@ -1075,15 +1079,14 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,token_to_offsets[token].push_back(token_index + 1);last_token = token;-- if(is_facet) {- uint64_t token_hash = Index::facet_token_hash(a_field, token);- if(token_index == 0) {- facet_hash = token_hash;- } else {- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);- }- }+ // if(is_facet) {+ // uint64_t token_hash = Index::facet_token_hash(a_field, token);+ // if(token_index == 0) {+ // facet_hash = token_hash;+ // } else {+ // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);+ // }+ // }}if(!token_to_offsets.empty()) {@@ -1091,17 +1094,17 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,token_to_offsets[last_token].push_back(0);}- if(is_facet) {- facet_hashes.push_back(facet_hash);- }+ // if(is_facet) {+ // facet_hashes.push_back(facet_hash);+ // }}void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes) {+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,+ std::vector<uint32_t>& facet_hashes*/) {for(size_t array_index = 0; array_index < strings.size(); array_index++) {const std::string& str = strings[array_index];@@ -1126,19 +1129,19 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& sttoken_set.insert(token);last_token = token;- if(is_facet) {- uint64_t token_hash = Index::facet_token_hash(a_field, token);- if(token_index == 0) {- facet_hash = token_hash;- } else {- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);- }- }+ // if(is_facet) {+ // uint64_t token_hash = Index::facet_token_hash(a_field, token);+ // if(token_index == 0) {+ // facet_hash = token_hash;+ // } else {+ // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);+ // }+ // }}- if(is_facet) {- facet_hashes.push_back(facet_hash);- }+ // if(is_facet) {+ // facet_hashes.push_back(facet_hash);+ // }if(token_set.empty()) {continue;@@ -1157,22 +1160,22 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st}}void Index::initialize_facet_indexes(const field& facet_field) {- // if(facet_field.is_array()) {- // array_mapped_facet_t facet_array;- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- // facet_array[i] = new facet_map_t();- // }- // facet_index_v3.emplace(facet_field.name, facet_array);- // } else {- // array_mapped_single_val_facet_t facet_array;- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- // facet_array[i] = new single_val_facet_map_t();- // }- // single_val_facet_index_v3.emplace(facet_field.name, facet_array);- // }+ if(facet_field.is_array()) {+ array_mapped_facet_t facet_array;+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ facet_array[i] = new facet_map_t();+ }+ facet_index_v3.emplace(facet_field.name, facet_array);+ } else {+ array_mapped_single_val_facet_t facet_array;+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ facet_array[i] = new single_val_facet_map_t();+ }+ single_val_facet_index_v3.emplace(facet_field.name, facet_array);+ }}-void Index::compute_facet_stats(facet &a_facet, std::string raw_value, const std::string & field_type) {+void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {int32_t val = std::stoi(raw_value);if (val < a_facet.stats.fvmin) {@@ -1184,7 +1187,7 @@ void Index::compute_facet_stats(facet &a_facet, std::string raw_value, const stda_facet.stats.fvsum += val;a_facet.stats.fvcount++;} else if(field_type == field_types::INT64 || field_type == field_types::INT64_ARRAY) {- int64_t val = std::stol(raw_value);+ int64_t val = std::stoll(raw_value);if(val < a_facet.stats.fvmin) {a_facet.stats.fvmin = val;}@@ -1217,126 +1220,136 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,auto& a_facet = facets[findex];const auto& facet_field = facet_infos[findex].facet_field;const bool use_facet_query = facet_infos[findex].use_facet_query;- //const auto& fquery_hashes = facet_infos[findex].hashes;- const auto& fquery_facet_tokens = facet_infos[findex].facet_tokens;+ const auto& fquery_hashes = facet_infos[findex].hashes;const bool should_compute_stats = facet_infos[findex].should_compute_stats;auto sort_index_it = sort_index.find(a_facet.field_name);size_t mod_value = 100 / facet_sample_percent;- // facet_map_t::iterator facet_map_it;- // single_val_facet_map_t::iterator single_facet_map_it;- // uint64_t fhash = 0;- // size_t facet_hash_count = 1;- // const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- // const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);- std::map<std::string, uint32_t> facet_results;- facet_index_v4->intersect(a_facet.field_name, result_ids,- results_size, max_facet_count, facet_results);- //LOG(INFO) << "facet_results size " << facet_results.size();-- for(const auto& kv : facet_results) {- //range facet processing- if(a_facet.is_range_query) {- const auto doc_val = kv.first;- std::pair<std::string, std::string> range_pair {};- if(a_facet.get_range(doc_val, range_pair)) {- const auto& range_id = range_pair.first;- facet_count_t& facet_count = a_facet.result_map[range_id];+ const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);+ const auto records_to_search = facet_records * results_size;++ //LOG(INFO) << "results size " << results_size;+ //LOG(INFO) << "records_to_search : " << records_to_search;+ if(use_facet_query == false && group_limit == 0 && records_to_search != 0+ && records_to_search < 10000) {+ //LOG(INFO) << "Using intersection to find facets";+ a_facet.is_intersected = true;++ std::map<std::string, uint32_t> facet_results;+ facet_index_v4->intersect(a_facet.field_name, result_ids,+ results_size, max_facet_count, facet_results);+ //LOG(INFO) << "facet_results size " << facet_results.size();++ for(const auto& kv : facet_results) {+ //range facet processing+ if(a_facet.is_range_query) {+ const auto doc_val = kv.first;+ std::pair<std::string, std::string> range_pair {};+ if(a_facet.get_range(doc_val, range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count = kv.second;+ }+ } else {+ facet_count_t& facet_count = a_facet.result_map[kv.first];facet_count.count = kv.second;}- } else if(use_facet_query) {- if (fquery_facet_tokens.find(kv.first) != fquery_facet_tokens.end()) {- a_facet.facet_tokens[kv.first] = fquery_facet_tokens.at(kv.first);-- facet_count_t& facet_count = a_facet.result_map[kv.first];- facet_count.count = kv.second;++ if(should_compute_stats) {+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;+ for(int i = 0; i < kv.second; ++i) {+ compute_facet_stats(a_facet, kv.first, facet_field.type);}- } else {- facet_count_t& facet_count = a_facet.result_map[kv.first];- facet_count.count = kv.second;- }+ }+ }+ } else {+ facet_map_t::iterator facet_map_it;+ single_val_facet_map_t::iterator single_facet_map_it;+ uint32_t fhash = 0;+ size_t facet_hash_count = 1;++ const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);- if(should_compute_stats) {- //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;- for(int i = 0; i < kv.second; ++i) {- compute_facet_stats(a_facet, kv.first, facet_field.type);+ for(size_t i = 0; i < results_size; i++) {+ // if sampling is enabled, we will skip a portion of the results to speed up things+ if(estimate_facets) {+ if(i % mod_value != 0) {+ continue;+ }}- }- }- }- // for(size_t i = 0; i < results_size; i++) {- // // if sampling is enabled, we will skip a portion of the results to speed up things- // if(estimate_facets) {- // if(i % mod_value != 0) {- // continue;- // }- // }-- // uint32_t doc_seq_id = result_ids[i];- // if(facet_field.is_array()) {- // const auto& field_facet_mapping = field_facet_mapping_it->second;- // facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- // if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- // continue;- // }- // facet_hash_count = facet_map_it->second.size();- // } else {- // const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- // single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- // if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- // continue;- // }- // facet_hash_count = 1;- // fhash = single_facet_map_it->second;- // }+ uint32_t doc_seq_id = result_ids[i];+ if(facet_field.is_array()) {+ const auto& field_facet_mapping = field_facet_mapping_it->second;+ facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);- // const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;- // //for(size_t j = 0; j < facet_hash_count; j++) {- // // if(facet_field.is_array()) {- // // fhash = facet_map_it->second.hashes[j];- // // }- // const auto& doc_id = found_doc_seq_ids[j].second;- // if(should_compute_stats) {- // compute_facet_stats(a_facet, doc_id, facet_field.type);- // }- // if(a_facet.is_range_query) {- // auto sort_index_it = sort_index.find(a_facet.field_name);- // if(sort_index_it != sort_index.end()){- // auto doc_id_val_map = sort_index_it->second;- // auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);- // if(doc_seq_id_it != doc_id_val_map->end()){-- // int64_t doc_val = doc_seq_id_it->second;- // std::pair<int64_t, std::string> range_pair {};- // if(a_facet.get_range(doc_val, range_pair)) {- // int64_t range_id = range_pair.first;- // facet_count_t& facet_count = a_facet.result_map[range_id];- // facet_count.count += 1;- // }- // }- // }- // } else if(!use_facet_query || fquery_doc_id_tokens.find(doc_id) != fquery_doc_id_tokens.end()) {- // facet_count_t& facet_count = a_facet.result_map[doc_id];- // //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;- // facet_count.doc_id = doc_seq_id;- // facet_count.array_pos = j;- // if(group_limit) {- // a_facet.hash_groups[doc_id].emplace(distinct_id);- // } else {- // facet_count.count += 1;- // }- // if(use_facet_query) {- // a_facet.hash_tokens[doc_id] = fquery_doc_id_tokens.at(doc_id);- // }- // }- // }- //}- //}+ if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }+ facet_hash_count = facet_map_it->second.size();+ } else {+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;+ single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);++ if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }+ facet_hash_count = 1;+ fhash = single_facet_map_it->second;+ }++ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;+ for(size_t j = 0; j < facet_hash_count; j++) {+ if(facet_field.is_array()) {+ fhash = facet_map_it->second.hashes[j];+ }+ if(should_compute_stats) {+ std::string fvalue =+ facet_index_v4->get_facet_by_count_index(a_facet.field_name, fhash);+ if(!fvalue.empty()) {+ compute_facet_stats(a_facet, fvalue, facet_field.type);+ }+ }+ if(a_facet.is_range_query) {+ auto sort_index_it = sort_index.find(a_facet.field_name);+ if(sort_index_it != sort_index.end()){+ auto doc_id_val_map = sort_index_it->second;+ auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);+ if(doc_seq_id_it != doc_id_val_map->end()){++ std::string doc_val = std::to_string(doc_seq_id_it->second);+ std::pair<std::string, std::string> range_pair {};+ if(a_facet.get_range(doc_val, range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count += 1;+ }+ }+ }+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {+ std::string fhash_str = std::to_string(fhash);+ facet_count_t& facet_count = a_facet.result_map[fhash_str];+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;+ facet_count.doc_id = doc_seq_id;+ facet_count.array_pos = j;+ if(group_limit) {+ a_facet.hash_groups[fhash].emplace(distinct_id);+ } else {+ facet_count.count += 1;+ }+ if(use_facet_query) {+ //LOG (INFO) << "adding hash tokens for hash " << fhash;+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);+ }+ }+ }+ }+ }+ }}void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {@@ -2429,23 +2442,19 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conswhile (it.valid()) {uint32_t seq_id = it.id();uint64_t distinct_id = seq_id;- // if (group_limit != 0) {- // distinct_id = get_distinct_id(group_by_fields, seq_id);- // if(excluded_group_ids.count(distinct_id) != 0) {- // continue;- // }- // }+ if (group_limit != 0) {+ distinct_id = get_distinct_id(group_by_fields, seq_id);+ if(excluded_group_ids.count(distinct_id) != 0) {+ continue;+ }+ }int64_t scores[3] = {0};scores[0] = seq_id;int64_t match_score_index = -1;result_ids.push_back(seq_id);- if(group_limit == 0) {- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);- topster->add(&kv);- }-+KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, nullptr);int ret = topster->add(&kv);@@ -2541,12 +2550,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}uint64_t distinct_id = seq_id;- // if (group_limit != 0) {- // distinct_id = get_distinct_id(group_by_fields, seq_id);- // if(excluded_group_ids.count(distinct_id) != 0) {- // continue;- // }- // }+ if (group_limit != 0) {+ distinct_id = get_distinct_id(group_by_fields, seq_id);+ if(excluded_group_ids.count(distinct_id) != 0) {+ continue;+ }+ }auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :dist_label.first;@@ -2564,9 +2573,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consKV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, nullptr);int ret = topster->add(&kv);- // if(group_limit != 0 && ret < 2) {- // groups_processed[distinct_id]++;- // }+ if(group_limit != 0 && ret < 2) {+ groups_processed[distinct_id]++;+ }nearest_ids.push_back(seq_id);}@@ -2860,8 +2869,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consbool estimate_facets = (facet_sample_percent < 100 && all_result_ids_len > facet_sample_threshold);if(!facets.empty()) {- //const size_t num_threads = std::min(concurrency, all_result_ids_len);- const size_t num_threads = 1;+ const size_t num_threads = std::min(concurrency, all_result_ids_len);+ //const size_t num_threads = 1;const size_t window_size = (num_threads == 0) ? 0 :(all_result_ids_len + num_threads - 1) / num_threads; // rounds upsize_t num_processed = 0;@@ -2875,7 +2884,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<std::vector<facet>> facet_batches(num_threads);for(size_t i = 0; i < num_threads; i++) {for(const auto& this_facet: facets) {- facet_batches[i].emplace_back(facet(this_facet.field_name, this_facet.facet_range_map, this_facet.is_range_query));+ facet_batches[i].emplace_back(facet(this_facet.field_name, this_facet.facet_range_map,+ this_facet.is_range_query));}}@@ -2930,12 +2940,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consauto& this_facet = facet_batch[fi];auto& acc_facet = facets[fi];for(auto & facet_kv: this_facet.result_map) {+ uint32_t fhash = 0;if(group_limit) {+ fhash = std::stoul(facet_kv.first);// we have to add all group sets- // acc_facet.hash_groups[facet_kv.first].insert(- // this_facet.hash_groups[facet_kv.first].begin(),- // this_facet.hash_groups[facet_kv.first].end()- // );+ acc_facet.hash_groups[fhash].insert(+ this_facet.hash_groups[fhash].begin(),+ this_facet.hash_groups[fhash].end()+ );} else {size_t count = 0;if(acc_facet.result_map.count(facet_kv.first) == 0) {@@ -2947,10 +2959,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consacc_facet.result_map[facet_kv.first].count = count;}- //acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;- //acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;- //acc_facet.hash_tokens[facet_kv.first] = this_facet.hash_tokens[facet_kv.first];- acc_facet.facet_tokens[facet_kv.first] = this_facet.facet_tokens[facet_kv.first];+ acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;+ acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;+ acc_facet.is_intersected = this_facet.is_intersected;+ if(!facet_query.query.empty()) {+ fhash = std::stoul(facet_kv.first);+ //LOG(INFO) << "copying hash tokens for hash " << fhash;+ acc_facet.hash_tokens[fhash] = this_facet.hash_tokens[fhash];+ }}if(this_facet.stats.fvcount != 0) {@@ -2964,9 +2980,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor(auto & acc_facet: facets) {for(auto& facet_kv: acc_facet.result_map) {- // if(group_limit) {- // facet_kv.second.count = acc_facet.hash_groups[facet_kv.first].size();- // }+ if(group_limit) {+ facet_kv.second.count = acc_facet.hash_groups[std::stoul(facet_kv.first)].size();+ }if(estimate_facets) {facet_kv.second.count = size_t(double(facet_kv.second.count) * (100.0f / facet_sample_percent));@@ -3699,12 +3715,12 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,}uint64_t distinct_id = seq_id;- // if(group_limit != 0) {- // distinct_id = get_distinct_id(group_by_fields, seq_id);- // if(excluded_group_ids.count(distinct_id) != 0) {- // return;- // }- // }+ if(group_limit != 0) {+ distinct_id = get_distinct_id(group_by_fields, seq_id);+ if(excluded_group_ids.count(distinct_id) != 0) {+ return;+ }+ }int64_t scores[3] = {0};int64_t match_score_index = -1;@@ -4301,20 +4317,18 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se100, scores, match_score_index);uint64_t distinct_id = seq_id;- // if(group_limit != 0) {- // distinct_id = get_distinct_id(group_by_fields, seq_id);- // if(excluded_group_ids.count(distinct_id) != 0) {- // continue;- // }- // }- if(group_limit == 0) {+ if(group_limit != 0) {+ distinct_id = get_distinct_id(group_by_fields, seq_id);+ if(excluded_group_ids.count(distinct_id) != 0) {+ continue;+ }+ }KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);int ret = actual_topster->add(&kv);- }- // if(group_limit != 0 && ret < 2) {- // groups_processed[distinct_id]++;- // }+ if(group_limit != 0 && ret < 2) {+ groups_processed[distinct_id]++;+ }if(((i + 1) % (1 << 12)) == 0) {@@ -4404,13 +4418,13 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(size_t findex=0; findex < facets.size(); findex++) {const auto& a_facet = facets[findex];- // const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- // const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);+ const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);+ const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);- // if((field_facet_mapping_it == facet_index_v3.end())- // && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {- // continue;- // }+ if((field_facet_mapping_it == facet_index_v3.end())+ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ continue;+ }facet_infos[findex].use_facet_query = false;@@ -4475,16 +4489,6 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(auto leaf: searched_query) {posting_lists.push_back(leaf->values);std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);-- //convert again to boolean string to help search in facet_index map- if (facet_field.is_bool()) {- if (tok == "1") {- tok = "true";- } else if (tok == "0") {- tok = "false";- }- }-searched_tokens.push_back(tok);//LOG(INFO) << "tok: " << tok;}@@ -4513,45 +4517,38 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&}//LOG(INFO) << "seq_id matched : " << seq_id;- // if(facet_field.is_array()) {- // const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- // if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- // continue;- // }-- // std::vector<size_t> array_indices;- // posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);-- // for(size_t array_index: array_indices) {- // if(array_index < doc_fvalues_it->second.length) {- // uint64_t hash = doc_fvalues_it->second.hashes[array_index];-- // /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "- // << array_index;*/-- // if(facet_infos[findex].hashes.count(hash) == 0) {- // facet_infos[findex].hashes.emplace(hash, searched_tokens);- // }- // }- // }- // } else {- // const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- // if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- // continue;- // }+ if(facet_field.is_array()) {+ const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }++ std::vector<size_t> array_indices;+ posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);++ for(size_t array_index: array_indices) {+ if(array_index < doc_fvalues_it->second.length) {+ uint32_t hash = doc_fvalues_it->second.hashes[array_index];++ /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "+ << array_index;*/++ if(facet_infos[findex].hashes.count(hash) == 0) {+ //LOG(INFO) << "adding searched_tokens for hash " << hash;+ facet_infos[findex].hashes.emplace(hash, searched_tokens);+ }+ }+ }+ } else {+ const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }- // uint64_t hash = doc_single_fvalues_it->second;- // if(facet_infos[findex].hashes.count(hash) == 0) {- // facet_infos[findex].hashes.emplace(hash, searched_tokens);- // }- // }- }- std::vector<std::string> matched_facets;- if(facet_index_v4->get_facet(a_facet.field_name, searched_tokens, matched_facets)) {- for(const auto& facet : matched_facets) {- if(facet_infos[findex].facet_tokens.count(facet) == 0) {- LOG(INFO) << "adding facet " << facet << " in facet_info";- facet_infos[findex].facet_tokens.emplace(facet, searched_tokens);+ uint32_t hash = doc_single_fvalues_it->second;+ if(facet_infos[findex].hashes.count(hash) == 0) {+ //LOG(INFO) << "adding searched_tokens for hash " << hash;+ facet_infos[findex].hashes.emplace(hash, searched_tokens);}}}@@ -4664,20 +4661,18 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,100, scores, match_score_index);uint64_t distinct_id = seq_id;- // if(group_limit != 0) {- // distinct_id = get_distinct_id(group_by_fields, seq_id);- // if(excluded_group_ids.count(distinct_id) != 0) {- // continue;- // }- // }- if(group_limit == 0) {+ if(group_limit != 0) {+ distinct_id = get_distinct_id(group_by_fields, seq_id);+ if(excluded_group_ids.count(distinct_id) != 0) {+ continue;+ }+ }KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);int ret = topsters[thread_id]->add(&kv);- }- // if(group_limit != 0 && ret < 2) {- // tgroups_processed[thread_id][distinct_id]++;- // }+ if(group_limit != 0 && ret < 2) {+ tgroups_processed[thread_id][distinct_id]++;+ }if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {// check only once every 2^15 docs to reduce overheadBREAK_CIRCUIT_BREAKER@@ -5263,16 +5258,12 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16uint64_t distinct_id = seq_id;- // if(group_limit != 0) {- // distinct_id = get_distinct_id(group_by_fields, seq_id);- // groups_processed.emplace(distinct_id);- // }+ if(group_limit != 0) {+ distinct_id = get_distinct_id(group_by_fields, seq_id);+ groups_processed.emplace(distinct_id);+ }//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;- if(group_limit == 0) {- KV kv(query_index, seq_id, distinct_id, match_score_index, scores);- topster->add(&kv);- }KV kv(query_index, seq_id, distinct_id, match_score_index, scores);int ret = topster->add(&kv);if(group_limit != 0 && ret < 2) {@@ -5286,46 +5277,46 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16// pre-filter group_by_fields such that we can avoid the find() checkuint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,const uint32_t seq_id) const {- uint64_t distinct_id = 1; // some constant initial value+ uint32_t distinct_id = 1; // some constant initial value// calculate hash from group_by_fieldsfor(const auto& field: group_by_fields) {- // const auto& field_facet_mapping_it = facet_index_v3.find(field);- // const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);- // if((field_facet_mapping_it == facet_index_v3.end())- // && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {- // continue;- // }- if(!facet_index_v4->contains(field)) {+ const auto& field_facet_mapping_it = facet_index_v3.find(field);+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);+ if((field_facet_mapping_it == facet_index_v3.end())+ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {continue;}+ // if(!facet_index_v4->contains(field)) {+ // continue;+ // }- // if(search_schema.at(field).is_array()) {- // const auto& field_facet_mapping = field_facet_mapping_it->second;- // const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);-- // if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- // continue;- // }-- // const auto& facet_hashes = facet_hashes_it->second;+ if(search_schema.at(field).is_array()) {+ const auto& field_facet_mapping = field_facet_mapping_it->second;+ const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);- // for(size_t i = 0; i < facet_hashes.size(); i++) {- // distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);- // }- // } else {- // const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- // const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }- // if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- // continue;- // }+ const auto& facet_hashes = facet_hashes_it->second;+ //LOG(INFO) << "combining hashes for facet array ";+ for(size_t i = 0; i < facet_hashes.size(); i++) {+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);+ }+ } else {+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;+ const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);- // const auto& facet_hash = facet_hashes_it->second;+ if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }- // distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);- // }+ const auto& facet_hash = facet_hashes_it->second;+ //LOG(INFO) << "combining hashes for facet ";+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);+ }}-+ //LOG(INFO) << "distinct id " << distinct_id;return distinct_id;}@@ -5559,21 +5550,21 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const}// remove facets- // const auto& field_facets_it = facet_index_v3.find(field_name);- // if(field_facets_it != facet_index_v3.end()) {- // const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- // if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- // field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);- // }- // }+ const auto& field_facets_it = facet_index_v3.find(field_name);+ if(field_facets_it != facet_index_v3.end()) {+ const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);+ }+ }- // const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);- // if(field_single_val_facets_it != single_val_facet_index_v3.end()) {- // const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- // if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- // field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);- // }- // }+ const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);+ if(field_single_val_facets_it != single_val_facet_index_v3.end()) {+ const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);+ }+ }facet_index_v4->erase(field_name);@@ -5713,7 +5704,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecif(new_field.is_facet()) {- //initialize_facet_indexes(new_field);+ initialize_facet_indexes(new_field);// initialize for non-string facet fieldsif(!new_field.is_string()) {@@ -5777,21 +5768,21 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec}if(del_field.is_facet()) {- // if(del_field.is_array()) {- // auto& arr = facet_index_v3[del_field.name];- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- // delete arr[i];- // }+ if(del_field.is_array()) {+ auto& arr = facet_index_v3[del_field.name];+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ delete arr[i];+ }- // facet_index_v3.erase(del_field.name);- // } else {- // auto& arr = single_val_facet_index_v3[del_field.name];- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- // delete arr[i];- // }+ facet_index_v3.erase(del_field.name);+ } else {+ auto& arr = single_val_facet_index_v3[del_field.name];+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ delete arr[i];+ }- // single_val_facet_index_v3.erase(del_field.name);- // }+ single_val_facet_index_v3.erase(del_field.name);+ }facet_index_v4->erase(del_field.name);if(!del_field.is_string()) {diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex ad1cb4db..8827c678 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -70,7 +70,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {std::vector<std::string> facets = {"tags"};// single facet with no filters- nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());@@ -97,17 +97,18 @@ TEST_F(CollectionFacetingTest, FacetCounts) {results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1,FREQUENCY, {false}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 2).get();+ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][1]["count"]);// 2 facets, 1 text query with no filtersfacets.clear();@@ -230,12 +231,13 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_STREQ("age", results["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("24", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("21", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("21", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("24", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+// facet on a float field without query to check on statsresults = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, {0}, 10, 1, FREQUENCY,@@ -615,8 +617,6 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "categories:cell ph").get();- LOG(INFO) << results.dump();-ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());@@ -636,8 +636,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());collectionManager.drop_collection("coll1");@@ -674,7 +674,6 @@ TEST_F(CollectionFacetingTest, FacetStatOnFloatFields) {1, FREQUENCY, {false});auto results = res_op.get();- LOG(INFO) << results.dump();ASSERT_EQ(7, results["hits"].size());@@ -1219,14 +1218,20 @@ TEST_F(CollectionFacetingTest, RangeFacetTest) {ASSERT_TRUE(coll1->add(doc4.dump()).ok());ASSERT_TRUE(coll1->add(doc5.dump()).ok());- auto results = coll1->search("Karnataka", {"state"},+ auto result = coll1->search("Karnataka", {"state"},"", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},{}, {2}, 10,1, FREQUENCY, {true},10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000,- true, false, true, "", true).get();+ true, false, true, "", true);+ if(!result.ok()) {+ LOG(INFO) << result.error();+ }++ auto results = result.get();+ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);ASSERT_EQ("Busy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());@@ -1642,8 +1647,6 @@ TEST_F(CollectionFacetingTest, FacetIndexRefactor) {// single facet with no filtersnlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();- LOG(INFO) << results.dump();-ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(1, results["facet_counts"].size());ASSERT_EQ(4, results["facet_counts"][0].size());diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex c094f9fc..f88d9551 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -69,8 +69,6 @@ TEST_F(CollectionGroupingTest, GroupingBasics) {"", 10,{}, {}, {"size"}, 2).get();- LOG(INFO) << res.dump();-ASSERT_EQ(3, res["found"].get<size_t>());ASSERT_EQ(3, res["grouped_hits"].size());ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());@@ -454,10 +452,10 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());-+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex ab0e951e..e01e7b27 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2384,7 +2384,7 @@ TEST_F(CollectionTest, UpdateDocument) {res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();-+ASSERT_EQ(1, res["hits"].size());ASSERT_EQ(1, res["facet_counts"].size());ASSERT_STREQ("tags", res["facet_counts"][0]["field_name"].get<std::string>().c_str());