Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
inital changes for refactor
krunal13138b4e95e421e9
diff --git a/include/facet_index.h b/include/facet_index.hnew file mode 100644index 00000000..eced703f--- /dev/null+++ b/include/facet_index.h@@ -0,0 +1,43 @@+#pragma once++#include "ids_t.h"+#include "tsl/htrie_map.h"+#include <list>+#include <set>+#include <mutex>++class facet_index_t {+private:+ struct count_list {+ count_list(const std::string& sv, uint32_t facet_count) {+ facet_value = sv;+ count = facet_count;+ }++ std::string facet_value;+ uint32_t count;+ };++ struct facet_index_counter {+ tsl::htrie_map<char, void*> facet_index_map;+ std::list<count_list> counter_list;+ };++ std::map<std::string, facet_index_counter> facet_field_map;+public:++ ~facet_index_t();++ void insert(const std::string& field, const std::string& value, uint32_t id);++ size_t get(const std::string& field, std::map<std::string,std::vector<uint32_t>>& result_ids);++ void erase(const std::string& field);++ bool contains(const std::string& field);++ size_t size();++ int intersect(const std::string& val, const uint32_t* result_ids, int result_id_len,+ int max_facet_count, std::map<std::string, uint32_t>& found);+};\ No newline at end of filediff --git a/include/field.h b/include/field.hindex 8b4fe04e..5c1ed7ac 100644--- a/include/field.h+++ b/include/field.h@@ -682,7 +682,9 @@ struct facet {struct facet_info_t {// facet hash => resolved tokens- std::unordered_map<uint64_t, std::vector<std::string>> hashes;+ //std::unordered_map<uint64_t, std::vector<std::string>> hashes;+ //facet name => resolved tokens+ std::unordered_map<uint32_t, std::vector<std::string>> doc_id_tokens;bool use_facet_query = false;bool should_compute_stats = false;field facet_field{"", "", false};diff --git a/include/index.h b/include/index.hindex c01ed672..e29879ed 100644--- a/include/index.h+++ b/include/index.h@@ -30,12 +30,13 @@#include "vector_query_ops.h"#include "hnswlib/hnswlib.h"#include "filter.h"+#include "facet_index.h"static constexpr size_t ARRAY_FACET_DIM = 4;-using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;-using single_val_facet_map_t = spp::sparse_hash_map<uint32_t, uint64_t>;-using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;-using array_mapped_single_val_facet_t = std::array<single_val_facet_map_t*, ARRAY_FACET_DIM>;+//using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;+//using single_val_facet_map_t = spp::sparse_hash_map<uint32_t, uint64_t>;+//using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;+//using array_mapped_single_val_facet_t = std::array<single_val_facet_map_t*, ARRAY_FACET_DIM>;static constexpr size_t ARRAY_INFIX_DIM = 4;using array_mapped_infix_t = std::vector<tsl::htrie_set<char>*>;@@ -280,6 +281,8 @@ struct hnsw_index_t {}};+extern std::map<std::string, std::map<std::string, uint32_t>> facet_results;+class Index {private:mutable std::shared_mutex mutex;@@ -308,10 +311,11 @@ private:spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;// facet_field => (seq_id => values)- spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;+ //spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;+ facet_index_t* facet_index_v4 = nullptr;// facet_field => (seq_id => hash)- spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;+ //spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;// sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;@@ -363,7 +367,8 @@ private:bool estimate_facets, size_t facet_sample_percent,const std::vector<facet_info_t>& facet_infos,size_t group_limit, const std::vector<std::string>& group_by_fields,- const uint32_t* result_ids, size_t results_size) const;+ const uint32_t* result_ids, size_t results_size,+ int max_facet_count) const;bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,filter_node_t*& filter_tree_root) const;@@ -626,6 +631,7 @@ public:Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,const text_match_type_t match_type,filter_node_t*& filter_tree_root, std::vector<facet>& facets, facet_query_t& facet_query,+ const int max_facet_values,const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,const std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,diff --git a/src/collection.cpp b/src/collection.cppindex 976ee70e..064f0de0 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -22,6 +22,7 @@const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";+std::map<std::string, std::map<std::string, uint32_t>> facet_results;struct sort_fields_guard_t {std::vector<sort_by> sort_fields_std;@@ -1944,99 +1945,103 @@ Option<nlohmann::json> Collection::search(std::string raw_query,auto the_field = search_schema.at(a_facet.field_name);// keep only top K facets- auto max_facets = std::min(max_facet_values, facet_hash_counts.size());- std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,- facet_hash_counts.end(), Collection::facet_count_compare);- for(size_t fi = 0; fi < max_facets; fi++) {+ //auto max_facets = std::min(max_facet_values, facet_hash_counts.size());+ auto max_facets = std::min(max_facet_values, facet_results[a_facet.field_name].size());+ // std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,+ // facet_hash_counts.end(), Collection::facet_count_compare);+ LOG (INFO) << "found_doc_seq_ids size " << facet_results[a_facet.field_name].size();+ //for(size_t fi = 0; fi < max_facets; fi++) {+ for(auto& it : facet_results[a_facet.field_name]) {if(a_facet.is_range_query){break;}// remap facet value hash with actual string- auto & kv = facet_hash_counts[fi];- auto & facet_count = kv.second;- // fetch actual facet value from representative doc id- const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);- nlohmann::json document;- const Option<bool> & document_op = get_document_from_store(seq_id_key, document);- if(!document_op.ok()) {- LOG(ERROR) << "Facet fetch error. " << document_op.error();- continue;- }- std::string value;- bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);- if(!facet_found) {- continue;- }- std::unordered_map<std::string, size_t> ftoken_pos;- std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];- for(size_t ti = 0; ti < ftokens.size(); ti++) {- if(the_field.is_bool()) {- if(ftokens[ti] == "1") {- ftokens[ti] = "true";- } else {- ftokens[ti] = "false";- }- }- const std::string& resolved_token = ftokens[ti];- ftoken_pos[resolved_token] = ti;- }- const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();- // 2 passes: first identify tokens that need to be highlighted and then construct highlighted text- bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);- bool normalise = is_cyrillic ? false : true;- Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);- // secondary tokenizer used for specific languages that requires transliteration- // we use 2 tokenizers so that the original text offsets are available for highlighting- Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);- std::string raw_token;- size_t raw_token_index = 0, tok_start = 0, tok_end = 0;- // need an ordered map here to ensure that it is ordered by the key (start offset)- std::map<size_t, size_t> token_offsets;- size_t prefix_token_start_index = 0;- while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {- if(is_cyrillic) {- word_tokenizer.tokenize(raw_token);- }- auto token_pos_it = ftoken_pos.find(raw_token);- if(token_pos_it != ftoken_pos.end()) {- token_offsets[tok_start] = tok_end;- if(raw_token == last_full_q_token) {- prefix_token_start_index = tok_start;- }- }- }- auto offset_it = token_offsets.begin();- size_t i = 0;- std::stringstream highlightedss;- // loop until end index, accumulate token and complete highlighting- while(i < value.size()) {- if(offset_it != token_offsets.end()) {- if (i == offset_it->first) {- highlightedss << highlight_start_tag;- // do prefix highlighting for non-dropped last token- size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?- facet_query_last_token.size() :- (offset_it->second - i + 1);- if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {- token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());- } else {- token_len = (offset_it->second - i + 1);- }- for(size_t j = 0; j < token_len; j++) {- highlightedss << value[i + j];- }- highlightedss << highlight_end_tag;- offset_it++;- i += token_len;- continue;- }- }- highlightedss << value[i];- i++;- }- facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};+ // auto & kv = facet_hash_counts[fi];+ // auto & facet_count = kv.second;+ // // fetch actual facet value from representative doc id+ // const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);+ // nlohmann::json document;+ // const Option<bool> & document_op = get_document_from_store(seq_id_key, document);+ // if(!document_op.ok()) {+ // LOG(ERROR) << "Facet fetch error. " << document_op.error();+ // continue;+ // }+ //std::string value;+ // bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);+ // if(!facet_found) {+ // continue;+ // }+ // std::unordered_map<std::string, size_t> ftoken_pos;+ // std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];+ // for(size_t ti = 0; ti < ftokens.size(); ti++) {+ // if(the_field.is_bool()) {+ // if(ftokens[ti] == "1") {+ // ftokens[ti] = "true";+ // } else {+ // ftokens[ti] = "false";+ // }+ // }+ // const std::string& resolved_token = ftokens[ti];+ // ftoken_pos[resolved_token] = ti;+ // }+ // const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();+ // // 2 passes: first identify tokens that need to be highlighted and then construct highlighted text+ // bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);+ // bool normalise = is_cyrillic ? false : true;+ // Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);+ // // secondary tokenizer used for specific languages that requires transliteration+ // // we use 2 tokenizers so that the original text offsets are available for highlighting+ // Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);+ // std::string raw_token;+ // size_t raw_token_index = 0, tok_start = 0, tok_end = 0;+ // // need an ordered map here to ensure that it is ordered by the key (start offset)+ // std::map<size_t, size_t> token_offsets;+ // size_t prefix_token_start_index = 0;+ // while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {+ // if(is_cyrillic) {+ // word_tokenizer.tokenize(raw_token);+ // }+ // auto token_pos_it = ftoken_pos.find(raw_token);+ // if(token_pos_it != ftoken_pos.end()) {+ // token_offsets[tok_start] = tok_end;+ // if(raw_token == last_full_q_token) {+ // prefix_token_start_index = tok_start;+ // }+ // }+ // }+ // auto offset_it = token_offsets.begin();+ // size_t i = 0;+ // std::stringstream highlightedss;+ // // loop until end index, accumulate token and complete highlighting+ // while(i < value.size()) {+ // if(offset_it != token_offsets.end()) {+ // if (i == offset_it->first) {+ // highlightedss << highlight_start_tag;+ // // do prefix highlighting for non-dropped last token+ // size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?+ // facet_query_last_token.size() :+ // (offset_it->second - i + 1);+ // if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {+ // token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());+ // } else {+ // token_len = (offset_it->second - i + 1);+ // }+ // for(size_t j = 0; j < token_len; j++) {+ // highlightedss << value[i + j];+ // }+ // highlightedss << highlight_end_tag;+ // offset_it++;+ // i += token_len;+ // continue;+ // }+ // }+ // highlightedss << value[i];+ // i++;+ // }+ //facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};+ facet_value_t facet_value = { it.first, std::string(), it.second};facet_values.emplace_back(facet_value);}diff --git a/src/facet_index.cpp b/src/facet_index.cppnew file mode 100644index 00000000..9f8b0340--- /dev/null+++ b/src/facet_index.cpp@@ -0,0 +1,170 @@+#include "facet_index.h"++void facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {++ auto& facet_index_map = facet_field_map[field].facet_index_map;++ const auto sv = value.substr(0, 100);+ if(facet_index_map.count(sv) == 0) {+ facet_index_map.emplace(sv, SET_COMPACT_IDS(compact_id_list_t::create(1, {id})));+ }else {+ auto ids = facet_index_map[sv];+ if (!ids_t::contains(ids, id)) {+ ids_t::upsert(ids, id);+ facet_index_map[sv] = ids;+ }+ }++ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv));+ //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;+ auto& counter_list = facet_field_map[field].counter_list;++ if(counter_list.empty()) {+ counter_list.emplace_back(count_list(sv, facet_count));+ } else {+ auto it = counter_list.begin();+ //remove node from list+ for(it = counter_list.begin(); it != counter_list.end(); ++it) {+ if(it->facet_value == sv) {+ //found facet in first node+ counter_list.erase(it);+ break;+ }+ }++ //find position in list and add node with updated count+ count_list node(sv, facet_count);++ for(it = counter_list.begin(); it != counter_list.end(); ++it) {+ // LOG (INFO) << "inserting in middle or front facet " << node.facet_value+ // << " with count " << node.count;+ if(it->count >= facet_count) {+ counter_list.emplace(it, node);+ break;+ }+ }+ if(it == counter_list.end()) {+ // LOG (INFO) << "inserting at last facet " << node.facet_value+ // << " with count " << node.count;+ counter_list.emplace_back(node);+ }+ }+}++size_t facet_index_t::get(const std::string& field,+ std::map<std::string,std::vector<uint32_t>>& result_ids) {++ const auto& facet_field_it = facet_field_map.find(field);+ if(facet_field_it == facet_field_map.end()) {+ return 0;+ }+ auto& facet_index_map = facet_field_it->second.facet_index_map;++ for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {+ auto ids = ids_t::uncompress(it.value());+ for(auto i = 0; i < ids_t::num_ids(ids); ++i) {+ result_ids[it.key()].emplace_back(ids[i]);+ }+ }++ return result_ids.size();+}++bool facet_index_t::contains(const std::string& field) {++ const auto& facet_field_it = facet_field_map.find(field);+ if(facet_field_it == facet_field_map.end()) {+ return false;+ }++ // auto& facet_index_map = facet_field_it->second.facet_index_map;+ // LOG(INFO) << "Size of facet_field " << field << " " << facet_index_map.size();++ // for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {+ // LOG (INFO) << "facet_value " << it.key() << " with ids as follow";++ // auto ids = ids_t::uncompress(it.value());+ // for(auto i = 0; i < ids_t::num_ids(ids); ++i) {+ // LOG(INFO) << ids[i];+ // }+ // }+ return true;+}++void facet_index_t::erase(const std::string& field) {++ const auto& facet_field_it = facet_field_map.find(field);+ if(facet_field_it == facet_field_map.end()) {+ return;+ }++ auto& facet_index_map = facet_field_it->second.facet_index_map;++ for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {+ ids_t::destroy_list(it.value());+ }++ facet_index_map.clear();++ facet_field_it->second.counter_list.clear();++ facet_field_map.erase(field);+}++size_t facet_index_t::size() {+ return facet_field_map.size();+}++//returns the count of matching seq_ids from result array+int facet_index_t::intersect(const std::string& field, const uint32_t* result_ids,+ int result_ids_len, int max_facet_count,+ std::map<std::string, uint32_t>& found) {+ //LOG (INFO) << "intersecting field " << field;++ const auto& facet_field_it = facet_field_map.find(field);+ if(facet_field_it == facet_field_map.end()) {+ return 0;+ }++ const auto facet_index_map = facet_field_it->second.facet_index_map;+ const auto counter_list = facet_field_it->second.counter_list;++ // LOG (INFO) << "facet_index_map size " << facet_index_map.size()+ // << " , counter_list size " << counter_list.size();++ auto counter_list_it = counter_list.begin();+ int facet_count = 0;++ const auto max_facets = std::min((int)counter_list.size(), max_facet_count);+ while(facet_count < max_facets) {+ //LOG (INFO) << "checking ids in facet_value " << counter_list_it->facet_value+ // << " having total count " << counter_list_it->count;++ auto ids = facet_index_map.at(counter_list_it->facet_value);+ auto id_list = ids_t::uncompress(ids);+ int count = 0;++ for(int i = 0; i < result_ids_len; ++i) {+ if(std::binary_search(id_list, id_list + ids_t::num_ids(id_list), result_ids[i])) {+ ++count;+ }+ }+ if(count) {+ //LOG (INFO) << "fount count " << count << " for facet " << counter_list_it->facet_value;+ found[counter_list_it->facet_value] += count;+ }++ ++facet_count;+ ++counter_list_it;+ }++ return found.size();+}++facet_index_t::~facet_index_t() {+ for(auto it = facet_field_map.begin(); it != facet_field_map.end(); ++it) {+ erase(it->first);+ }+ facet_field_map.clear();+}+diff --git a/src/index.cpp b/src/index.cppindex b2a4ff39..b4f2001c 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -101,7 +101,10 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*}if(a_field.facet) {- initialize_facet_indexes(a_field);+ //initialize_facet_indexes(a_field);+ if(facet_index_v4 == nullptr) {+ facet_index_v4 = new facet_index_t();+ }}// initialize for non-string facet fields@@ -184,24 +187,25 @@ Index::~Index() {str_sort_index.clear();- for(auto& field_name_facet_map_array: facet_index_v3) {- for(auto& facet_map: field_name_facet_map_array.second) {- delete facet_map;- facet_map = nullptr;- }- }-- facet_index_v3.clear();+ //for(auto& field_name_facet_map_array: facet_index_v3) {+ // for(auto& facet_map: field_name_facet_map_array.second) {+ // delete facet_map;+ // facet_map = nullptr;+ // }+ //}//- for(auto& field_name_facet_map_array: single_val_facet_index_v3) {- for(auto& facet_map: field_name_facet_map_array.second) {- delete facet_map;- facet_map = nullptr;- }- }+ //facet_index_v3.clear();- single_val_facet_index_v3.clear();+ ////for(auto& field_name_facet_map_array: single_val_facet_index_v3) {+ // for(auto& facet_map: field_name_facet_map_array.second) {+ //// delete facet_map;+ // facet_map = nullptr;+ // }+ //}+ //single_val_facet_index_v3.clear();+ delete facet_index_v4;+delete seq_ids;for(auto& vec_index_kv: vector_index) {@@ -651,30 +655,95 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}if(afield.facet) {- if(afield.is_array()) {- facet_hash_values_t fhashvalues;- fhashvalues.length = field_index_it->second.facet_hashes.size();- fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];-- for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {- fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];+ //if(afield.is_array()) {+ // facet_hash_values_t fhashvalues;+ // fhashvalues.length = field_index_it->second.facet_hashes.size();+ // fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];++ // for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {+ // fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];+ // }++ // auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];+ // if(facet_dim_index == nullptr) {+ // LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;+ // } else {+ // facet_dim_index->emplace(seq_id, std::move(fhashvalues));+ // }+ //} else {+ // uint64_t fhash;+ // fhash = field_index_it->second.facet_hashes[0];++ // auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];+ // if(facet_dim_index == nullptr) {+ // LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;+ // } else {+ // facet_dim_index->emplace(seq_id, std::move(fhash));+ // }+ //}+ std::string value;+ if(afield.type == field_types::INT32) {+ int32_t raw_val = document[afield.name].get<int32_t>();+ value = std::to_string(raw_val);+ facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::INT32_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ int32_t raw_val = document[afield.name][i].get<int32_t>();+ value = std::to_string(raw_val);+ facet_index_v4->insert(afield.name, value, seq_id);}-- auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- } else {- facet_dim_index->emplace(seq_id, std::move(fhashvalues));+ }+ else if(afield.type == field_types::INT64) {+ int64_t raw_val = document[afield.name].get<int64_t>();+ value = std::to_string(raw_val);+ facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::INT64_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ int64_t raw_val = document[afield.name][i].get<int64_t>();+ value = std::to_string(raw_val);+ facet_index_v4->insert(afield.name, value, seq_id);}- } else {- uint64_t fhash;- fhash = field_index_it->second.facet_hashes[0];-- auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- } else {- facet_dim_index->emplace(seq_id, std::move(fhash));+ }+ else if(afield.type == field_types::STRING) {+ value = document[afield.name];+ facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::STRING_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ value = document[afield.name][i];+ facet_index_v4->insert(afield.name, value, seq_id);+ }+ }+ else if(afield.type == field_types::FLOAT) {+ float raw_val = document[afield.name].get<float>();+ value = StringUtils::float_to_str(raw_val);+ if(value != "0") {+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros+ }+ facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::FLOAT) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ float raw_val = document[afield.name][i].get<float>();+ value = StringUtils::float_to_str(raw_val);+ if(value != "0") {+ value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros+ }+ facet_index_v4->insert(afield.name, value, seq_id);+ }+ }+ else if(afield.type == field_types::BOOL) {+ value = std::to_string(document[afield.name].get<bool>());+ value = (value == "1") ? "true" : "false";+ facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::BOOL) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ value = std::to_string(document[afield.name][i].get<bool>());+ value = (value == "1") ? "true" : "false";+ facet_index_v4->insert(afield.name, value, seq_id);}}}@@ -1089,19 +1158,19 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st}}void Index::initialize_facet_indexes(const field& facet_field) {- if(facet_field.is_array()) {- array_mapped_facet_t facet_array;- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- facet_array[i] = new facet_map_t();- }- facet_index_v3.emplace(facet_field.name, facet_array);- } else {- array_mapped_single_val_facet_t facet_array;- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- facet_array[i] = new single_val_facet_map_t();- }- single_val_facet_index_v3.emplace(facet_field.name, facet_array);- }+ // if(facet_field.is_array()) {+ // array_mapped_facet_t facet_array;+ // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ // facet_array[i] = new facet_map_t();+ // }+ // facet_index_v3.emplace(facet_field.name, facet_array);+ // } else {+ // array_mapped_single_val_facet_t facet_array;+ // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ // facet_array[i] = new single_val_facet_map_t();+ // }+ // single_val_facet_index_v3.emplace(facet_field.name, facet_array);+ // }}void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type) {@@ -1142,106 +1211,101 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,bool estimate_facets, size_t facet_sample_percent,const std::vector<facet_info_t>& facet_infos,const size_t group_limit, const std::vector<std::string>& group_by_fields,- const uint32_t* result_ids, size_t results_size) const {-+ const uint32_t* result_ids, size_t results_size,+ int max_facet_count) const {// assumed that facet fields have already been validated upstreamfor(size_t findex=0; findex < facets.size(); findex++) {auto& a_facet = facets[findex];const auto& facet_field = facet_infos[findex].facet_field;const bool use_facet_query = facet_infos[findex].use_facet_query;- const auto& fquery_hashes = facet_infos[findex].hashes;+ //const auto& fquery_hashes = facet_infos[findex].hashes;+ const auto& fquery_doc_id_tokens = facet_infos[findex].doc_id_tokens;const bool should_compute_stats = facet_infos[findex].should_compute_stats;auto sort_index_it = sort_index.find(a_facet.field_name);size_t mod_value = 100 / facet_sample_percent;- facet_map_t::iterator facet_map_it;- single_val_facet_map_t::iterator single_facet_map_it;- uint64_t fhash = 0;- size_t facet_hash_count = 1;- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);-- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {- continue;- }+ // facet_map_t::iterator facet_map_it;+ // single_val_facet_map_t::iterator single_facet_map_it;+ // uint64_t fhash = 0;+ // size_t facet_hash_count = 1;+ // const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);+ // const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);+ facet_index_v4->intersect(a_facet.field_name, result_ids,+ results_size, max_facet_count, facet_results[a_facet.field_name]);+- for(size_t i = 0; i < results_size; i++) {- // if sampling is enabled, we will skip a portion of the results to speed up things- if(estimate_facets) {- if(i % mod_value != 0) {- continue;- }- }+ // for(size_t i = 0; i < results_size; i++) {+ // // if sampling is enabled, we will skip a portion of the results to speed up things+ // if(estimate_facets) {+ // if(i % mod_value != 0) {+ // continue;+ // }+ // }- uint32_t doc_seq_id = result_ids[i];- if(facet_field.is_array()) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);+ // uint32_t doc_seq_id = result_ids[i];+ // if(facet_field.is_array()) {+ // const auto& field_facet_mapping = field_facet_mapping_it->second;+ // facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }- facet_hash_count = facet_map_it->second.size();- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);+ // if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ // continue;+ // }+ // facet_hash_count = facet_map_it->second.size();+ // } else {+ // const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;+ // single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }- facet_hash_count = 1;- fhash = single_facet_map_it->second;- }-- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;-- if(((i + 1) % 16384) == 0) {- RETURN_CIRCUIT_BREAKER- }-- for(size_t j = 0; j < facet_hash_count; j++) {+ // if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ // continue;+ // }+ // facet_hash_count = 1;+ // fhash = single_facet_map_it->second;+ // }++ // const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;+ // //for(size_t j = 0; j < facet_hash_count; j++) {+ // for(size_t j = 0; j < found_doc_seq_ids.size(); j++) {+ // // if(facet_field.is_array()) {+ // // fhash = facet_map_it->second.hashes[j];+ // // }+ // const auto& doc_id = found_doc_seq_ids[j].second;+ // if(should_compute_stats) {+ // compute_facet_stats(a_facet, doc_id, facet_field.type);+ // }+ // if(a_facet.is_range_query) {+ // auto sort_index_it = sort_index.find(a_facet.field_name);+ // if(sort_index_it != sort_index.end()){+ // auto doc_id_val_map = sort_index_it->second;+ // auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);+ // if(doc_seq_id_it != doc_id_val_map->end()){- if(facet_field.is_array()) {- fhash = facet_map_it->second.hashes[j];- }-- if(should_compute_stats) {- compute_facet_stats(a_facet, fhash, facet_field.type);- }- if(a_facet.is_range_query) {- if(sort_index_it != sort_index.end()){- auto doc_id_val_map = sort_index_it->second;- auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);-- if(doc_seq_id_it != doc_id_val_map->end()){- int64_t doc_val = doc_seq_id_it->second;- std::pair<int64_t, std::string> range_pair {};- if(a_facet.get_range(doc_val, range_pair)) {- int64_t range_id = range_pair.first;- facet_count_t& facet_count = a_facet.result_map[range_id];- facet_count.count += 1;- }- }- }- } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {- facet_count_t& facet_count = a_facet.result_map[fhash];- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;- facet_count.doc_id = doc_seq_id;- facet_count.array_pos = j;- if(group_limit) {- a_facet.hash_groups[fhash].emplace(distinct_id);- } else {- facet_count.count += 1;- }- if(use_facet_query) {- a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);- }- }- }- }+ // int64_t doc_val = doc_seq_id_it->second;+ // std::pair<int64_t, std::string> range_pair {};+ // if(a_facet.get_range(doc_val, range_pair)) {+ // int64_t range_id = range_pair.first;+ // facet_count_t& facet_count = a_facet.result_map[range_id];+ // facet_count.count += 1;+ // }+ // }+ // }+ // } else if(!use_facet_query || fquery_doc_id_tokens.find(doc_id) != fquery_doc_id_tokens.end()) {+ // facet_count_t& facet_count = a_facet.result_map[doc_id];+ // //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;+ // facet_count.doc_id = doc_seq_id;+ // facet_count.array_pos = j;+ // if(group_limit) {+ // a_facet.hash_groups[doc_id].emplace(distinct_id);+ // } else {+ // facet_count.count += 1;+ // }+ // if(use_facet_query) {+ // a_facet.hash_tokens[doc_id] = fquery_doc_id_tokens.at(doc_id);+ // }+ // }+ // }+ //}}}@@ -1759,6 +1823,7 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->search_fields,search_params->match_type,search_params->filter_tree_root, search_params->facets, search_params->facet_query,+ search_params->max_facet_values,search_params->included_ids, search_params->excluded_ids,search_params->sort_fields_std, search_params->num_typos,search_params->topster, search_params->curated_topster,@@ -2219,6 +2284,7 @@ void Index::search_infix(const std::string& query, const std::string& field_nameOption<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,const text_match_type_t match_type,filter_node_t*& filter_tree_root, std::vector<facet>& facets, facet_query_t& facet_query,+ const int max_facet_values,const std::vector<std::pair<uint32_t, uint32_t>>& included_ids,const std::vector<uint32_t>& excluded_ids, std::vector<sort_by>& sort_fields_std,const std::vector<uint32_t>& num_typos, Topster* topster, Topster* curated_topster,@@ -2798,7 +2864,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consnum_queued++;thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,- batch_result_ids, batch_res_len, &facet_infos,+ batch_result_ids, batch_res_len, &facet_infos, max_facet_values,estimate_facets, facet_sample_percent,&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff,&num_processed, &m_process, &cv_process]() {@@ -2807,9 +2873,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conssearch_cutoff = parent_search_cutoff;auto fq = facet_query;+ std::vector<std::pair<std::string, uint32_t>> found_docs;do_facets(facet_batches[thread_id], fq, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields,- batch_result_ids, batch_res_len);+ batch_result_ids, batch_res_len, max_facet_values);std::unique_lock<std::mutex> lock(m_process);num_processed++;parent_search_cutoff = parent_search_cutoff || search_cutoff;@@ -2827,7 +2894,6 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor(size_t fi = 0; fi < facet_batch.size(); fi++) {auto& this_facet = facet_batch[fi];auto& acc_facet = facets[fi];-for(auto & facet_kv: this_facet.result_map) {if(group_limit) {// we have to add all group sets@@ -2880,13 +2946,17 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::chrono::high_resolution_clock::now() - beginF).count();LOG(INFO) << "Time for faceting: " << timeMillisF;*/}-+ std::vector<std::pair<std::string, uint32_t>> found_docs;std::vector<facet_info_t> facet_infos(facets.size());compute_facet_infos(facets, facet_query, facet_query_num_typos,- &included_ids_vec[0], included_ids_vec.size(), group_by_fields, max_candidates, facet_infos);+ &included_ids_vec[0], included_ids_vec.size(), group_by_fields,+ max_candidates, facet_infos);do_facets(facets, facet_query, estimate_facets, facet_sample_percent,- facet_infos, group_limit, group_by_fields, &included_ids_vec[0], included_ids_vec.size());+ facet_infos, group_limit, group_by_fields, &included_ids_vec[0],+ included_ids_vec.size(), max_facet_values);+ facet_index_v4->contains("tags");+all_result_ids_len += curated_topster->size;delete [] all_result_ids;@@ -4297,11 +4367,15 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(size_t findex=0; findex < facets.size(); findex++) {const auto& a_facet = facets[findex];- const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);+ // const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);+ // const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ // if((field_facet_mapping_it == facet_index_v3.end())+ // && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ // continue;+ // }+ std::map<std::string, std::vector<uint32_t>> found_doc_ids;+ if(facet_index_v4->get(a_facet.field_name, found_doc_ids) == 0) {continue;}@@ -4395,37 +4469,47 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&continue;}- if(facet_field.is_array()) {- const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }-- std::vector<size_t> array_indices;- posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);-- for(size_t array_index: array_indices) {- if(array_index < doc_fvalues_it->second.length) {- uint64_t hash = doc_fvalues_it->second.hashes[array_index];-- /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "- << array_index;*/-- if(facet_infos[findex].hashes.count(hash) == 0) {- facet_infos[findex].hashes.emplace(hash, searched_tokens);+ // if(facet_field.is_array()) {+ // const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ // if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ // continue;+ // }++ // std::vector<size_t> array_indices;+ // posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);++ // for(size_t array_index: array_indices) {+ // if(array_index < doc_fvalues_it->second.length) {+ // uint64_t hash = doc_fvalues_it->second.hashes[array_index];++ // /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "+ // << array_index;*/++ // if(facet_infos[findex].hashes.count(hash) == 0) {+ // facet_infos[findex].hashes.emplace(hash, searched_tokens);+ // }+ // }+ // }+ // } else {+ // const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ // if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ // continue;+ // }++ // uint64_t hash = doc_single_fvalues_it->second;+ // if(facet_infos[findex].hashes.count(hash) == 0) {+ // facet_infos[findex].hashes.emplace(hash, searched_tokens);+ // }+ // }+ for(const auto& found_doc_it : found_doc_ids) {+ const auto& ids = found_doc_it.second;+ if(std::binary_search(ids.begin(), ids.end(), seq_id)){+ for(const auto& doc_id : ids) {+ if(facet_infos[findex].doc_id_tokens.count(doc_id) == 0) {+ facet_infos[findex].doc_id_tokens.emplace(doc_id, searched_tokens);}}}- } else {- const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }-- uint64_t hash = doc_single_fvalues_it->second;- if(facet_infos[findex].hashes.count(hash) == 0) {- facet_infos[findex].hashes.emplace(hash, searched_tokens);- }}}}@@ -5155,41 +5239,47 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,const uint32_t seq_id) const {uint64_t distinct_id = 1; // some constant initial value-+ std::hash<std::string> hasher;// calculate hash from group_by_fieldsfor(const auto& field: group_by_fields) {- const auto& field_facet_mapping_it = facet_index_v3.find(field);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ // const auto& field_facet_mapping_it = facet_index_v3.find(field);+ // const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);+ // if((field_facet_mapping_it == facet_index_v3.end())+ // && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ // continue;+ // }+ if(!facet_index_v4->contains(field)) {continue;}- if(search_schema.at(field).is_array()) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ // if(search_schema.at(field).is_array()) {+ // const auto& field_facet_mapping = field_facet_mapping_it->second;+ // const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }+ // if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {+ // continue;+ // }- const auto& facet_hashes = facet_hashes_it->second;+ // const auto& facet_hashes = facet_hashes_it->second;- for(size_t i = 0; i < facet_hashes.size(); i++) {- distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);- }- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ // for(size_t i = 0; i < facet_hashes.size(); i++) {+ // distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);+ // }+ // } else {+ // const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;+ // const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }+ // if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {+ // continue;+ // }- const auto& facet_hash = facet_hashes_it->second;+ // const auto& facet_hash = facet_hashes_it->second;- distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);- }+ // distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);+ // }++ const auto& hash = hasher(field);+ distinct_id = StringUtils::hash_combine(distinct_id, hash);}return distinct_id;@@ -5425,21 +5515,23 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const}// remove facets- const auto& field_facets_it = facet_index_v3.find(field_name);- if(field_facets_it != facet_index_v3.end()) {- const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);- }- }-- const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);- if(field_single_val_facets_it != single_val_facet_index_v3.end()) {- const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);- }- }+ // const auto& field_facets_it = facet_index_v3.find(field_name);+ // if(field_facets_it != facet_index_v3.end()) {+ // const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ // if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ // field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);+ // }+ // }++ // const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);+ // if(field_single_val_facets_it != single_val_facet_index_v3.end()) {+ // const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ // if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ // field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);+ // }+ // }++ facet_index_v4->erase(field_name);// remove sort fieldif(sort_index.count(field_name) != 0) {@@ -5577,7 +5669,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecif(new_field.is_facet()) {- initialize_facet_indexes(new_field);+ //initialize_facet_indexes(new_field);// initialize for non-string facet fieldsif(!new_field.is_string()) {@@ -5641,21 +5733,22 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec}if(del_field.is_facet()) {- if(del_field.is_array()) {- auto& arr = facet_index_v3[del_field.name];- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- delete arr[i];- }+ // if(del_field.is_array()) {+ // auto& arr = facet_index_v3[del_field.name];+ // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ // delete arr[i];+ // }- facet_index_v3.erase(del_field.name);- } else {- auto& arr = single_val_facet_index_v3[del_field.name];- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- delete arr[i];- }-- single_val_facet_index_v3.erase(del_field.name);- }+ // facet_index_v3.erase(del_field.name);+ // } else {+ // auto& arr = single_val_facet_index_v3[del_field.name];+ // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ // delete arr[i];+ // }++ // single_val_facet_index_v3.erase(del_field.name);+ // }+ facet_index_v4->erase(del_field.name);if(!del_field.is_string()) {art_tree_destroy(search_index[del_field.faceted_name()]);diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex ee9b798f..261e702c 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -1603,3 +1603,59 @@ TEST_F(CollectionFacetingTest, FacetOnArrayFieldWithSpecialChars) {}}}++TEST_F(CollectionFacetingTest, FacetIndexRefactor) {+ Collection *coll_array_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("name_facet", field_types::STRING, true),+ field("age", field_types::INT32, true),+ field("years", field_types::INT32_ARRAY, true),+ field("rating", field_types::FLOAT, true),+ field("timestamps", field_types::INT64_ARRAY, true),+ field("tags", field_types::STRING_ARRAY, true)};++ std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };++ coll_array_fields = collectionManager.get_collection("coll_array_fields").get();+ if(coll_array_fields == nullptr) {+ coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ nlohmann::json document = nlohmann::json::parse(json_line);+ document["name_facet"] = document["name"];+ const std::string & patched_json_line = document.dump();+ coll_array_fields->add(patched_json_line);+ }++ infile.close();++ query_fields = {"name"};+ std::vector<std::string> facets = {"tags"};++ // single facet with no filters+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();++ LOG(INFO) << results.dump();++ ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(4, results["facet_counts"][0].size());+ ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);++ ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);++ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][1]["count"]);++ ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][2]["count"]);++ ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][3]["count"]);+}\ No newline at end of file