Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
refactoring changes
krunal1313b0a369f8ab31
diff --git a/include/facet_index.h b/include/facet_index.hindex db1e2162..c97d6097 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -11,19 +11,22 @@ private:~count_list () = default;- count_list(const std::string& sv, uint32_t facet_count) {+ count_list(const std::string& sv, uint32_t facet_count, uint32_t facet_index) {facet_value = sv;count = facet_count;+ index = facet_index;}count_list& operator=(count_list& obj) {facet_value = obj.facet_value;count = obj.count;+ index = obj.index;return *this;}std::string facet_value;uint32_t count;+ uint32_t index;};struct facet_index_struct {@@ -61,8 +64,6 @@ private:std::map<std::string, facet_index_counter> facet_field_map;uint32_t count_index = 0;- std::unordered_set<std::string> dropped_fields;-public:facet_index_t() = default;@@ -73,6 +74,8 @@ public:void erase(const std::string& field);+ void remove(const std::string& field, const uint32_t seq_id);+bool contains(const std::string& field);size_t get_facet_count(const std::string& field);@@ -81,11 +84,8 @@ public:int max_facet_count, std::map<std::string, uint32_t>& found,bool is_wildcard_no_filter_query);- void get_facet_indexes(const std::string& field, std::function<void(uint32_t, uint32_t)> functor);-- bool get_migrated (const std::string& field) const;-- void set_migrated(const std::string& field, bool val);-- void set_dropped(const std::string& field);+ size_t get_facet_indexes(const std::string& field,+ std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes);++ void initialize(const std::string& field);};\ No newline at end of filediff --git a/include/index.h b/include/index.hindex 1ae09e59..78f04f76 100644--- a/include/index.h+++ b/include/index.h@@ -304,13 +304,10 @@ private:// geo_array_field => (seq_id => values) used for exact filtering of geo array recordsspp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;- // facet_field => (seq_id => values)- spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;-- facet_index_t* facet_index_v4 = nullptr;+ spp::sparse_hash_map<std::string, posting_list_t*> facet_hash_index;- // facet_field => (seq_id => hash)- spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;+ facet_index_t* facet_index_v4 = nullptr;+// sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;diff --git a/include/num_tree.h b/include/num_tree.hindex ce0a226c..c31f0b99 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -43,7 +43,6 @@ private:};std::vector<count_list> counter_list;- bool is_migrated = false;public:@@ -85,13 +84,5 @@ public:size_t counter_list_size() const;- void get_facet_indexes(std::function<void(uint32_t, uint32_t)> functor);-- bool get_migrated () const {- return is_migrated;- }-- void set_migrated(bool val) {- is_migrated = true;- }+ size_t get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes);};\ No newline at end of filediff --git a/include/posting_list.h b/include/posting_list.hindex a0df81fa..d71a8fbd 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -188,6 +188,8 @@ public:);static bool take_id(result_iter_state_t& istate, uint32_t id);++ static void get_offsets(iterator_t& iter, std::vector<uint32_t>& positions);static bool get_offsets(const std::vector<iterator_t>& its,diff --git a/src/facet_index.cpp b/src/facet_index.cppindex e007710b..88e8ec69 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -3,22 +3,24 @@#include "string_utils.h"#include "array_utils.h"+void facet_index_t::initialize(const std::string& field) {+ const auto facet_field_map_it = facet_field_map.find(field);+ if(facet_field_map_it == facet_field_map.end()) {+ facet_field_map.emplace(field, facet_index_counter{});+ }+}+uint32_t facet_index_t::insert(const std::string& field, const std::string& value,uint32_t id, bool is_string) {- if(dropped_fields.find(field) != dropped_fields.end()) {- //field has been dropped- return 0;- }-const auto facet_field_map_it = facet_field_map.find(field);if(facet_field_map_it == facet_field_map.end()) {- facet_field_map.emplace(field, facet_index_counter{});+ return 0; //field is not initialized or dropped}uint32_t index = 0;- auto& facet_index_map = facet_field_map[field].facet_index_map;+ auto& facet_index_map = facet_field_map_it->second.facet_index_map;const auto sv = value.substr(0, 100);const auto it = facet_index_map.find(sv);@@ -44,22 +46,26 @@ uint32_t facet_index_t::insert(const std::string& field, const std::string& valuif(is_string) {const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);//LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;- auto& counter_list = facet_field_map[field].counter_list;+ auto& counter_list = facet_field_map_it->second.counter_list;if(counter_list.empty()) {- counter_list.emplace_back(sv, facet_count);+ counter_list.emplace_back(sv, facet_count, index);} else {- count_list node(sv, facet_count);- uint32_t ind = 0;- for(ind; ind < counter_list.size(); ++ind) {- if(counter_list[ind].facet_value == sv) {+ count_list node(sv, facet_count, index);+ auto ind = 0;++ for(; ind < counter_list.size(); ++ind) {+ if(counter_list[ind].index == index) {counter_list[ind].count = facet_count;- if((ind > 1) &&- (counter_list[ind-1].count < counter_list[ind].count)) {- count_list temp = counter_list[ind-1];- counter_list[ind-1] = counter_list[ind];- counter_list[ind] = temp;+ if(ind > 1) {+ auto curr = ind;+ while (curr && (counter_list[curr-1].count < counter_list[curr].count)) {+ count_list temp = counter_list[curr-1];+ counter_list[curr-1] = counter_list[curr];+ counter_list[curr] = temp;+ --curr;+ }}break;}@@ -92,6 +98,23 @@ void facet_index_t::erase(const std::string& field) {}}+void facet_index_t::remove(const std::string& field, const uint32_t seq_id) {+ const auto facet_field_it = facet_field_map.find(field);+ if(facet_field_it != facet_field_map.end()) {+ auto facet_index_map = facet_field_it->second.facet_index_map;+ auto facet_index_map_it = facet_index_map.begin();++ for(; facet_index_map_it != facet_index_map.end(); ++facet_index_map_it) {+ void* ids = facet_index_map_it.value().id_list_ptr;+ if(ids && ids_t::contains(ids, seq_id)) {+ // ids_t::erase(ids, seq_id);+ // facet_index_map_it.value().id_list_ptr = ids;+ break;+ }+ }+ }+}+size_t facet_index_t::get_facet_count(const std::string& field) {const auto it = facet_field_map.find(field);@@ -157,12 +180,12 @@ facet_index_t::~facet_index_t() {}//used for migrating string and int64 facets-void facet_index_t::get_facet_indexes(const std::string& field,- std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {+size_t facet_index_t::get_facet_indexes(const std::string& field,+ std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes) {const auto& facet_field_it = facet_field_map.find(field);if(facet_field_it == facet_field_map.end()) {- return;+ return 0;}auto& facet_index_map = facet_field_it->second.facet_index_map;@@ -177,37 +200,11 @@ void facet_index_t::get_facet_indexes(const std::string& field,//emplacing seq_id=>count_indexfor(const auto& id : id_list) {- functor(id, facet_index_map_it->index);+ seqid_countIndexes[id].emplace_back(facet_index_map_it->index);}id_list.clear();}- return;-}---bool facet_index_t::get_migrated (const std::string& field) const {- const auto it = facet_field_map.find(field);- if(it != facet_field_map.end()) {- return it->second.is_migrated;- }-- return false;-}--void facet_index_t::set_migrated(const std::string& field, bool val) {- const auto it = facet_field_map.find(field);- if(it != facet_field_map.end()) {- it->second.is_migrated = val;- }-- return;+ return seqid_countIndexes.size();}--void facet_index_t::set_dropped(const std::string& field) {-- dropped_fields.emplace(field);-- return;-}\ No newline at end of filediff --git a/src/index.cpp b/src/index.cppindex b366c0de..460aef95 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -187,23 +187,11 @@ Index::~Index() {str_sort_index.clear();- for(auto& field_name_facet_map_array: facet_index_v3) {- for(auto& facet_map: field_name_facet_map_array.second) {- delete facet_map;- facet_map = nullptr;- }+ for(const auto& facet_hash_index_it : facet_hash_index) {+ delete facet_hash_index_it.second;}+ facet_hash_index.clear();- facet_index_v3.clear();-- for(auto& field_name_facet_map_array: single_val_facet_index_v3) {- for(auto& facet_map: field_name_facet_map_array.second) {- delete facet_map;- facet_map = nullptr;- }- }-- single_val_facet_index_v3.clear();delete facet_index_v4;delete seq_ids;@@ -607,59 +595,26 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bvoid Index::create_facet_hash_index(const field& facet_field) {//LOG(INFO) << "migrating facet " << field << " to old index";- if(facet_field.is_array()) {- auto& facet_index = facet_index_v3[facet_field.name];-- facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {+ std::map<uint32_t, std::vector<uint32_t>> seqid_index_map;+ auto facet_index = facet_hash_index[facet_field.name];- auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;- } else {- facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);- }- });-- auto numerical_index_it = numerical_index.find(facet_field.name);- if(numerical_index_it != numerical_index_it) {- auto num_tree = numerical_index_it->second;- num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {-- auto& facet_dim_index = facet_index[seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;- } else {- facet_dim_index->operator[](seq_id).hashes.emplace_back(count_index);- }- });+ if(facet_index_v4->get_facet_indexes(facet_field.name, seqid_index_map)) {+ for(const auto& kv : seqid_index_map) {+ facet_index->upsert(kv.first, std::move(kv.second));}- } else {- auto& single_facet_index = single_val_facet_index_v3[facet_field.name];+ }- facet_index_v4->get_facet_indexes(facet_field.name, [&] (uint32_t seq_id, uint32_t count_index) {+ seqid_index_map.clear();- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;- } else {- facet_dim_index->operator[](seq_id) = count_index;+ auto numerical_index_it = numerical_index.find(facet_field.name);+ if(numerical_index_it != numerical_index_it) {+ auto num_tree = numerical_index_it->second;+ if(num_tree->get_facet_indexes(seqid_index_map)) {+ for(const auto& kv : seqid_index_map) {+ facet_index->upsert(kv.first, std::move(kv.second));}- });-- auto numerical_index_it = numerical_index.find(facet_field.name);- if(numerical_index_it != numerical_index_it) {- auto num_tree = numerical_index_it->second;- num_tree->get_facet_indexes([&] (uint32_t seq_id, uint32_t count_index) {-- auto& facet_dim_index = single_facet_index[seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << facet_field.name;- } else {- facet_dim_index->operator[](seq_id) = count_index;- }- });- }- }+ }+ }}void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch) {@@ -697,43 +652,38 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>auto facet_threshold_count = 0;- bool is_migrated = false;if(afield.is_string()) {facet_threshold_count = facet_index_v4->get_facet_count(afield.name);- is_migrated = facet_index_v4->get_migrated(afield.name);} else {auto numerical_index_it = numerical_index.find(afield.name);if(numerical_index_it != numerical_index.end()) {facet_threshold_count = numerical_index_it->second->counter_list_size();- is_migrated = numerical_index_it->second->get_migrated();}}#ifdef TEST_BUILDfacet_threshold_count = FACET_INDEX_THRESHOLD + 1;- is_migrated = true;#endifconst auto total_num_docs = aggragate_docs_count.load();- if(!is_migrated && (facet_threshold_count > FACET_INDEX_THRESHOLD)+ bool facet_hash_index_exist = facet_hash_index.find(afield.name) != facet_hash_index.end();++ if(!facet_hash_index_exist && (facet_threshold_count > FACET_INDEX_THRESHOLD)&& total_num_docs < 1000000) {create_facet_hash_index(afield);- if(afield.is_string()) {- facet_index_v4->set_migrated(afield.name, true);- } else {- auto numerical_index_it = numerical_index.find(afield.name);- if(numerical_index_it != numerical_index.end()) {- numerical_index_it->second->set_migrated(true);- }- }-auto cardinality_ratio = total_num_docs / facet_threshold_count;if(cardinality_ratio < 5) {+ //drop the fieldfacet_index_v4->erase(afield.name);- facet_index_v4->set_dropped(afield.name);}}+ posting_list_t* facet_index = nullptr;+ auto facet_index_it = facet_hash_index.find(afield.name);+ if(facet_index_it != facet_hash_index.end()) {+ facet_index = facet_index_it->second;+ }+for(const auto& record: iter_batch) {if(!record.indexed.ok()) {// some records could have been invalidated upstream@@ -757,7 +707,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>if(afield.facet) {if(afield.is_array()) {- facet_hash_values_t fhashvalues;+ std::vector<uint32_t> fhashvalues;for(size_t i = 0; i < document[afield.name].size(); ++i) {if(afield.type == field_types::INT32_ARRAY) {int32_t raw_val = document[afield.name][i].get<int32_t>();@@ -778,19 +728,13 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}if(facet_threshold_count > FACET_INDEX_THRESHOLD) {- fhashvalues.hashes.emplace_back(fhash);+ fhashvalues.emplace_back(fhash);}}if(facet_threshold_count > FACET_INDEX_THRESHOLD) {- fhashvalues.length = fhashvalues.hashes.size();-- auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- } else {- facet_dim_index->emplace(seq_id, std::move(fhashvalues));- }+ facet_index->upsert(seq_id, std::move(fhashvalues));+ fhashvalues.clear();}} else {uint32_t fhash;@@ -818,13 +762,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}if(facet_threshold_count > FACET_INDEX_THRESHOLD) {-- auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- } else {- facet_dim_index->emplace(seq_id, std::move(fhash));- }+ facet_index->upsert(seq_id, {fhash});}}}@@ -1195,19 +1133,13 @@ void Index::tokenize_string_array(const std::vector<std::string>& strings, bool}}void Index::initialize_facet_indexes(const field& facet_field) {- if(facet_field.is_array()) {- array_mapped_facet_t facet_array;- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- facet_array[i] = new facet_map_t();- }- facet_index_v3.emplace(facet_field.name, facet_array);- } else {- array_mapped_single_val_facet_t facet_array;- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- facet_array[i] = new single_val_facet_map_t();- }- single_val_facet_index_v3.emplace(facet_field.name, facet_array);++ if(facet_field.is_string() || facet_field.is_int64()) {+ facet_index_v4->initialize(facet_field.name);}++ facet_hash_index.emplace(facet_field.name, new posting_list_t(256));+}void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {@@ -1337,12 +1269,10 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,use_facet_intersection = false;use_hashes = false;#endif- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);-+ const auto& facet_hash_index_it = facet_hash_index.find(a_facet.field_name);+bool is_wildcard_no_filter_query = is_wildcard_query && no_filters_provided;- bool facet_hash_index_exists = ((field_facet_mapping_it != facet_index_v3.end())- || (field_single_val_facet_mapping_it != single_val_facet_index_v3.end()));+ bool facet_hash_index_exists = facet_hash_index_it != facet_hash_index.end();if(results_size && num_facet_values && ((!facet_hash_index_exists || is_wildcard_no_filter_query|| num_facet_values > 50000) && group_limit == 0) && !use_hashes || use_facet_intersection) {@@ -1419,17 +1349,18 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,} else {facet_map_t::iterator facet_map_it;single_val_facet_map_t::iterator single_facet_map_it;- uint32_t fhash = 0;- size_t facet_hash_count = 1;- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ if(!facet_hash_index_exists) {continue;}auto sort_index_it = sort_index.find(a_facet.field_name);const auto facet_field_is_array = facet_field.is_array();+ const auto& facet_index = facet_hash_index_it->second;+ std::vector<uint32_t> facet_hashes;+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();+for(size_t i = 0; i < results_size; i++) {// if sampling is enabled, we will skip a portion of the results to speed up thingsif(estimate_facets) {@@ -1439,64 +1370,58 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}uint32_t doc_seq_id = result_ids[i];- if(facet_field_is_array) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }- facet_hash_count = facet_map_it->second.size();- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }- facet_hash_count = 1;- fhash = single_facet_map_it->second;+ if(!facet_index->contains(doc_seq_id)) {+ continue;}- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;- //LOG(INFO) << "facet_hash_count " << facet_hash_count;- if(((i + 1) % 16384) == 0) {- RETURN_CIRCUIT_BREAKER- }+ facet_index_it.skip_to(doc_seq_id);- for(size_t j = 0; j < facet_hash_count; j++) {- if(facet_field_is_array) {- fhash = facet_map_it->second.hashes[j];- }- if(should_compute_stats) {- compute_facet_stats(a_facet, fhash, facet_field.type);- }+ if(facet_index_it.valid()) {+ facet_hashes.clear();+ posting_list_t::get_offsets(facet_index_it, facet_hashes);- std::string fhash_str = std::to_string(fhash);- if(a_facet.is_range_query) {- int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);-- std::pair<std::string, std::string> range_pair {};- if(a_facet.get_range(std::to_string(doc_val), range_pair)) {- const auto& range_id = range_pair.first;- facet_count_t& facet_count = a_facet.result_map[range_id];- facet_count.count += 1;- }- } else if(!use_facet_query || fquery_hashes.find(fhash_str) != fquery_hashes.end()) {- facet_count_t& facet_count = a_facet.result_map[fhash_str];- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;- facet_count.doc_id = doc_seq_id;- facet_count.array_pos = j;- if(group_limit) {- a_facet.hash_groups[fhash].emplace(distinct_id);- } else {- facet_count.count += 1;+ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;+ if(((i + 1) % 16384) == 0) {+ RETURN_CIRCUIT_BREAKER+ }++ for(size_t j = 0; j < facet_hashes.size(); j++) {++ const auto& fhash = facet_hashes[j];++ if(should_compute_stats) {+ compute_facet_stats(a_facet, fhash, facet_field.type);}- if(use_facet_query) {- //LOG (INFO) << "adding hash tokens for hash " << fhash;- a_facet.hash_tokens[fhash_str] = fquery_hashes.at(fhash_str);++ std::string fhash_str = std::to_string(fhash);+ if(a_facet.is_range_query) {+ int64_t doc_val = get_doc_val_from_sort_index(sort_index_it, doc_seq_id);++ std::pair<std::string, std::string> range_pair {};+ if(a_facet.get_range(std::to_string(doc_val), range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count += 1;+ }+ } else if(!use_facet_query || fquery_hashes.find(fhash_str) != fquery_hashes.end()) {+ facet_count_t& facet_count = a_facet.result_map[fhash_str];+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;+ facet_count.doc_id = doc_seq_id;+ facet_count.array_pos = j;+ if(group_limit) {+ a_facet.hash_groups[fhash].emplace(distinct_id);+ } else {+ facet_count.count += 1;+ }+ if(use_facet_query) {+ //LOG (INFO) << "adding hash tokens for hash " << fhash;+ a_facet.hash_tokens[fhash_str] = fquery_hashes.at(fhash_str);+ }}}+ } else {+ break;}}}@@ -4564,10 +4489,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(size_t findex=0; findex < facets.size(); findex++) {const auto& a_facet = facets[findex];- const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);- const auto facet_hash_index_exists = ((field_facet_mapping_it != facet_index_v3.end())- || (field_single_val_facet_mapping_it != single_val_facet_index_v3.end()));+ bool facet_hash_index_exist = facet_hash_index.find(a_facet.field_name) != facet_hash_index.end();facet_infos[findex].use_facet_query = false;@@ -4665,40 +4587,40 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&if(!use_facet_intersection) {#else//checking if facets is migrated to old index- if(facet_hash_index_exists) {-#endif- if(facet_field.is_array()) {- const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }+ if(facet_hash_index_exist) {+#endif+ std::vector<uint32_t> facet_hashes;++ auto facet_index = facet_hash_index.at(a_facet.field_name);+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();+ facet_index_it.skip_to(seq_id);++ if(facet_index_it.valid()) {+ posting_list_t::get_offsets(facet_index_it, facet_hashes);- std::vector<size_t> array_indices;- posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);+ if(facet_field.is_array()) {+ std::vector<size_t> array_indices;+ posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);- for(size_t array_index: array_indices) {- if(array_index < doc_fvalues_it->second.length) {- std::string hash = std::to_string(doc_fvalues_it->second.hashes[array_index]);+ for(size_t array_index: array_indices) {+ if(array_index < facet_hashes.size()) {+ std::string hash = std::to_string(facet_hashes[array_index]);- /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "- << array_index;*/+ /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "+ << array_index;*/- if(facet_infos[findex].hashes.count(hash) == 0) {- //LOG(INFO) << "adding searched_tokens for hash " << hash;- facet_infos[findex].hashes.emplace(hash, searched_tokens);+ if(facet_infos[findex].hashes.count(hash) == 0) {+ //LOG(INFO) << "adding searched_tokens for hash " << hash;+ facet_infos[findex].hashes.emplace(hash, searched_tokens);+ }}}- }- } else {- const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }-- std::string hash = std::to_string(doc_single_fvalues_it->second);- if(facet_infos[findex].hashes.count(hash) == 0) {- //LOG(INFO) << "adding searched_tokens for hash " << hash;- facet_infos[findex].hashes.emplace(hash, searched_tokens);+ } else {+ std::string hash = std::to_string(facet_hashes[0]);+ if(facet_infos[findex].hashes.count(hash) == 0) {+ //LOG(INFO) << "adding searched_tokens for hash " << hash;+ facet_infos[findex].hashes.emplace(hash, searched_tokens);+ }}}} else { //for new index adding search tokens@@ -5433,37 +5355,29 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,uint32_t distinct_id = 1; // some constant initial value// calculate hash from group_by_fieldsfor(const auto& field: group_by_fields) {- const auto& field_facet_mapping_it = facet_index_v3.find(field);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ const auto& facet_hash_index_it = facet_hash_index.find(field);+ if(facet_hash_index_it == facet_hash_index.end()) {continue;}-- if(search_schema.at(field).is_array()) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- continue;- }+ auto facet_index = facet_hash_index_it->second;+ std::vector<uint32_t> facet_hashes;+ posting_list_t::iterator_t facet_index_it = facet_index->new_iterator();+ facet_index_it.skip_to(seq_id);- const auto& facet_hashes = facet_hashes_it->second;- //LOG(INFO) << "combining hashes for facet array ";- for(size_t i = 0; i < facet_hashes.size(); i++) {- distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);- }- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(facet_index_it.valid()) {+ posting_list_t::get_offsets(facet_index_it, facet_hashes);- if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {- continue;+ if(search_schema.at(field).is_array()) {+ //LOG(INFO) << "combining hashes for facet array ";+ for(size_t i = 0; i < facet_hashes.size(); i++) {+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes[i]);+ }+ } else {+ const auto& facet_hash = facet_hashes[0];+ //LOG(INFO) << "combining hashes for facet ";+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);}-- const auto& facet_hash = facet_hashes_it->second;- //LOG(INFO) << "combining hashes for facet ";- distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);}}//LOG(INFO) << "distinct id " << distinct_id;@@ -5700,23 +5614,16 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const}// remove facets- const auto& field_facets_it = facet_index_v3.find(field_name);- if(field_facets_it != facet_index_v3.end()) {- const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);+ const auto& facet_hash_index_it = facet_hash_index.find(field_name);+ if(facet_hash_index_it != facet_hash_index.end()) {+ auto facet_index = facet_hash_index_it->second;+ if(facet_index) {+ facet_index->erase(seq_id);}}- const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);- if(field_single_val_facets_it != single_val_facet_index_v3.end()) {- const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {- field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);- }- }- facet_index_v4->erase(field_name);+ facet_index_v4->remove(field_name, seq_id);// remove sort fieldif(sort_index.count(field_name) != 0) {@@ -5918,21 +5825,9 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec}if(del_field.is_facet()) {- if(del_field.is_array()) {- auto& arr = facet_index_v3[del_field.name];- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- delete arr[i];- }-- facet_index_v3.erase(del_field.name);- } else {- auto& arr = single_val_facet_index_v3[del_field.name];- for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {- delete arr[i];- }+ delete facet_hash_index.at(del_field.name);+ facet_hash_index.erase(del_field.name);- single_val_facet_index_v3.erase(del_field.name);- }facet_index_v4->erase(del_field.name);if(!del_field.is_string()) {diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 6a6dbe0d..9b2fc866 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -22,16 +22,21 @@ void num_tree_t::insert(int64_t value, uint32_t id, bool is_facet) {counter_list.emplace_back(value, facet_count);} else {count_list node(value, facet_count);- uint32_t ind = 0;- for(ind; ind < counter_list.size(); ++ind) {++ auto ind = 0;++ for(; ind < counter_list.size(); ++ind) {if(counter_list[ind].facet_value == value) {counter_list[ind].count = facet_count;- if((ind > 1) &&- (counter_list[ind-1].count < counter_list[ind].count)) {- count_list temp = counter_list[ind-1];- counter_list[ind-1] = counter_list[ind];- counter_list[ind] = temp;+ if(ind > 1) {+ auto curr = ind;+ while (curr && (counter_list[curr-1].count < counter_list[curr].count)) {+ count_list temp = counter_list[curr-1];+ counter_list[curr-1] = counter_list[curr];+ counter_list[curr] = temp;+ --curr;+ }}break;}@@ -426,11 +431,11 @@ size_t num_tree_t::intersect(const uint32_t* result_ids, int result_ids_len, intreturn found.size();}-void num_tree_t::get_facet_indexes(std::function<void(uint32_t seq_id, uint32_t count_index)> functor) {+size_t num_tree_t::get_facet_indexes(std::map<uint32_t, std::vector<uint32_t>>& seqid_countIndexes) {//check if facet fieldif(counter_list.empty()) {- return;+ return 0;}std::vector<uint32_t> id_list;@@ -442,11 +447,11 @@ void num_tree_t::get_facet_indexes(std::function<void(uint32_t seq_id, uint32_t//emplacing seq_id=>count_indexfor(const auto& id : id_list) {- functor(id, int64map_it->first);+ seqid_countIndexes[id].emplace_back(int64map_it->first);}id_list.clear();}- return;+ return seqid_countIndexes.size();}diff --git a/src/posting_list.cpp b/src/posting_list.cppindex dc82fb69..3e28cad9 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -808,6 +808,27 @@ bool posting_list_t::take_id(result_iter_state_t& istate, uint32_t id) {return true;}+void posting_list_t::get_offsets(iterator_t& iter, std::vector<uint32_t>& positions) {+ block_t* curr_block = iter.block();+ uint32_t curr_index = iter.index();++ if(curr_block == nullptr || curr_index == UINT32_MAX) {+ return;+ }++ uint32_t* offsets = iter.offsets;+ uint32_t start_offset = iter.offset_index[curr_index];+ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?+ curr_block->offsets.getLength() :+ iter.offset_index[curr_index + 1];++ while(start_offset < end_offset) {+ int pos = offsets[start_offset];+ positions.push_back(pos);+ ++start_offset;+ }+}+bool posting_list_t::get_offsets(const std::vector<iterator_t>& its,std::map<size_t, std::vector<token_positions_t>>& array_token_pos) {@@ -1657,7 +1678,12 @@ void posting_list_t::iterator_t::next() {}uint32_t posting_list_t::iterator_t::last_block_id() const {- return ids[curr_block->size() - 1];+ auto size = curr_block->size();+ if(size == 0) {+ return 0;+ }++ return ids[size - 1];}uint32_t posting_list_t::iterator_t::id() const {