Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Shard facet index.
Kishore Nallan47c13b12570f
diff --git a/include/index.h b/include/index.hindex feb2beb1..7d8f83b8 100644--- a/include/index.h+++ b/include/index.h@@ -22,6 +22,10 @@#include "posting_list.h"#include "threadpool.h"+static constexpr size_t ARRAY_FACET_DIM = 4;+using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;+using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;+struct token_t {size_t position;std::string value;@@ -403,7 +407,7 @@ private:spp::sparse_hash_map<std::string, spp::sparse_hash_map<std::string, std::vector<uint32_t>>*> geopoint_index;// facet_field => (seq_id => values)- spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, facet_hash_values_t>*> facet_index_v3;+ spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;// sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;diff --git a/src/index.cpp b/src/index.cppindex 6168d107..94993f10 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -80,8 +80,12 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*}for(const auto& pair: facet_schema) {- auto doc_to_values = new spp::sparse_hash_map<uint32_t, facet_hash_values_t>();- facet_index_v3.emplace(pair.first, doc_to_values);+ array_mapped_facet_t facet_array;+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ facet_array[i] = new facet_map_t();+ }++ facet_index_v3.emplace(pair.first, facet_array);}num_documents = 0;@@ -130,9 +134,11 @@ Index::~Index() {sort_index.clear();- for(auto& kv: facet_index_v3) {- delete kv.second;- kv.second = nullptr;+ for(auto& field_name_facet_map_array: facet_index_v3) {+ for(auto& facet_map: field_name_facet_map_array.second) {+ delete facet_map;+ facet_map = nullptr;+ }}facet_index_v3.clear();@@ -793,7 +799,7 @@ void Index::index_strings_field(const int64_t score, art_tree *t,fhashvalues.hashes[i] = facet_hashes[i];}- facet_index_v3[a_field.name]->emplace(seq_id, std::move(fhashvalues));+ facet_index_v3[a_field.name][seq_id % ARRAY_FACET_DIM]->emplace(seq_id, std::move(fhashvalues));}}@@ -904,9 +910,9 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,for(size_t i = 0; i < results_size; i++) {uint32_t doc_seq_id = result_ids[i];- const auto& facet_hashes_it = field_facet_mapping->find(doc_seq_id);+ const auto& facet_hashes_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);- if(facet_hashes_it == field_facet_mapping->end()) {+ if(facet_hashes_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {continue;}@@ -2468,8 +2474,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&for(size_t i = 0; i < std::min<size_t>(1000, field_result_ids_len); i++) {uint32_t seq_id = field_result_ids[i];- const auto doc_fvalues_it = field_facet_mapping_it->second->find(seq_id);- if(doc_fvalues_it == field_facet_mapping_it->second->end()) {+ const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {continue;}@@ -3117,9 +3123,9 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,}const auto& field_facet_mapping = field_facet_mapping_it->second;- const auto& facet_hashes_it = field_facet_mapping->find(seq_id);+ const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);- if(facet_hashes_it == field_facet_mapping->end()) {+ if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {continue;}@@ -3285,9 +3291,9 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docconst auto& field_facets_it = facet_index_v3.find(field_name);if(field_facets_it != facet_index_v3.end()) {- const auto& fvalues_it = field_facets_it->second->find(seq_id);- if(fvalues_it != field_facets_it->second->end()) {- field_facets_it->second->erase(fvalues_it);+ const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);+ if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {+ field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);}}@@ -3361,8 +3367,12 @@ void Index::refresh_schemas(const std::vector<field>& new_fields) {if(new_field.is_facet()) {facet_schema.emplace(new_field.name, new_field);- auto doc_to_values = new spp::sparse_hash_map<uint32_t, facet_hash_values_t>();- facet_index_v3.emplace(new_field.name, doc_to_values);+ array_mapped_facet_t facet_array;+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {+ facet_array[i] = new facet_map_t();+ }++ facet_index_v3.emplace(new_field.name, facet_array);// initialize for non-string facet fieldsif(!new_field.is_string()) {