Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Spread the hash for sort index key.
Kishore Nallan9921d2dc9c53
diff --git a/include/collection.h b/include/collection.hindex 34b79b28..c6c4ee43 100644--- a/include/collection.h+++ b/include/collection.h@@ -639,7 +639,7 @@ public:void reference_populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,std::vector<sort_by>& sort_fields_std,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values) const;int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;diff --git a/include/index.h b/include/index.hindex 40589a3b..92d6ac5e 100644--- a/include/index.h+++ b/include/index.h@@ -329,6 +329,12 @@ struct group_by_field_it_t {posting_list_t::iterator_t it;};+struct Hasher32 {+ // Helps to spread the hash key and is used for sort index.+ // see: https://github.com/greg7mdp/sparsepp/issues/21#issuecomment-270816275+ size_t operator()(uint32_t k) const { return (k ^ 2166136261U) * 16777619UL; }+};+class Index {private:mutable std::shared_mutex mutex;@@ -365,9 +371,9 @@ private:facet_index_t* facet_index_v4 = nullptr;// sort_field => (seq_id => value)- spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*> sort_index;typedef spp::sparse_hash_map<std::string,- spp::sparse_hash_map<uint32_t, int64_t>*>::iterator sort_index_iterator;+ spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*>::iterator sort_index_iterator;// str_sort_field => adi_tree_tspp::sparse_hash_map<std::string, adi_tree_t*> str_sort_index;@@ -389,13 +395,13 @@ private:// used as sentinels- static spp::sparse_hash_map<uint32_t, int64_t> text_match_sentinel_value;- static spp::sparse_hash_map<uint32_t, int64_t> seq_id_sentinel_value;- static spp::sparse_hash_map<uint32_t, int64_t> eval_sentinel_value;- static spp::sparse_hash_map<uint32_t, int64_t> geo_sentinel_value;- static spp::sparse_hash_map<uint32_t, int64_t> str_sentinel_value;- static spp::sparse_hash_map<uint32_t, int64_t> vector_distance_sentinel_value;- static spp::sparse_hash_map<uint32_t, int64_t> vector_query_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t, Hasher32> text_match_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t, Hasher32> seq_id_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t, Hasher32> eval_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t, Hasher32> geo_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t, Hasher32> str_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t, Hasher32> vector_distance_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t, Hasher32> vector_query_sentinel_value;// Internal utility functions@@ -496,7 +502,7 @@ private:const size_t max_candidates,int syn_orig_num_tokens,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,std::set<uint64>& query_hashes,std::vector<uint32_t>& id_buff, const std::string& collection_name = "") const;@@ -623,7 +629,7 @@ public:Topster *topster, const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,const uint32_t seq_id, const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const size_t group_limit,const std::vector<std::string> &group_by_fields,@@ -774,7 +780,7 @@ public:filter_result_iterator_t* const filter_result_iterator,const size_t concurrency,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,const std::string& collection_name = "") const;@@ -787,11 +793,11 @@ public:void populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,std::vector<sort_by>& sort_fields_std,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values) const;void populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>& geopoint_indices,std::vector<sort_by>& sort_fields_std,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values) const;int64_t reference_string_sort_score(const std::string& field_name, const uint32_t& seq_id) const;@@ -824,7 +830,7 @@ public:const size_t max_extra_suffix, const std::vector<token_t>& query_tokens, Topster* actual_topster,filter_result_iterator_t* const filter_result_iterator,const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const std::vector<uint32_t>& curated_ids_sorted,const std::unordered_set<uint32_t>& excluded_group_ids,@@ -860,7 +866,7 @@ public:filter_result_iterator_t* const filter_result_iterator,std::set<uint64>& query_hashes,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,tsl::htrie_map<char, token_leaf>& qtoken_set,const std::string& collection_name = "") const;@@ -873,7 +879,7 @@ public:const bool group_missing_values,Topster* actual_topster,const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const std::vector<uint32_t>& curated_ids_sorted,filter_result_iterator_t*& filter_result_iterator,@@ -916,7 +922,7 @@ public:size_t min_len_2typo,int syn_orig_num_tokens,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,const std::string& collection_name = "") const;@@ -955,7 +961,7 @@ public:size_t exclude_token_ids_size,const std::unordered_set<uint32_t>& excluded_group_ids,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,std::vector<uint32_t>& id_buff,uint32_t*& all_result_ids, size_t& all_result_ids_len,@@ -990,7 +996,7 @@ public:std::vector<const override_t*>& matched_dynamic_overrides) const;Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices, uint32_t seq_id,const std::map<basic_string<char>, reference_filter_result_t>& references,std::vector<uint32_t>& filter_indexes,diff --git a/src/collection.cpp b/src/collection.cppindex 817faf24..92a6e15b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -6061,7 +6061,7 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t kvoid Collection::reference_populate_sort_mapping(int *sort_order, std::vector<size_t> &geopoint_indices,std::vector<sort_by> &sort_fields_std,- std::array<spp::sparse_hash_map<uint32_t, int64_t> *, 3> &field_values)+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32> *, 3> &field_values)const {std::shared_lock lock(mutex);index->populate_sort_mapping_with_lock(sort_order, geopoint_indices, sort_fields_std, field_values);diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex e1c14489..6385e80c 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -1036,7 +1036,7 @@ void filter_result_iterator_t::init() {std::vector<uint32_t> exact_geo_result_ids;if (f.is_single_geopoint()) {- spp::sparse_hash_map<uint32_t, int64_t>* sort_field_index = index->sort_index.at(f.name);+ auto sort_field_index = index->sort_index.at(f.name);for (auto result_id : geo_result_ids) {// no need to check for existence of `result_id` because of indexer based pre-filtering abovediff --git a/src/index.cpp b/src/index.cppindex c508955d..5b1fc8e2 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -44,13 +44,13 @@}#define FACET_INDEX_THRESHOLD 1000000000-spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;-spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;-spp::sparse_hash_map<uint32_t, int64_t> Index::eval_sentinel_value;-spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;-spp::sparse_hash_map<uint32_t, int64_t> Index::str_sentinel_value;-spp::sparse_hash_map<uint32_t, int64_t> Index::vector_distance_sentinel_value;-spp::sparse_hash_map<uint32_t, int64_t> Index::vector_query_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t, Hasher32> Index::text_match_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t, Hasher32> Index::seq_id_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t, Hasher32> Index::eval_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t, Hasher32> Index::geo_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t, Hasher32> Index::str_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t, Hasher32> Index::vector_distance_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t, Hasher32> Index::vector_query_sentinel_value;Index::Index(const std::string& name, const uint32_t collection_id, const Store* store,SynonymIndex* synonym_index, ThreadPool* thread_pool,@@ -100,7 +100,7 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*adi_tree_t* tree = new adi_tree_t();str_sort_index.emplace(a_field.name, tree);} else if(a_field.type != field_types::GEOPOINT_ARRAY) {- spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();+ auto doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t, Hasher32>();sort_index.emplace(a_field.name, doc_to_score);}}@@ -1059,7 +1059,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>// add numerical values automatically into sort index if sorting is enabledif(afield.is_num_sortable() && afield.type != field_types::GEOPOINT_ARRAY) {- spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = sort_index.at(afield.name);+ auto doc_to_score = sort_index.at(afield.name);bool is_integer = afield.is_integer();bool is_float = afield.is_float();@@ -1547,7 +1547,7 @@ Option<bool> Index::search_all_candidates(const size_t num_search_fields,const size_t max_candidates,int syn_orig_num_tokens,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,std::set<uint64>& query_hashes,std::vector<uint32_t>& id_buff, const std::string& collection_name) const {@@ -1640,7 +1640,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);int sort_order[3]; // 1 or -1 based on DESC or ASC respectively- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values;std::vector<size_t> geopoint_indices;populate_sort_mapping(sort_order, geopoint_indices, sort_fields, field_values);@@ -2410,7 +2410,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conshandle_exclusion(num_search_fields, field_query_tokens, the_fields, exclude_token_ids, exclude_token_ids_size);int sort_order[3]; // 1 or -1 based on DESC or ASC respectively- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values;std::vector<size_t> geopoint_indices;populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);@@ -3425,7 +3425,7 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fsize_t min_len_2typo,int syn_orig_num_tokens,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,const std::string& collection_name) const {@@ -3835,7 +3835,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_tokenconst uint32_t* exclude_token_ids, size_t exclude_token_ids_size,const std::unordered_set<uint32_t>& excluded_group_ids,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,std::vector<uint32_t>& id_buff,uint32_t*& all_result_ids, size_t& all_result_ids_len,@@ -4160,7 +4160,7 @@ Option<bool> Index::search_across_fields(const std::vector<token_t>& query_token}Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices,uint32_t seq_id, const std::map<basic_string<char>, reference_filter_result_t>& references,std::vector<uint32_t>& filter_indexes, int64_t max_field_match_score, int64_t* scores,@@ -4170,7 +4170,7 @@ Option<bool> Index::compute_sort_scores(const std::vector<sort_by>& sort_fields,int64_t geopoint_distances[3];for(auto& i: geopoint_indices) {- spp::sparse_hash_map<uint32_t, int64_t>* geopoints = field_values[i];+ auto geopoints = field_values[i];int64_t dist = INT32_MAX;S2LatLng reference_lat_lng;@@ -4751,7 +4751,7 @@ Option<bool> Index::do_phrase_search(const size_t num_search_fields, const std::const bool group_missing_values,Topster* actual_topster,const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const std::vector<uint32_t>& curated_ids_sorted,filter_result_iterator_t*& filter_result_iterator,@@ -4951,7 +4951,7 @@ Option<bool> Index::do_synonym_search(const std::vector<search_field_t>& the_fiefilter_result_iterator_t* const filter_result_iterator,std::set<uint64>& query_hashes,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,tsl::htrie_map<char, token_leaf>& qtoken_set,const std::string& collection_name) const {@@ -4991,7 +4991,7 @@ Option<bool> Index::do_infix_search(const size_t num_search_fields, const std::vconst std::vector<token_t>& query_tokens, Topster* actual_topster,filter_result_iterator_t* const filter_result_iterator,const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const std::vector<uint32_t>& curated_ids_sorted,const std::unordered_set<uint32_t>& excluded_group_ids,@@ -5369,7 +5369,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootfilter_result_iterator_t* const filter_result_iterator,const size_t concurrency,const int* sort_order,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,const std::string& collection_name) const {@@ -5527,7 +5527,7 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_rootvoid Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopoint_indices,std::vector<sort_by>& sort_fields_std,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const {+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values) const {for (size_t i = 0; i < sort_fields_std.size(); i++) {if (!sort_fields_std[i].reference_collection_name.empty()) {auto& cm = CollectionManager::get_instance();@@ -5538,7 +5538,7 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopointstd::vector<sort_by> ref_sort_fields_std;ref_sort_fields_std.emplace_back(sort_fields_std[i]);ref_sort_fields_std.front().reference_collection_name.clear();- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> ref_field_values;+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> ref_field_values;ref_collection->reference_populate_sort_mapping(ref_sort_order, ref_geopoint_indices,ref_sort_fields_std, ref_field_values);@@ -5602,7 +5602,7 @@ void Index::populate_sort_mapping(int* sort_order, std::vector<size_t>& geopointvoid Index::populate_sort_mapping_with_lock(int* sort_order, std::vector<size_t>& geopoint_indices,std::vector<sort_by>& sort_fields_std,- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values) const {+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values) const {std::shared_lock lock(mutex);populate_sort_mapping(sort_order, geopoint_indices, sort_fields_std, field_values);}@@ -5923,7 +5923,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_map<uint64_t, uint32_t>& groups_processed,const uint32_t seq_id, const int sort_order[3],- std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const size_t group_limit, const std::vector<std::string>& group_by_fields,const bool group_missing_values,@@ -5936,7 +5936,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16int64_t geopoint_distances[3];for(auto& i: geopoint_indices) {- spp::sparse_hash_map<uint32_t, int64_t>* geopoints = field_values[i];+ auto geopoints = field_values[i];int64_t dist = INT32_MAX;S2LatLng reference_lat_lng;@@ -6543,7 +6543,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecif(new_field.is_sortable()) {if(new_field.is_num_sortable()) {- spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();+ auto doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t, Hasher32>();sort_index.emplace(new_field.name, doc_to_score);} else if(new_field.is_str_sortable()) {str_sort_index.emplace(new_field.name, new adi_tree_t);