Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Separate candidate limit and combination limit.
Kishore Nallan266a08c56839
diff --git a/include/index.h b/include/index.hindex 992f6ac6..7d7e3d05 100644--- a/include/index.h+++ b/include/index.h@@ -255,7 +255,7 @@ private:const size_t group_limit, const std::vector<std::string>& group_by_fields,const std::vector<token_t>& query_tokens,bool prioritize_exact_match,- size_t combination_limit,+ bool exhaustive_search,size_t concurrency) const;void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,@@ -314,7 +314,7 @@ public:enum {FIELD_LIMIT_NUM = 100};enum {COMBINATION_MAX_LIMIT = 10000};- enum {COMBINATION_MIN_LIMIT = 4};+ enum {COMBINATION_MIN_LIMIT = 10};// If the number of results found is less than this threshold, Typesense will attempt to drop the tokens// in the query that have the least individual hits one by one until enough results are found.@@ -394,7 +394,7 @@ public:const std::vector<std::string>& group_by_fields,const std::string& default_sorting_field,bool prioritize_exact_match,- const size_t combination_limit,+ bool exhaustive_search,size_t concurrency) const;Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);diff --git a/src/index.cpp b/src/index.cppindex f6356c34..8a75f6b5 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -916,7 +916,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,const std::vector<std::string>& group_by_fields,const std::vector<token_t>& query_tokens,bool prioritize_exact_match,- const size_t combination_limit,+ const bool exhaustive_search,const size_t concurrency) const {auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };@@ -945,6 +945,8 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,}}+ size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;+for(long long n=0; n<N && n<combination_limit; ++n) {// every element in `query_suggestion` contains a token and its associated hitsstd::vector<art_leaf*> query_suggestion(token_candidates_vec.size());@@ -1482,7 +1484,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,const std::vector<std::string>& group_by_fields,const std::string& default_sorting_field,bool prioritize_exact_match,- const size_t combination_limit,+ const bool exhaustive_search,const size_t concurrency) const {std::shared_lock lock(mutex);@@ -1669,7 +1671,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,token_order, field_prefix,- drop_tokens_threshold, typo_tokens_threshold, combination_limit);+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);// do synonym based searchesfor(const auto& syn_tokens: q_pos_synonyms) {@@ -1682,7 +1684,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,token_order, field_prefix,- drop_tokens_threshold, typo_tokens_threshold, combination_limit);+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);}// concat is done only for multi-field searches as `ftopster` will be empty for single-field search@@ -2010,8 +2012,6 @@ void Index::search_field(const uint8_t & field_id,max_cost = 0;}- size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;-// To prevent us from doing ART search repeatedly as we iterate through possible correctionsspp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;@@ -2039,6 +2039,9 @@ void Index::search_field(const uint8_t & field_id,long long n = 0;long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);+ const size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;+ const size_t num_fuzzy_candidates = exhaustive_search ? 10000 : 4;+while(n < N && n < combination_limit) {// Outerloop generates combinations of [cost to max_cost] for each token// For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.@@ -2070,7 +2073,7 @@ void Index::search_field(const uint8_t & field_id,// need less candidates for filtered searches since we already only pick tokens with resultsart_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,- costs[token_index], costs[token_index], combination_limit, token_order, prefix_search,+ costs[token_index], costs[token_index], num_fuzzy_candidates, token_order, prefix_search,filter_ids, filter_ids_length, leaves, unique_tokens);if(!leaves.empty()) {diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 684c77bf..5d4fdc31 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -20,14 +20,21 @@ void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** istd::vector<uint32_t> consolidated_ids;while(it_start != int64map.end() && it_start->first <= end) {+ uint32_t* values = it_start->second->uncompress();+for(size_t i = 0; i < it_start->second->getLength(); i++) {- consolidated_ids.push_back(it_start->second->at(i));+ consolidated_ids.push_back(values[i]);}+ delete [] values;it_start++;}- std::sort(consolidated_ids.begin(), consolidated_ids.end());+ if(consolidated_ids.size() > 50000) {+ parasort(consolidated_ids.size(), &consolidated_ids[0], 4);+ } else {+ std::sort(consolidated_ids.begin(), consolidated_ids.end());+ }uint32_t *out = nullptr;ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),