Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Expose max_candidates parameter to the API.
Kishore Nallan609ac35c6d41
diff --git a/include/collection.h b/include/collection.hindex 6a273710..28d45334 100644--- a/include/collection.h+++ b/include/collection.h@@ -394,7 +394,8 @@ public:size_t search_stop_millis = 6000*1000,size_t min_len_1typo = 4,size_t min_len_2typo = 7,- bool split_join_tokens = true) const;+ bool split_join_tokens = true,+ size_t max_candidates = 4) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/include/index.h b/include/index.hindex 6b5edc19..32fbf52b 100644--- a/include/index.h+++ b/include/index.h@@ -281,6 +281,7 @@ struct search_args {size_t search_cutoff_ms;size_t min_len_1typo;size_t min_len_2typo;+ size_t max_candidates;spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;@@ -305,7 +306,8 @@ struct search_args {const std::vector<const override_t*>& dynamic_overrides,size_t search_cutoff_ms,size_t min_len_1typo,- size_t min_len_2typo):+ size_t min_len_2typo,+ size_t max_candidates):field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -316,7 +318,7 @@ struct search_args {prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),exhaustive_search(exhaustive_search), concurrency(concurrency),filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms),- min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo) {+ min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo), max_candidates(max_candidates) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -503,7 +505,8 @@ private:size_t typo_tokens_threshold,bool exhaustive_search,size_t min_len_1typo,- size_t min_len_2typo) const;+ size_t min_len_2typo,+ size_t max_candidates) const;void search_candidates(const uint8_t & field_id,bool field_is_array,@@ -688,7 +691,8 @@ public:size_t concurrency,size_t search_cutoff_ms,size_t min_len_1typo,- size_t min_len_2typo) const;+ size_t min_len_2typo,+ size_t max_candidates) const;Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);diff --git a/src/collection.cpp b/src/collection.cppindex a326b15e..1b00130a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -678,7 +678,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sconst size_t search_stop_millis,const size_t min_len_1typo,const size_t min_len_2typo,- bool split_join_tokens) const {+ bool split_join_tokens,+ const size_t max_candidates) const {std::shared_lock lock(mutex);@@ -994,7 +995,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sgroup_by_fields, group_limit, default_sorting_field, prioritize_exact_match,exhaustive_search, 4, filter_overrides,search_stop_millis,- min_len_1typo, min_len_2typo);+ min_len_1typo, min_len_2typo, max_candidates);index->run_search(search_params);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex c3e0eae1..c2322b8c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -565,6 +565,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *PINNED_HITS = "pinned_hits";const char *HIDDEN_HITS = "hidden_hits";const char *ENABLE_OVERRIDES = "enable_overrides";+ const char *MAX_CANDIDATES = "max_candidates";// strings under this length will be fully highlighted, instead of showing a snippet of relevant portionconst char *SNIPPET_THRESHOLD = "snippet_threshold";@@ -828,6 +829,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rereq_params[ENABLE_OVERRIDES] = "true";}+ if(req_params.count(MAX_CANDIDATES) == 0) {+ req_params[MAX_CANDIDATES] = exhaustive_search ? "10000" : "4";+ }+bool enable_overrides = (req_params[ENABLE_OVERRIDES] == "true");CollectionManager & collectionManager = CollectionManager::get_instance();@@ -892,7 +897,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restatic_cast<size_t>(std::stol(req_params[SEARCH_CUTOFF_MS])),static_cast<size_t>(std::stol(req_params[MIN_LEN_1TYPO])),static_cast<size_t>(std::stol(req_params[MIN_LEN_2TYPO])),- split_join_tokens+ split_join_tokens,+ static_cast<size_t>(std::stol(req_params[MAX_CANDIDATES])));uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex ee2c5cc6..4f33b5ee 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1629,7 +1629,8 @@ void Index::run_search(search_args* search_params) {search_params->concurrency,search_params->search_cutoff_ms,search_params->min_len_1typo,- search_params->min_len_2typo);+ search_params->min_len_2typo,+ search_params->max_candidates);}void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,@@ -1974,7 +1975,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&search_field(0, window_tokens, search_tokens, nullptr, 0, num_toks_dropped, field_it->second, field_name,nullptr, 0, {}, {}, 2, searched_queries, topster, groups_processed,&result_ids, result_ids_len, field_num_results, 0, group_by_fields,- false, 4, query_hashes, token_order, false, 0, 1, false, 3, 7);+ false, 4, query_hashes, token_order, false, 0, 1, false, 3, 7, 4);delete [] result_ids;@@ -2030,7 +2031,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,const size_t concurrency,const size_t search_cutoff_ms,size_t min_len_1typo,- size_t min_len_2typo) const {+ size_t min_len_2typo,+ const size_t max_candidates) const {search_begin = std::chrono::high_resolution_clock::now();search_stop_ms = search_cutoff_ms;@@ -2258,7 +2260,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,query_hashes, token_order, field_prefix,drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,- min_len_1typo, min_len_2typo);+ min_len_1typo, min_len_2typo, max_candidates);} else if(actual_filter_ids_length != 0) {// indicates exact match querycurate_filtered_ids(filters, curated_ids, exclude_token_ids,@@ -2306,7 +2308,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,query_hashes, token_order, field_prefix,drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,- min_len_1typo, min_len_2typo);+ min_len_1typo, min_len_2typo, max_candidates);}}@@ -2702,7 +2704,7 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&facet_field, facet_field.faceted_name(),all_result_ids, all_result_ids_len, {}, {}, 2, searched_queries, topster, groups_processed,&field_result_ids, field_result_ids_len, field_num_results, 0, group_by_fields,- false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, 3, 1000);+ false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, 3, 1000, 4);//LOG(INFO) << "searched_queries.size: " << searched_queries.size();@@ -2974,7 +2976,8 @@ void Index::search_field(const uint8_t & field_id,const size_t typo_tokens_threshold,const bool exhaustive_search,size_t min_len_1typo,- size_t min_len_2typo) const {+ size_t min_len_2typo,+ const size_t max_candidates) const {// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens@@ -3013,7 +3016,6 @@ void Index::search_field(const uint8_t & field_id,long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);const size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;- const size_t num_fuzzy_candidates = exhaustive_search ? 10000 : 4;while(n < N && n < combination_limit) {RETURN_CIRCUIT_BREAKER@@ -3051,7 +3053,7 @@ void Index::search_field(const uint8_t & field_id,// need less candidates for filtered searches since we already only pick tokens with resultsart_fuzzy_search(search_index.at(field_name), (const unsigned char *) token.c_str(), token_len,- costs[token_index], costs[token_index], num_fuzzy_candidates, token_order, prefix_search,+ costs[token_index], costs[token_index], max_candidates, token_order, prefix_search,filter_ids, filter_ids_length, leaves, unique_tokens);/*auto timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(@@ -3168,7 +3170,7 @@ void Index::search_field(const uint8_t & field_id,all_result_ids_len, field_num_results, group_limit, group_by_fields,prioritize_exact_match, concurrency, query_hashes,token_order, prefix, drop_tokens_threshold, typo_tokens_threshold,- exhaustive_search, min_len_1typo, min_len_2typo);+ exhaustive_search, min_len_1typo, min_len_2typo, max_candidates);}}