Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Separate candidate limit and combination limit.

Kishore Nallan266a08c56839
diff --git a/include/index.h b/include/index.h
index 992f6ac6..7d7e3d05 100644
--- a/include/index.h
+++ b/include/index.h
@@ -255,7 +255,7 @@ private:
const size_t group_limit, const std::vector<std::string>& group_by_fields,
const std::vector<token_t>& query_tokens,
bool prioritize_exact_match,
- size_t combination_limit,
+ bool exhaustive_search,
size_t concurrency) const;
void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,
@@ -314,7 +314,7 @@ public:
enum {FIELD_LIMIT_NUM = 100};
enum {COMBINATION_MAX_LIMIT = 10000};
- enum {COMBINATION_MIN_LIMIT = 4};
+ enum {COMBINATION_MIN_LIMIT = 10};
// If the number of results found is less than this threshold, Typesense will attempt to drop the tokens
// in the query that have the least individual hits one by one until enough results are found.
@@ -394,7 +394,7 @@ public:
const std::vector<std::string>& group_by_fields,
const std::string& default_sorting_field,
bool prioritize_exact_match,
- const size_t combination_limit,
+ bool exhaustive_search,
size_t concurrency) const;
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);
diff --git a/src/index.cpp b/src/index.cpp
index f6356c34..8a75f6b5 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -916,7 +916,7 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
const std::vector<std::string>& group_by_fields,
const std::vector<token_t>& query_tokens,
bool prioritize_exact_match,
- const size_t combination_limit,
+ const bool exhaustive_search,
const size_t concurrency) const {
auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };
@@ -945,6 +945,8 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
}
}
+ size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
+
for(long long n=0; n<N && n<combination_limit; ++n) {
// every element in `query_suggestion` contains a token and its associated hits
std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());
@@ -1482,7 +1484,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
const std::vector<std::string>& group_by_fields,
const std::string& default_sorting_field,
bool prioritize_exact_match,
- const size_t combination_limit,
+ const bool exhaustive_search,
const size_t concurrency) const {
std::shared_lock lock(mutex);
@@ -1669,7 +1671,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
token_order, field_prefix,
- drop_tokens_threshold, typo_tokens_threshold, combination_limit);
+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);
// do synonym based searches
for(const auto& syn_tokens: q_pos_synonyms) {
@@ -1682,7 +1684,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
token_order, field_prefix,
- drop_tokens_threshold, typo_tokens_threshold, combination_limit);
+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);
}
// concat is done only for multi-field searches as `ftopster` will be empty for single-field search
@@ -2010,8 +2012,6 @@ void Index::search_field(const uint8_t & field_id,
max_cost = 0;
}
- size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
-
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
@@ -2039,6 +2039,9 @@ void Index::search_field(const uint8_t & field_id,
long long n = 0;
long long int N = std::accumulate(token_to_costs.begin(), token_to_costs.end(), 1LL, product);
+ const size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
+ const size_t num_fuzzy_candidates = exhaustive_search ? 10000 : 4;
+
while(n < N && n < combination_limit) {
// Outerloop generates combinations of [cost to max_cost] for each token
// For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.
@@ -2070,7 +2073,7 @@ void Index::search_field(const uint8_t & field_id,
// need less candidates for filtered searches since we already only pick tokens with results
art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,
- costs[token_index], costs[token_index], combination_limit, token_order, prefix_search,
+ costs[token_index], costs[token_index], num_fuzzy_candidates, token_order, prefix_search,
filter_ids, filter_ids_length, leaves, unique_tokens);
if(!leaves.empty()) {
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index 684c77bf..5d4fdc31 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -20,14 +20,21 @@ void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** i
std::vector<uint32_t> consolidated_ids;
while(it_start != int64map.end() && it_start->first <= end) {
+ uint32_t* values = it_start->second->uncompress();
+
for(size_t i = 0; i < it_start->second->getLength(); i++) {
- consolidated_ids.push_back(it_start->second->at(i));
+ consolidated_ids.push_back(values[i]);
}
+ delete [] values;
it_start++;
}
- std::sort(consolidated_ids.begin(), consolidated_ids.end());
+ if(consolidated_ids.size() > 50000) {
+ parasort(consolidated_ids.size(), &consolidated_ids[0], 4);
+ } else {
+ std::sort(consolidated_ids.begin(), consolidated_ids.end());
+ }
uint32_t *out = nullptr;
ids_len = ArrayUtils::or_scalar(&consolidated_ids[0], consolidated_ids.size(),