Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Exhaustive search should ignore typo and drop token thresholds.

Kishore Nallan9659d60047a1
diff --git a/include/collection.h b/include/collection.h
index 065038bf..04a7f06f 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -541,7 +541,7 @@ public:
bool pre_segmented_query=false,
bool enable_overrides=true,
const std::string& highlight_fields="",
- const size_t combination_limit = Index::COMBINATION_LIMIT) const;
+ const bool exhaustive_search = false) const;
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/include/index.h b/include/index.h
index 77f447cd..208ac025 100644
--- a/include/index.h
+++ b/include/index.h
@@ -68,7 +68,7 @@ struct search_args {
std::string default_sorting_field;
bool prioritize_exact_match;
size_t all_result_ids_len;
- size_t combination_limit;
+ bool exhaustive_search;
spp::sparse_hash_set<uint64_t> groups_processed;
std::vector<std::vector<art_leaf*>> searched_queries;
Topster* topster;
@@ -90,7 +90,7 @@ struct search_args {
const std::vector<std::string>& group_by_fields, size_t group_limit,
const std::string& default_sorting_field,
bool prioritize_exact_match,
- size_t combination_limit):
+ bool exhaustive_search):
field_query_tokens(field_query_tokens),
search_fields(search_fields), filters(filters), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
@@ -99,7 +99,7 @@ struct search_args {
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),
prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),
- combination_limit(combination_limit) {
+ exhaustive_search(exhaustive_search) {
const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
topster = new Topster(topster_size, group_limit);
@@ -233,7 +233,7 @@ private:
const token_ordering token_order = FREQUENCY, const bool prefix = false,
const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,
- const size_t combination_limit = Index::COMBINATION_LIMIT) const;
+ const bool exhaustive_search = false) const;
void search_candidates(const uint8_t & field_id,
bool field_is_array,
@@ -307,7 +307,8 @@ public:
// for limiting number of fields that can be searched on
enum {FIELD_LIMIT_NUM = 100};
- enum {COMBINATION_LIMIT = 10};
+ enum {COMBINATION_MAX_LIMIT = 10000};
+ enum {COMBINATION_MIN_LIMIT = 10};
// If the number of results found is less than this threshold, Typesense will attempt to drop the tokens
// in the query that have the least individual hits one by one until enough results are found.
diff --git a/src/collection.cpp b/src/collection.cpp
index b1017d16..98b9f6bd 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -521,7 +521,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
bool pre_segmented_query,
bool enable_overrides,
const std::string& highlight_fields,
- const size_t combination_limit) const {
+ const bool exhaustive_search) const {
std::shared_lock lock(mutex);
@@ -948,7 +948,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
per_page, page, token_order, prefixes,
drop_tokens_threshold, typo_tokens_threshold,
group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,
- combination_limit);
+ exhaustive_search);
search_args_vec.push_back(search_params);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index b11edbf3..d941234d 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -789,8 +789,6 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
}
}
- const size_t combination_limit = exhaustive_search ? 10000 : 10;
-
Option<nlohmann::json> result_op = collection->search(req_params[QUERY], search_fields, filter_str, facet_fields,
sort_fields, num_typos,
static_cast<size_t>(std::stol(req_params[PER_PAGE])),
@@ -815,7 +813,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
pre_segmented_query,
enable_overrides,
req_params[HIGHLIGHT_FIELDS],
- combination_limit
+ exhaustive_search
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/src/index.cpp b/src/index.cpp
index f483cfdf..42a4f508 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1353,7 +1353,7 @@ void Index::run_search(search_args* search_params) {
search_params->group_limit, search_params->group_by_fields,
search_params->default_sorting_field,
search_params->prioritize_exact_match,
- search_params->combination_limit);
+ search_params->exhaustive_search);
}
void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,
@@ -1879,7 +1879,7 @@ void Index::search_field(const uint8_t & field_id,
const token_ordering token_order, const bool prefix,
const size_t drop_tokens_threshold,
const size_t typo_tokens_threshold,
- const size_t combination_limit) const {
+ const bool exhaustive_search) const {
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
@@ -1891,6 +1891,8 @@ void Index::search_field(const uint8_t & field_id,
max_cost = 0;
}
+ size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;
+
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
@@ -1974,7 +1976,7 @@ void Index::search_field(const uint8_t & field_id,
if(token_to_costs[token_index].empty()) {
// we can try to drop the token and search with remaining tokens
- if(field_num_results >= drop_tokens_threshold) {
+ if(!exhaustive_search && field_num_results >= drop_tokens_threshold) {
// but if drop_tokens_threshold is breached, we are done
return ;
}
@@ -2007,7 +2009,7 @@ void Index::search_field(const uint8_t & field_id,
resume_typo_loop:
- if(field_num_results >= typo_tokens_threshold) {
+ if(!exhaustive_search && field_num_results >= typo_tokens_threshold) {
// if typo threshold is breached, we are done
return ;
}
@@ -2019,7 +2021,7 @@ void Index::search_field(const uint8_t & field_id,
if(!query_tokens.empty() && num_tokens_dropped < query_tokens.size()) {
// Drop tokens from right until (len/2 + 1), and then from left until (len/2 + 1)
- if(field_num_results >= drop_tokens_threshold) {
+ if(!exhaustive_search && field_num_results >= drop_tokens_threshold) {
// if drop_tokens_threshold is breached, we are done
return ;
}
@@ -2051,8 +2053,6 @@ void Index::search_field(const uint8_t & field_id,
}
int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) {
- int bounded_cost = max_cost;
-
if(token_len < 4) {
// typo correction is disabled for small tokens
return 0;
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 3f7e291a..1691cf7f 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1093,3 +1093,74 @@ TEST_F(CollectionSpecificTest, CustomSymbolsForIndexing) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionSpecificTest, TypoCorrectionWithFaceting) {
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("brand", field_types::STRING, true),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection(
+ "coll1", 1, fields, "points", 0, "", {}, {}
+ ).get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Salt";
+ doc1["brand"] = "Salpices";
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Turmeric";
+ doc2["brand"] = "Salpices";
+ doc2["points"] = 100;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Turmeric";
+ doc3["brand"] = "Salpices";
+ doc3["points"] = 100;
+
+ nlohmann::json doc4;
+ doc4["id"] = "3";
+ doc4["name"] = "Tomato";
+ doc4["brand"] = "Saltato";
+ doc4["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());
+
+ auto results = coll1->search("salt", {"name", "brand"},
+ "", {"brand"}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true).get();
+
+ ASSERT_EQ(3, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
+
+ results = coll1->search("salt", {"name", "brand"},
+ "brand: Salpices", {"brand"}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", true).get();
+
+ ASSERT_EQ(3, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
+
+ // without exhaustive search, count be just 1 for non-filtered query
+
+ results = coll1->search("salt", {"name", "brand"},
+ "", {"brand"}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000,
+ true, false, true, "", false).get();
+
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
+}