Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Exhaustive search should ignore typo and drop token thresholds.
Kishore Nallan9659d60047a1
diff --git a/include/collection.h b/include/collection.hindex 065038bf..04a7f06f 100644--- a/include/collection.h+++ b/include/collection.h@@ -541,7 +541,7 @@ public:bool pre_segmented_query=false,bool enable_overrides=true,const std::string& highlight_fields="",- const size_t combination_limit = Index::COMBINATION_LIMIT) const;+ const bool exhaustive_search = false) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/include/index.h b/include/index.hindex 77f447cd..208ac025 100644--- a/include/index.h+++ b/include/index.h@@ -68,7 +68,7 @@ struct search_args {std::string default_sorting_field;bool prioritize_exact_match;size_t all_result_ids_len;- size_t combination_limit;+ bool exhaustive_search;spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;Topster* topster;@@ -90,7 +90,7 @@ struct search_args {const std::vector<std::string>& group_by_fields, size_t group_limit,const std::string& default_sorting_field,bool prioritize_exact_match,- size_t combination_limit):+ bool exhaustive_search):field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -99,7 +99,7 @@ struct search_args {drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),- combination_limit(combination_limit) {+ exhaustive_search(exhaustive_search) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -233,7 +233,7 @@ private:const token_ordering token_order = FREQUENCY, const bool prefix = false,const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,- const size_t combination_limit = Index::COMBINATION_LIMIT) const;+ const bool exhaustive_search = false) const;void search_candidates(const uint8_t & field_id,bool field_is_array,@@ -307,7 +307,8 @@ public:// for limiting number of fields that can be searched onenum {FIELD_LIMIT_NUM = 100};- enum {COMBINATION_LIMIT = 10};+ enum {COMBINATION_MAX_LIMIT = 10000};+ enum {COMBINATION_MIN_LIMIT = 10};// If the number of results found is less than this threshold, Typesense will attempt to drop the tokens// in the query that have the least individual hits one by one until enough results are found.diff --git a/src/collection.cpp b/src/collection.cppindex b1017d16..98b9f6bd 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -521,7 +521,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::bool pre_segmented_query,bool enable_overrides,const std::string& highlight_fields,- const size_t combination_limit) const {+ const bool exhaustive_search) const {std::shared_lock lock(mutex);@@ -948,7 +948,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::per_page, page, token_order, prefixes,drop_tokens_threshold, typo_tokens_threshold,group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,- combination_limit);+ exhaustive_search);search_args_vec.push_back(search_params);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex b11edbf3..d941234d 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -789,8 +789,6 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}}- const size_t combination_limit = exhaustive_search ? 10000 : 10;-Option<nlohmann::json> result_op = collection->search(req_params[QUERY], search_fields, filter_str, facet_fields,sort_fields, num_typos,static_cast<size_t>(std::stol(req_params[PER_PAGE])),@@ -815,7 +813,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& repre_segmented_query,enable_overrides,req_params[HIGHLIGHT_FIELDS],- combination_limit+ exhaustive_search);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex f483cfdf..42a4f508 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1353,7 +1353,7 @@ void Index::run_search(search_args* search_params) {search_params->group_limit, search_params->group_by_fields,search_params->default_sorting_field,search_params->prioritize_exact_match,- search_params->combination_limit);+ search_params->exhaustive_search);}void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,@@ -1879,7 +1879,7 @@ void Index::search_field(const uint8_t & field_id,const token_ordering token_order, const bool prefix,const size_t drop_tokens_threshold,const size_t typo_tokens_threshold,- const size_t combination_limit) const {+ const bool exhaustive_search) const {// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens@@ -1891,6 +1891,8 @@ void Index::search_field(const uint8_t & field_id,max_cost = 0;}+ size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;+// To prevent us from doing ART search repeatedly as we iterate through possible correctionsspp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;@@ -1974,7 +1976,7 @@ void Index::search_field(const uint8_t & field_id,if(token_to_costs[token_index].empty()) {// we can try to drop the token and search with remaining tokens- if(field_num_results >= drop_tokens_threshold) {+ if(!exhaustive_search && field_num_results >= drop_tokens_threshold) {// but if drop_tokens_threshold is breached, we are donereturn ;}@@ -2007,7 +2009,7 @@ void Index::search_field(const uint8_t & field_id,resume_typo_loop:- if(field_num_results >= typo_tokens_threshold) {+ if(!exhaustive_search && field_num_results >= typo_tokens_threshold) {// if typo threshold is breached, we are donereturn ;}@@ -2019,7 +2021,7 @@ void Index::search_field(const uint8_t & field_id,if(!query_tokens.empty() && num_tokens_dropped < query_tokens.size()) {// Drop tokens from right until (len/2 + 1), and then from left until (len/2 + 1)- if(field_num_results >= drop_tokens_threshold) {+ if(!exhaustive_search && field_num_results >= drop_tokens_threshold) {// if drop_tokens_threshold is breached, we are donereturn ;}@@ -2051,8 +2053,6 @@ void Index::search_field(const uint8_t & field_id,}int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) {- int bounded_cost = max_cost;-if(token_len < 4) {// typo correction is disabled for small tokensreturn 0;diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 3f7e291a..1691cf7f 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1093,3 +1093,74 @@ TEST_F(CollectionSpecificTest, CustomSymbolsForIndexing) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionSpecificTest, TypoCorrectionWithFaceting) {+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("brand", field_types::STRING, true),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection(+ "coll1", 1, fields, "points", 0, "", {}, {}+ ).get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Salt";+ doc1["brand"] = "Salpices";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "Turmeric";+ doc2["brand"] = "Salpices";+ doc2["points"] = 100;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Turmeric";+ doc3["brand"] = "Salpices";+ doc3["points"] = 100;++ nlohmann::json doc4;+ doc4["id"] = "3";+ doc4["name"] = "Tomato";+ doc4["brand"] = "Saltato";+ doc4["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());+ ASSERT_TRUE(coll1->add(doc4.dump()).ok());++ auto results = coll1->search("salt", {"name", "brand"},+ "", {"brand"}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true).get();++ ASSERT_EQ(3, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());++ results = coll1->search("salt", {"name", "brand"},+ "brand: Salpices", {"brand"}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true).get();++ ASSERT_EQ(3, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());++ // without exhaustive search, count be just 1 for non-filtered query++ results = coll1->search("salt", {"name", "brand"},+ "", {"brand"}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", false).get();++ ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());+}