Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Configurable 1-typo and 2-typo min token lengths.
Kishore Nallan1fa2193c5090
diff --git a/include/collection.h b/include/collection.hindex 2c2bdd06..4819a438 100644--- a/include/collection.h+++ b/include/collection.h@@ -384,7 +384,9 @@ public:bool enable_overrides=true,const std::string& highlight_fields="",const bool exhaustive_search = false,- size_t search_stop_millis = 6000*1000) const;+ size_t search_stop_millis = 6000*1000,+ size_t min_len_1typo = 4,+ size_t min_len_2typo = 7) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/include/index.h b/include/index.hindex 1c1d02a1..ba53aac7 100644--- a/include/index.h+++ b/include/index.h@@ -268,6 +268,8 @@ struct search_args {size_t concurrency;const std::vector<const override_t*>& filter_overrides;size_t search_cutoff_ms;+ size_t min_len_1typo;+ size_t min_len_2typo;spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;@@ -290,7 +292,9 @@ struct search_args {bool exhaustive_search,size_t concurrency,const std::vector<const override_t*>& dynamic_overrides,- size_t search_cutoff_ms):+ size_t search_cutoff_ms,+ size_t min_len_1typo,+ size_t min_len_2typo):field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -300,7 +304,8 @@ struct search_args {group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),exhaustive_search(exhaustive_search), concurrency(concurrency),- filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms) {+ filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms),+ min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -476,10 +481,12 @@ private:bool prioritize_exact_match,size_t concurrency,std::set<uint64>& query_hashes,- token_ordering token_order = FREQUENCY, const bool prefix = false,- size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,- size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,- bool exhaustive_search = false) const;+ token_ordering token_order, const bool prefix,+ size_t drop_tokens_threshold,+ size_t typo_tokens_threshold,+ bool exhaustive_search,+ size_t min_len_1typo,+ size_t min_len_2typo) const;void search_candidates(const uint8_t & field_id,bool field_is_array,@@ -616,7 +623,8 @@ public:const spp::sparse_hash_map<std::string, num_tree_t*>& _get_numerical_index() const;- static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len);+ static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len,+ size_t min_len_1typo, size_t min_len_2typo);static int64_t float_to_in64_t(float n);@@ -662,7 +670,9 @@ public:bool prioritize_exact_match,bool exhaustive_search,size_t concurrency,- size_t search_cutoff_ms) const;+ size_t search_cutoff_ms,+ size_t min_len_1typo,+ size_t min_len_2typo) const;Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);diff --git a/src/collection.cpp b/src/collection.cppindex 7c28f7d1..e9de28a5 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -494,7 +494,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sbool enable_overrides,const std::string& highlight_fields,const bool exhaustive_search,- const size_t search_stop_millis) const {+ const size_t search_stop_millis,+ const size_t min_len_1typo,+ const size_t min_len_2typo) const {std::shared_lock lock(mutex);@@ -915,7 +917,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sdrop_tokens_threshold, typo_tokens_threshold,group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,exhaustive_search, 4, filter_overrides,- search_stop_millis);+ search_stop_millis,+ min_len_1typo, min_len_2typo);index->run_search(search_params);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 18edba3b..a4b039b6 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -491,6 +491,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reauto begin = std::chrono::high_resolution_clock::now();const char *NUM_TYPOS = "num_typos";+ const char *MIN_LEN_1TYPO = "min_len_1typo";+ const char *MIN_LEN_2TYPO = "min_len_2typo";+const char *PREFIX = "prefix";const char *DROP_TOKENS_THRESHOLD = "drop_tokens_threshold";const char *TYPO_TOKENS_THRESHOLD = "typo_tokens_threshold";@@ -541,6 +544,18 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rereq_params[NUM_TYPOS] = "2";}+ if(req_params.count(MIN_LEN_1TYPO) == 0) {+ req_params[MIN_LEN_1TYPO] = "4";+ } else if(!StringUtils::is_uint32_t(req_params[MIN_LEN_1TYPO])) {+ return Option<bool>(400, "Parameter `" + std::string(MIN_LEN_1TYPO) + "` must be an unsigned integer.");+ }++ if(req_params.count(MIN_LEN_2TYPO) == 0) {+ req_params[MIN_LEN_2TYPO] = "7";+ } else if(!StringUtils::is_uint32_t(req_params[MIN_LEN_2TYPO])) {+ return Option<bool>(400, "Parameter `" + std::string(MIN_LEN_2TYPO) + "` must be an unsigned integer.");+ }+if(req_params.count(PREFIX) == 0) {req_params[PREFIX] = "true";}diff --git a/src/index.cpp b/src/index.cppindex ba10642e..b603845f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1644,7 +1644,9 @@ void Index::run_search(search_args* search_params) {search_params->prioritize_exact_match,search_params->exhaustive_search,search_params->concurrency,- search_params->search_cutoff_ms);+ search_params->search_cutoff_ms,+ search_params->min_len_1typo,+ search_params->min_len_2typo);}void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,@@ -1982,7 +1984,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&search_field(0, window_tokens, search_tokens, nullptr, 0, num_toks_dropped, field_name,nullptr, 0, {}, facets, {}, 2, searched_queries, topster, groups_processed,&result_ids, result_ids_len, field_num_results, 0, group_by_fields,- false, 4, query_hashes, token_order, false, 0, 1, false);+ false, 4, query_hashes, token_order, false, 0, 1, false, 3, 7);if(result_ids_len != 0) {// remove window_tokens from `tokens`@@ -2034,7 +2036,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,bool prioritize_exact_match,const bool exhaustive_search,const size_t concurrency,- const size_t search_cutoff_ms) const {+ const size_t search_cutoff_ms,+ size_t min_len_1typo,+ size_t min_len_2typo) const {begin = std::chrono::high_resolution_clock::now();search_stop_ms = search_cutoff_ms;@@ -2170,7 +2174,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,query_hashes, token_order, field_prefix,- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,+ min_len_1typo, min_len_2typo);bool syn_wildcard_filter_init_done = false;@@ -2202,7 +2207,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,query_hashes, token_order, field_prefix,- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,+ min_len_1typo, min_len_2typo);}}@@ -2629,7 +2635,9 @@ void Index::search_field(const uint8_t & field_id,const token_ordering token_order, const bool prefix,const size_t drop_tokens_threshold,const size_t typo_tokens_threshold,- const bool exhaustive_search) const {+ const bool exhaustive_search,+ size_t min_len_1typo,+ size_t min_len_2typo) const {// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens@@ -2651,7 +2659,7 @@ void Index::search_field(const uint8_t & field_id,std::vector<int> all_costs;// This ensures that we don't end up doing a cost of 1 for a single char etc.- int bounded_cost = get_bounded_typo_cost(max_cost, token.length());+ int bounded_cost = get_bounded_typo_cost(max_cost, token.length(), min_len_1typo, min_len_2typo);for(int cost = 0; cost <= bounded_cost; cost++) {all_costs.push_back(cost);@@ -2804,17 +2812,18 @@ void Index::search_field(const uint8_t & field_id,all_result_ids_len, field_num_results, group_limit, group_by_fields,prioritize_exact_match, concurrency, query_hashes,token_order, prefix, drop_tokens_threshold, typo_tokens_threshold,- exhaustive_search);+ exhaustive_search, min_len_1typo, min_len_2typo);}}-int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) {- if(token_len < 4) {+int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len,+ const size_t min_len_1typo, const size_t min_len_2typo) {+ if(token_len < min_len_1typo) {// typo correction is disabled for small tokensreturn 0;}- if(token_len < 7) {+ if(token_len < min_len_2typo) {// 2-typos are enabled only at token length of 7 charsreturn std::min<int>(max_cost, 1);}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 151bd3cf..ccc01647 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1577,3 +1577,67 @@ TEST_F(CollectionSpecificTest, VerbatimMatchShouldConsiderTokensMatchedAcrossAllcollectionManager.drop_collection("coll1");}++TEST_F(CollectionSpecificTest, CustomNumTyposConfiguration) {+ // dropped tokens on a single field cannot be deemed as verbatim match++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("brand", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Hamburger";+ doc1["brand"] = "Burger and King";+ doc1["points"] = 10;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ // by default a typo on 3 char tokens are ignored (min 4 length is needed)++ auto results = coll1->search("asd", {"brand"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false},+ 2, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {1},+ 1000, true, false, true, "", false, 60000*100).get();++ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search("asd", {"brand"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false},+ 2, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {1},+ 1000, true, false, true, "", false, 60000*100, 3, 7).get();++ ASSERT_EQ(1, results["hits"].size());++ // 2 typos are not tolerated by default on 6-len word++ results = coll1->search("bixger", {"brand"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false},+ 2, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {1},+ 1000, true, false, true, "", false, 60000*100).get();++ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search("bixger", {"brand"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false},+ 2, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {1},+ 1000, true, false, true, "", false, 60000*100, 3, 6).get();++ ASSERT_EQ(1, results["hits"].size());++ collectionManager.drop_collection("coll1");+}