Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Disable typo tolerance for digits (#1478)
Krunal Gandhi85dd93bc1061
diff --git a/include/collection.h b/include/collection.hindex 7685d2b0..4f313729 100644--- a/include/collection.h+++ b/include/collection.h@@ -278,7 +278,8 @@ private:filter_node_t*& filter_tree_root,std::vector<std::pair<uint32_t, uint32_t>>& included_ids,std::vector<uint32_t>& excluded_ids,- nlohmann::json& override_metadata) const;+ nlohmann::json& override_metadata,+ bool enable_typos_for_numerical_tokens=false) const;void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type) const;@@ -578,7 +579,8 @@ public:const std::string& conversation_model_id = "",std::string conversation_id = "",const std::string& override_tags_str = "",- const std::string& voice_query = "") const;+ const std::string& voice_query = "",+ bool enable_typos_for_numerical_tokens = false) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/index.h b/include/index.hindex c767d280..3147d4a1 100644--- a/include/index.h+++ b/include/index.h@@ -461,12 +461,14 @@ private:bool resolve_override(const std::vector<std::string>& rule_tokens, bool exact_rule_match,const std::vector<std::string>& query_tokens,token_ordering token_order, std::set<std::string>& absorbed_tokens,- std::string& filter_by_clause) const;+ std::string& filter_by_clause,+ bool enable_typos_for_numerical_tokens) const;bool check_for_overrides(const token_ordering& token_order, const string& field_name, bool slide_window,bool exact_rule_match, std::vector<std::string>& tokens,std::set<std::string>& absorbed_tokens,- std::vector<std::string>& field_absorbed_tokens) const;+ std::vector<std::string>& field_absorbed_tokens,+ bool enable_typos_for_numerical_tokens) const;static void aggregate_topster(Topster* agg_topster, Topster* index_topster);@@ -498,7 +500,8 @@ private:int syn_orig_num_tokens,size_t min_len_1typo,size_t min_len_2typo,- size_t max_candidates) const;+ size_t max_candidates,+ bool enable_typos_for_numerical_tokens) const;Option<bool> search_all_candidates(const size_t num_search_fields,const text_match_type_t match_type,@@ -681,8 +684,8 @@ public:facet_index_t* _get_facet_index() const;- static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len,- size_t min_len_1typo, size_t min_len_2typo);+ static int get_bounded_typo_cost(const size_t max_cost, const std::string& token, const size_t token_len,+ size_t min_len_1typo, size_t min_len_2typo, bool enable_typos_for_numerical_tokens=false);static int64_t float_to_int64_t(float n);@@ -706,7 +709,7 @@ public:// Public operationsOption<bool> run_search(search_args* search_params, const std::string& collection_name,- facet_index_type_t facet_index_type);+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens);Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,const text_match_type_t match_type,@@ -735,7 +738,8 @@ public:const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold,const std::string& collection_name,const drop_tokens_param_t drop_tokens_mode,- facet_index_type_t facet_index_type = DETECT+ facet_index_type_t facet_index_type = DETECT,+ bool enable_typos_for_numerical_tokens = false) const;void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,@@ -841,7 +845,8 @@ public:const std::vector<std::string>& group_by_fields,size_t group_limit, bool is_wildcard_no_filter_query,size_t max_candidates,- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const;+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,+ bool enable_typos_for_numerical_tokens) const;void resolve_space_as_typos(std::vector<std::string>& qtokens, const std::string& field_name,std::vector<std::vector<std::string>>& resolved_queries) const;@@ -956,7 +961,8 @@ public:const int* sort_order,std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,- const std::string& collection_name = "") const;+ const std::string& collection_name = "",+ bool enable_typos_for_numerical_tokens = false) const;void find_across_fields(const token_t& previous_token,const std::string& previous_token_str,@@ -1026,7 +1032,8 @@ public:token_ordering token_order,filter_node_t*& filter_tree_root,std::vector<const override_t*>& matched_dynamic_overrides,- nlohmann::json& override_metadata) const;+ nlohmann::json& override_metadata,+ bool enable_typos_for_numerical_tokens) const;Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,diff --git a/src/collection.cpp b/src/collection.cppindex ba363284..80623c24 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1704,7 +1704,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const std::string& conversation_model_id,std::string conversation_id,const std::string& override_tags_str,- const std::string& voice_query) const {+ const std::string& voice_query,+ bool enable_typos_for_numerical_tokens) const {std::shared_lock lock(mutex);// setup thread local vars@@ -2238,7 +2239,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,false, stopwords_set);process_filter_overrides(filter_overrides, q_include_tokens, token_order, filter_tree_root,- included_ids, excluded_ids, override_metadata);+ included_ids, excluded_ids, override_metadata, enable_typos_for_numerical_tokens);for(size_t i = 0; i < q_include_tokens.size(); i++) {auto& q_include_token = q_include_tokens[i];@@ -2257,7 +2258,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,// included_ids, excluded_idsprocess_filter_overrides(filter_overrides, q_include_tokens, token_order, filter_tree_root,- included_ids, excluded_ids, override_metadata);+ included_ids, excluded_ids, override_metadata, enable_typos_for_numerical_tokens);for(size_t i = 0; i < q_include_tokens.size(); i++) {auto& q_include_token = q_include_tokens[i];@@ -2300,7 +2301,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,std::unique_ptr<search_args> search_params_guard(search_params);- auto search_op = index->run_search(search_params, name, facet_index_type);+ auto search_op = index->run_search(search_params, name, facet_index_type, enable_typos_for_numerical_tokens);// filter_tree_root might be updated in Index::static_filter_query_eval.filter_tree_root_guard.release();@@ -3322,11 +3323,13 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filterfilter_node_t*& filter_tree_root,std::vector<std::pair<uint32_t, uint32_t>>& included_ids,std::vector<uint32_t>& excluded_ids,- nlohmann::json& override_metadata) const {+ nlohmann::json& override_metadata,+ bool enable_typos_for_numerical_tokens) const {std::vector<const override_t*> matched_dynamic_overrides;index->process_filter_overrides(filter_overrides, q_include_tokens, token_order,- filter_tree_root, matched_dynamic_overrides, override_metadata);+ filter_tree_root, matched_dynamic_overrides, override_metadata,+ enable_typos_for_numerical_tokens);// we will check the dynamic overrides to see if they also have include/excludestd::set<uint32_t> excluded_set;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 6755bf0a..dbd2b97c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1391,6 +1391,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *VOICE_QUERY = "voice_query";+ const char *ENABLE_TYPOS_FOR_NUMERICAL_TOKENS = "enable_typos_for_numerical_tokens";+// enrich params with values from embedded paramsfor(auto& item: embedded_params.items()) {if(item.key() == "expires_at") {@@ -1505,6 +1507,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& resize_t max_extra_suffix = INT16_MAX;bool enable_highlight_v1 = true;text_match_type_t match_type = max_score;+ bool enable_typos_for_numerical_tokens = false;size_t remote_embedding_timeout_ms = 5000;size_t remote_embedding_num_tries = 2;@@ -1576,6 +1579,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{CONVERSATION, &conversation},{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},{GROUP_MISSING_VALUES, &group_missing_values},+ {ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},};std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {@@ -1789,7 +1793,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconversation_model_id,conversation_id,override_tags,- voice_query);+ voice_query,+ enable_typos_for_numerical_tokens);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/src/index.cpp b/src/index.cppindex 7393c1f0..aa85809d 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2320,7 +2320,7 @@ Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string}Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,- facet_index_type_t facet_index_type) {+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens) {return search(search_params->field_query_tokens,search_params->search_fields,search_params->match_type,@@ -2360,7 +2360,8 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->facet_sample_threshold,collection_name,search_params->drop_tokens_mode,- facet_index_type+ facet_index_type,+ enable_typos_for_numerical_tokens);}@@ -2449,7 +2450,7 @@ bool Index::static_filter_query_eval(const override_t* override,bool Index::resolve_override(const std::vector<std::string>& rule_tokens, const bool exact_rule_match,const std::vector<std::string>& query_tokens,token_ordering token_order, std::set<std::string>& absorbed_tokens,- std::string& filter_by_clause) const {+ std::string& filter_by_clause, bool enable_typos_for_numerical_tokens) const {bool resolved_override = false;size_t i = 0, j = 0;@@ -2492,7 +2493,7 @@ bool Index::resolve_override(const std::vector<std::string>& rule_tokens, conststd::vector<std::string> field_absorbed_tokens;resolved_override &= check_for_overrides(token_order, field_name, slide_window,exact_rule_match, matched_tokens, absorbed_tokens,- field_absorbed_tokens);+ field_absorbed_tokens, enable_typos_for_numerical_tokens);if(!resolved_override) {goto RETURN_EARLY;@@ -2543,7 +2544,8 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filtetoken_ordering token_order,filter_node_t*& filter_tree_root,std::vector<const override_t*>& matched_dynamic_overrides,- nlohmann::json& override_metadata) const {+ nlohmann::json& override_metadata,+ bool enable_typos_for_numerical_tokens) const {std::shared_lock lock(mutex);for (auto& override : filter_overrides) {@@ -2579,7 +2581,8 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filtestd::set<std::string> absorbed_tokens;bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens,- token_order, absorbed_tokens, filter_by_clause);+ token_order, absorbed_tokens, filter_by_clause,+ enable_typos_for_numerical_tokens);if (resolved_override) {if(override_metadata.empty()) {@@ -2636,7 +2639,8 @@ void Index::remove_matched_tokens(std::vector<std::string>& tokens, const std::sbool Index::check_for_overrides(const token_ordering& token_order, const string& field_name, const bool slide_window,bool exact_rule_match, std::vector<std::string>& tokens,std::set<std::string>& absorbed_tokens,- std::vector<std::string>& field_absorbed_tokens) const {+ std::vector<std::string>& field_absorbed_tokens,+ bool enable_typos_for_numerical_tokens) const {for(size_t window_len = tokens.size(); window_len > 0; window_len--) {for(size_t start_index = 0; start_index+window_len-1 < tokens.size(); start_index++) {@@ -2669,7 +2673,8 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it.value(), field_name,nullptr, 0, {}, sort_fields, -1, 0, searched_queries, topster, groups_processed,&result_ids, result_ids_len, field_num_results, 0, group_by_fields,- true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4);+ true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4,+ enable_typos_for_numerical_tokens);if(result_ids_len != 0) {// we need to narraw onto the exact matches@@ -2829,8 +2834,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conssize_t facet_sample_percent, size_t facet_sample_threshold,const std::string& collection_name,const drop_tokens_param_t drop_tokens_mode,- facet_index_type_t facet_index_type- ) const {+ facet_index_type_t facet_index_type,+ bool enable_typos_for_numerical_tokens) const {std::shared_lock lock(mutex);auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,@@ -3195,7 +3200,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, constypo_tokens_threshold, exhaustive_search,max_candidates, min_len_1typo, min_len_2typo,syn_orig_num_tokens, sort_order, field_values, geopoint_indices,- collection_name);+ collection_name, enable_typos_for_numerical_tokens);if (!fuzzy_search_fields_op.ok()) {return fuzzy_search_fields_op;}@@ -3598,7 +3603,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<facet_info_t> facet_infos(facets.size());compute_facet_infos(facets, facet_query, facet_query_num_typos, all_result_ids, all_result_ids_len,group_by_fields, group_limit, is_wildcard_no_filter_query,- max_candidates, facet_infos, facet_index_type);+ max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);std::vector<std::vector<facet>> facet_batches(num_threads);std::vector<facet> value_facets;@@ -3727,7 +3732,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, conscompute_facet_infos(facets, facet_query, facet_query_num_typos,&included_ids_vec[0], included_ids_vec.size(), group_by_fields,group_limit, is_wildcard_no_filter_query,- max_candidates, facet_infos, facet_index_type);+ max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);do_facets(facets, facet_query, estimate_facets, facet_sample_percent,facet_infos, group_limit, group_by_fields, group_missing_values, &included_ids_vec[0],included_ids_vec.size(), max_facet_values, is_wildcard_no_filter_query,@@ -3929,7 +3934,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fconst int* sort_order,std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,const std::vector<size_t>& geopoint_indices,- const std::string& collection_name) const {+ const std::string& collection_name,+ bool enable_typos_for_numerical_tokens) const {// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens@@ -3943,7 +3949,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fstd::vector<int> all_costs;// This ensures that we don't end up doing a cost of 1 for a single char etc.- int bounded_cost = get_bounded_typo_cost(2, token.length(), min_len_1typo, min_len_2typo);+ int bounded_cost = get_bounded_typo_cost(2, token , token.length(), min_len_1typo, min_len_2typo,+ enable_typos_for_numerical_tokens);for(int cost = 0; cost <= bounded_cost; cost++) {all_costs.push_back(cost);@@ -5683,7 +5690,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&const std::vector<std::string>& group_by_fields,const size_t group_limit, const bool is_wildcard_no_filter_query,const size_t max_candidates,- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const {+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,+ bool enable_typos_for_numerical_tokens) const {if(all_result_ids_len == 0) {return;@@ -5750,7 +5758,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&facet_field, facet_field.faceted_name(),all_result_ids, all_result_ids_len, {}, sort_fields, -1, facet_query_num_typos, searched_queries, topster,groups_processed, &field_result_ids, field_result_ids_len, field_num_results, 0, group_by_fields,- true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates);+ true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates,+ enable_typos_for_numerical_tokens);//LOG(INFO) << "searched_queries.size: " << searched_queries.size();@@ -6169,7 +6178,8 @@ void Index::search_field(const uint8_t & field_id,int syn_orig_num_tokens,size_t min_len_1typo,size_t min_len_2typo,- const size_t max_candidates) const {+ const size_t max_candidates,+ bool enable_typos_for_numerical_tokens) const {// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens@@ -6190,7 +6200,8 @@ void Index::search_field(const uint8_t & field_id,std::vector<int> all_costs;// This ensures that we don't end up doing a cost of 1 for a single char etc.- int bounded_cost = get_bounded_typo_cost(max_cost, token.length(), min_len_1typo, min_len_2typo);+ int bounded_cost = get_bounded_typo_cost(max_cost, token, token.length(), min_len_1typo, min_len_2typo,+ enable_typos_for_numerical_tokens);for(int cost = 0; cost <= bounded_cost; cost++) {all_costs.push_back(cost);@@ -6316,14 +6327,20 @@ void Index::search_field(const uint8_t & field_id,}}-int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len,- const size_t min_len_1typo, const size_t min_len_2typo) {- if(token_len < min_len_1typo) {+int Index::get_bounded_typo_cost(const size_t max_cost, const std::string& token, const size_t token_len,+ const size_t min_len_1typo, const size_t min_len_2typo,+ bool enable_typos_for_numerical_tokens) {++ if(enable_typos_for_numerical_tokens && std::all_of(token.begin(), token.end(), ::isdigit)) {+ return 0;+ }++ if (token_len < min_len_1typo) {// typo correction is disabled for small tokensreturn 0;}- if(token_len < min_len_2typo) {+ if (token_len < min_len_2typo) {// 2-typos are enabled only at token length of 7 charsreturn std::min<int>(max_cost, 1);}diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex e44afbf3..b84edc14 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2710,3 +2710,53 @@ TEST_F(CollectionSpecificMoreTest, HybridSearchTextMatchInfo) {ASSERT_EQ(0, results["hits"][0]["text_match_info"]["tokens_matched"].get<size_t>());ASSERT_EQ(0, results["hits"][1]["text_match_info"]["tokens_matched"].get<size_t>());}++TEST_F(CollectionSpecificMoreTest, DisableTyposForNumericalTokens) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"}+ ],+ "token_separators": ["-"]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["title"] = "XYZ-12345678";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "XYZ-22345678";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // with disabling typos for numerical tokens++ auto res_op = coll1->search("XYZ-12345678", {"title"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", true);++ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(1, res_op.get()["hits"].size());++ res_op = coll1->search("XYZ-12345678", {"title"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 400);++ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(2, res_op.get()["hits"].size());+}\ No newline at end of file