Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Disable typo tolerance for digits (#1478)

Krunal Gandhi85dd93bc1061
diff --git a/include/collection.h b/include/collection.h
index 7685d2b0..4f313729 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -278,7 +278,8 @@ private:
filter_node_t*& filter_tree_root,
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
std::vector<uint32_t>& excluded_ids,
- nlohmann::json& override_metadata) const;
+ nlohmann::json& override_metadata,
+ bool enable_typos_for_numerical_tokens=false) const;
void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type) const;
@@ -578,7 +579,8 @@ public:
const std::string& conversation_model_id = "",
std::string conversation_id = "",
const std::string& override_tags_str = "",
- const std::string& voice_query = "") const;
+ const std::string& voice_query = "",
+ bool enable_typos_for_numerical_tokens = false) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --git a/include/index.h b/include/index.h
index c767d280..3147d4a1 100644
--- a/include/index.h
+++ b/include/index.h
@@ -461,12 +461,14 @@ private:
bool resolve_override(const std::vector<std::string>& rule_tokens, bool exact_rule_match,
const std::vector<std::string>& query_tokens,
token_ordering token_order, std::set<std::string>& absorbed_tokens,
- std::string& filter_by_clause) const;
+ std::string& filter_by_clause,
+ bool enable_typos_for_numerical_tokens) const;
bool check_for_overrides(const token_ordering& token_order, const string& field_name, bool slide_window,
bool exact_rule_match, std::vector<std::string>& tokens,
std::set<std::string>& absorbed_tokens,
- std::vector<std::string>& field_absorbed_tokens) const;
+ std::vector<std::string>& field_absorbed_tokens,
+ bool enable_typos_for_numerical_tokens) const;
static void aggregate_topster(Topster* agg_topster, Topster* index_topster);
@@ -498,7 +500,8 @@ private:
int syn_orig_num_tokens,
size_t min_len_1typo,
size_t min_len_2typo,
- size_t max_candidates) const;
+ size_t max_candidates,
+ bool enable_typos_for_numerical_tokens) const;
Option<bool> search_all_candidates(const size_t num_search_fields,
const text_match_type_t match_type,
@@ -681,8 +684,8 @@ public:
facet_index_t* _get_facet_index() const;
- static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len,
- size_t min_len_1typo, size_t min_len_2typo);
+ static int get_bounded_typo_cost(const size_t max_cost, const std::string& token, const size_t token_len,
+ size_t min_len_1typo, size_t min_len_2typo, bool enable_typos_for_numerical_tokens=false);
static int64_t float_to_int64_t(float n);
@@ -706,7 +709,7 @@ public:
// Public operations
Option<bool> run_search(search_args* search_params, const std::string& collection_name,
- facet_index_type_t facet_index_type);
+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens);
Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,
const text_match_type_t match_type,
@@ -735,7 +738,8 @@ public:
const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold,
const std::string& collection_name,
const drop_tokens_param_t drop_tokens_mode,
- facet_index_type_t facet_index_type = DETECT
+ facet_index_type_t facet_index_type = DETECT,
+ bool enable_typos_for_numerical_tokens = false
) const;
void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
@@ -841,7 +845,8 @@ public:
const std::vector<std::string>& group_by_fields,
size_t group_limit, bool is_wildcard_no_filter_query,
size_t max_candidates,
- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const;
+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,
+ bool enable_typos_for_numerical_tokens) const;
void resolve_space_as_typos(std::vector<std::string>& qtokens, const std::string& field_name,
std::vector<std::vector<std::string>>& resolved_queries) const;
@@ -956,7 +961,8 @@ public:
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
- const std::string& collection_name = "") const;
+ const std::string& collection_name = "",
+ bool enable_typos_for_numerical_tokens = false) const;
void find_across_fields(const token_t& previous_token,
const std::string& previous_token_str,
@@ -1026,7 +1032,8 @@ public:
token_ordering token_order,
filter_node_t*& filter_tree_root,
std::vector<const override_t*>& matched_dynamic_overrides,
- nlohmann::json& override_metadata) const;
+ nlohmann::json& override_metadata,
+ bool enable_typos_for_numerical_tokens) const;
Option<bool> compute_sort_scores(const std::vector<sort_by>& sort_fields, const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3> field_values,
diff --git a/src/collection.cpp b/src/collection.cpp
index ba363284..80623c24 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1704,7 +1704,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const std::string& conversation_model_id,
std::string conversation_id,
const std::string& override_tags_str,
- const std::string& voice_query) const {
+ const std::string& voice_query,
+ bool enable_typos_for_numerical_tokens) const {
std::shared_lock lock(mutex);
// setup thread local vars
@@ -2238,7 +2239,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
false, stopwords_set);
process_filter_overrides(filter_overrides, q_include_tokens, token_order, filter_tree_root,
- included_ids, excluded_ids, override_metadata);
+ included_ids, excluded_ids, override_metadata, enable_typos_for_numerical_tokens);
for(size_t i = 0; i < q_include_tokens.size(); i++) {
auto& q_include_token = q_include_tokens[i];
@@ -2257,7 +2258,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
// included_ids, excluded_ids
process_filter_overrides(filter_overrides, q_include_tokens, token_order, filter_tree_root,
- included_ids, excluded_ids, override_metadata);
+ included_ids, excluded_ids, override_metadata, enable_typos_for_numerical_tokens);
for(size_t i = 0; i < q_include_tokens.size(); i++) {
auto& q_include_token = q_include_tokens[i];
@@ -2300,7 +2301,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
std::unique_ptr<search_args> search_params_guard(search_params);
- auto search_op = index->run_search(search_params, name, facet_index_type);
+ auto search_op = index->run_search(search_params, name, facet_index_type, enable_typos_for_numerical_tokens);
// filter_tree_root might be updated in Index::static_filter_query_eval.
filter_tree_root_guard.release();
@@ -3322,11 +3323,13 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filter
filter_node_t*& filter_tree_root,
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
std::vector<uint32_t>& excluded_ids,
- nlohmann::json& override_metadata) const {
+ nlohmann::json& override_metadata,
+ bool enable_typos_for_numerical_tokens) const {
std::vector<const override_t*> matched_dynamic_overrides;
index->process_filter_overrides(filter_overrides, q_include_tokens, token_order,
- filter_tree_root, matched_dynamic_overrides, override_metadata);
+ filter_tree_root, matched_dynamic_overrides, override_metadata,
+ enable_typos_for_numerical_tokens);
// we will check the dynamic overrides to see if they also have include/exclude
std::set<uint32_t> excluded_set;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 6755bf0a..dbd2b97c 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1391,6 +1391,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *VOICE_QUERY = "voice_query";
+ const char *ENABLE_TYPOS_FOR_NUMERICAL_TOKENS = "enable_typos_for_numerical_tokens";
+
// enrich params with values from embedded params
for(auto& item: embedded_params.items()) {
if(item.key() == "expires_at") {
@@ -1505,6 +1507,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
size_t max_extra_suffix = INT16_MAX;
bool enable_highlight_v1 = true;
text_match_type_t match_type = max_score;
+ bool enable_typos_for_numerical_tokens = false;
size_t remote_embedding_timeout_ms = 5000;
size_t remote_embedding_num_tries = 2;
@@ -1576,6 +1579,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
{CONVERSATION, &conversation},
{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},
{GROUP_MISSING_VALUES, &group_missing_values},
+ {ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},
};
std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {
@@ -1789,7 +1793,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
conversation_model_id,
conversation_id,
override_tags,
- voice_query);
+ voice_query,
+ enable_typos_for_numerical_tokens);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/src/index.cpp b/src/index.cpp
index 7393c1f0..aa85809d 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2320,7 +2320,7 @@ Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string
}
Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,
- facet_index_type_t facet_index_type) {
+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens) {
return search(search_params->field_query_tokens,
search_params->search_fields,
search_params->match_type,
@@ -2360,7 +2360,8 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& co
search_params->facet_sample_threshold,
collection_name,
search_params->drop_tokens_mode,
- facet_index_type
+ facet_index_type,
+ enable_typos_for_numerical_tokens
);
}
@@ -2449,7 +2450,7 @@ bool Index::static_filter_query_eval(const override_t* override,
bool Index::resolve_override(const std::vector<std::string>& rule_tokens, const bool exact_rule_match,
const std::vector<std::string>& query_tokens,
token_ordering token_order, std::set<std::string>& absorbed_tokens,
- std::string& filter_by_clause) const {
+ std::string& filter_by_clause, bool enable_typos_for_numerical_tokens) const {
bool resolved_override = false;
size_t i = 0, j = 0;
@@ -2492,7 +2493,7 @@ bool Index::resolve_override(const std::vector<std::string>& rule_tokens, const
std::vector<std::string> field_absorbed_tokens;
resolved_override &= check_for_overrides(token_order, field_name, slide_window,
exact_rule_match, matched_tokens, absorbed_tokens,
- field_absorbed_tokens);
+ field_absorbed_tokens, enable_typos_for_numerical_tokens);
if(!resolved_override) {
goto RETURN_EARLY;
@@ -2543,7 +2544,8 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
token_ordering token_order,
filter_node_t*& filter_tree_root,
std::vector<const override_t*>& matched_dynamic_overrides,
- nlohmann::json& override_metadata) const {
+ nlohmann::json& override_metadata,
+ bool enable_typos_for_numerical_tokens) const {
std::shared_lock lock(mutex);
for (auto& override : filter_overrides) {
@@ -2579,7 +2581,8 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
std::set<std::string> absorbed_tokens;
bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens,
- token_order, absorbed_tokens, filter_by_clause);
+ token_order, absorbed_tokens, filter_by_clause,
+ enable_typos_for_numerical_tokens);
if (resolved_override) {
if(override_metadata.empty()) {
@@ -2636,7 +2639,8 @@ void Index::remove_matched_tokens(std::vector<std::string>& tokens, const std::s
bool Index::check_for_overrides(const token_ordering& token_order, const string& field_name, const bool slide_window,
bool exact_rule_match, std::vector<std::string>& tokens,
std::set<std::string>& absorbed_tokens,
- std::vector<std::string>& field_absorbed_tokens) const {
+ std::vector<std::string>& field_absorbed_tokens,
+ bool enable_typos_for_numerical_tokens) const {
for(size_t window_len = tokens.size(); window_len > 0; window_len--) {
for(size_t start_index = 0; start_index+window_len-1 < tokens.size(); start_index++) {
@@ -2669,7 +2673,8 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&
search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it.value(), field_name,
nullptr, 0, {}, sort_fields, -1, 0, searched_queries, topster, groups_processed,
&result_ids, result_ids_len, field_num_results, 0, group_by_fields,
- true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4);
+ true, false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4,
+ enable_typos_for_numerical_tokens);
if(result_ids_len != 0) {
// we need to narraw onto the exact matches
@@ -2829,8 +2834,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
size_t facet_sample_percent, size_t facet_sample_threshold,
const std::string& collection_name,
const drop_tokens_param_t drop_tokens_mode,
- facet_index_type_t facet_index_type
- ) const {
+ facet_index_type_t facet_index_type,
+ bool enable_typos_for_numerical_tokens) const {
std::shared_lock lock(mutex);
auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,
@@ -3195,7 +3200,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
typo_tokens_threshold, exhaustive_search,
max_candidates, min_len_1typo, min_len_2typo,
syn_orig_num_tokens, sort_order, field_values, geopoint_indices,
- collection_name);
+ collection_name, enable_typos_for_numerical_tokens);
if (!fuzzy_search_fields_op.ok()) {
return fuzzy_search_fields_op;
}
@@ -3598,7 +3603,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<facet_info_t> facet_infos(facets.size());
compute_facet_infos(facets, facet_query, facet_query_num_typos, all_result_ids, all_result_ids_len,
group_by_fields, group_limit, is_wildcard_no_filter_query,
- max_candidates, facet_infos, facet_index_type);
+ max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);
std::vector<std::vector<facet>> facet_batches(num_threads);
std::vector<facet> value_facets;
@@ -3727,7 +3732,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
compute_facet_infos(facets, facet_query, facet_query_num_typos,
&included_ids_vec[0], included_ids_vec.size(), group_by_fields,
group_limit, is_wildcard_no_filter_query,
- max_candidates, facet_infos, facet_index_type);
+ max_candidates, facet_infos, facet_index_type, enable_typos_for_numerical_tokens);
do_facets(facets, facet_query, estimate_facets, facet_sample_percent,
facet_infos, group_limit, group_by_fields, group_missing_values, &included_ids_vec[0],
included_ids_vec.size(), max_facet_values, is_wildcard_no_filter_query,
@@ -3929,7 +3934,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
const int* sort_order,
std::array<spp::sparse_hash_map<uint32_t, int64_t, Hasher32>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices,
- const std::string& collection_name) const {
+ const std::string& collection_name,
+ bool enable_typos_for_numerical_tokens) const {
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
@@ -3943,7 +3949,8 @@ Option<bool> Index::fuzzy_search_fields(const std::vector<search_field_t>& the_f
std::vector<int> all_costs;
// This ensures that we don't end up doing a cost of 1 for a single char etc.
- int bounded_cost = get_bounded_typo_cost(2, token.length(), min_len_1typo, min_len_2typo);
+ int bounded_cost = get_bounded_typo_cost(2, token , token.length(), min_len_1typo, min_len_2typo,
+ enable_typos_for_numerical_tokens);
for(int cost = 0; cost <= bounded_cost; cost++) {
all_costs.push_back(cost);
@@ -5683,7 +5690,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
const std::vector<std::string>& group_by_fields,
const size_t group_limit, const bool is_wildcard_no_filter_query,
const size_t max_candidates,
- std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type) const {
+ std::vector<facet_info_t>& facet_infos, facet_index_type_t facet_index_type,
+ bool enable_typos_for_numerical_tokens) const {
if(all_result_ids_len == 0) {
return;
@@ -5750,7 +5758,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
facet_field, facet_field.faceted_name(),
all_result_ids, all_result_ids_len, {}, sort_fields, -1, facet_query_num_typos, searched_queries, topster,
groups_processed, &field_result_ids, field_result_ids_len, field_num_results, 0, group_by_fields,
- true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates);
+ true, false, 4, query_hashes, MAX_SCORE, true, 0, 1, false, -1, 3, 1000, max_candidates,
+ enable_typos_for_numerical_tokens);
//LOG(INFO) << "searched_queries.size: " << searched_queries.size();
@@ -6169,7 +6178,8 @@ void Index::search_field(const uint8_t & field_id,
int syn_orig_num_tokens,
size_t min_len_1typo,
size_t min_len_2typo,
- const size_t max_candidates) const {
+ const size_t max_candidates,
+ bool enable_typos_for_numerical_tokens) const {
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
@@ -6190,7 +6200,8 @@ void Index::search_field(const uint8_t & field_id,
std::vector<int> all_costs;
// This ensures that we don't end up doing a cost of 1 for a single char etc.
- int bounded_cost = get_bounded_typo_cost(max_cost, token.length(), min_len_1typo, min_len_2typo);
+ int bounded_cost = get_bounded_typo_cost(max_cost, token, token.length(), min_len_1typo, min_len_2typo,
+ enable_typos_for_numerical_tokens);
for(int cost = 0; cost <= bounded_cost; cost++) {
all_costs.push_back(cost);
@@ -6316,14 +6327,20 @@ void Index::search_field(const uint8_t & field_id,
}
}
-int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len,
- const size_t min_len_1typo, const size_t min_len_2typo) {
- if(token_len < min_len_1typo) {
+int Index::get_bounded_typo_cost(const size_t max_cost, const std::string& token, const size_t token_len,
+ const size_t min_len_1typo, const size_t min_len_2typo,
+ bool enable_typos_for_numerical_tokens) {
+
+ if(enable_typos_for_numerical_tokens && std::all_of(token.begin(), token.end(), ::isdigit)) {
+ return 0;
+ }
+
+ if (token_len < min_len_1typo) {
// typo correction is disabled for small tokens
return 0;
}
- if(token_len < min_len_2typo) {
+ if (token_len < min_len_2typo) {
// 2-typos are enabled only at token length of 7 chars
return std::min<int>(max_cost, 1);
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index e44afbf3..b84edc14 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2710,3 +2710,53 @@ TEST_F(CollectionSpecificMoreTest, HybridSearchTextMatchInfo) {
ASSERT_EQ(0, results["hits"][0]["text_match_info"]["tokens_matched"].get<size_t>());
ASSERT_EQ(0, results["hits"][1]["text_match_info"]["tokens_matched"].get<size_t>());
}
+
+TEST_F(CollectionSpecificMoreTest, DisableTyposForNumericalTokens) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ],
+ "token_separators": ["-"]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "XYZ-12345678";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "XYZ-22345678";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // with disabling typos for numerical tokens
+
+ auto res_op = coll1->search("XYZ-12345678", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", true);
+
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(1, res_op.get()["hits"].size());
+
+ res_op = coll1->search("XYZ-12345678", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 400);
+
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(2, res_op.get()["hits"].size());
+}
\ No newline at end of file