Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Configurable 1-typo and 2-typo min token lengths.

Kishore Nallan1fa2193c5090
diff --git a/include/collection.h b/include/collection.h
index 2c2bdd06..4819a438 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -384,7 +384,9 @@ public:
bool enable_overrides=true,
const std::string& highlight_fields="",
const bool exhaustive_search = false,
- size_t search_stop_millis = 6000*1000) const;
+ size_t search_stop_millis = 6000*1000,
+ size_t min_len_1typo = 4,
+ size_t min_len_2typo = 7) const;
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/include/index.h b/include/index.h
index 1c1d02a1..ba53aac7 100644
--- a/include/index.h
+++ b/include/index.h
@@ -268,6 +268,8 @@ struct search_args {
size_t concurrency;
const std::vector<const override_t*>& filter_overrides;
size_t search_cutoff_ms;
+ size_t min_len_1typo;
+ size_t min_len_2typo;
spp::sparse_hash_set<uint64_t> groups_processed;
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -290,7 +292,9 @@ struct search_args {
bool exhaustive_search,
size_t concurrency,
const std::vector<const override_t*>& dynamic_overrides,
- size_t search_cutoff_ms):
+ size_t search_cutoff_ms,
+ size_t min_len_1typo,
+ size_t min_len_2typo):
field_query_tokens(field_query_tokens),
search_fields(search_fields), filters(filters), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
@@ -300,7 +304,8 @@ struct search_args {
group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),
prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),
exhaustive_search(exhaustive_search), concurrency(concurrency),
- filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms) {
+ filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms),
+ min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo) {
const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
topster = new Topster(topster_size, group_limit);
@@ -476,10 +481,12 @@ private:
bool prioritize_exact_match,
size_t concurrency,
std::set<uint64>& query_hashes,
- token_ordering token_order = FREQUENCY, const bool prefix = false,
- size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
- size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,
- bool exhaustive_search = false) const;
+ token_ordering token_order, const bool prefix,
+ size_t drop_tokens_threshold,
+ size_t typo_tokens_threshold,
+ bool exhaustive_search,
+ size_t min_len_1typo,
+ size_t min_len_2typo) const;
void search_candidates(const uint8_t & field_id,
bool field_is_array,
@@ -616,7 +623,8 @@ public:
const spp::sparse_hash_map<std::string, num_tree_t*>& _get_numerical_index() const;
- static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len);
+ static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len,
+ size_t min_len_1typo, size_t min_len_2typo);
static int64_t float_to_in64_t(float n);
@@ -662,7 +670,9 @@ public:
bool prioritize_exact_match,
bool exhaustive_search,
size_t concurrency,
- size_t search_cutoff_ms) const;
+ size_t search_cutoff_ms,
+ size_t min_len_1typo,
+ size_t min_len_2typo) const;
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);
diff --git a/src/collection.cpp b/src/collection.cpp
index 7c28f7d1..e9de28a5 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -494,7 +494,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
bool enable_overrides,
const std::string& highlight_fields,
const bool exhaustive_search,
- const size_t search_stop_millis) const {
+ const size_t search_stop_millis,
+ const size_t min_len_1typo,
+ const size_t min_len_2typo) const {
std::shared_lock lock(mutex);
@@ -915,7 +917,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
drop_tokens_threshold, typo_tokens_threshold,
group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,
exhaustive_search, 4, filter_overrides,
- search_stop_millis);
+ search_stop_millis,
+ min_len_1typo, min_len_2typo);
index->run_search(search_params);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 18edba3b..a4b039b6 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -491,6 +491,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
auto begin = std::chrono::high_resolution_clock::now();
const char *NUM_TYPOS = "num_typos";
+ const char *MIN_LEN_1TYPO = "min_len_1typo";
+ const char *MIN_LEN_2TYPO = "min_len_2typo";
+
const char *PREFIX = "prefix";
const char *DROP_TOKENS_THRESHOLD = "drop_tokens_threshold";
const char *TYPO_TOKENS_THRESHOLD = "typo_tokens_threshold";
@@ -541,6 +544,18 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
req_params[NUM_TYPOS] = "2";
}
+ if(req_params.count(MIN_LEN_1TYPO) == 0) {
+ req_params[MIN_LEN_1TYPO] = "4";
+ } else if(!StringUtils::is_uint32_t(req_params[MIN_LEN_1TYPO])) {
+ return Option<bool>(400, "Parameter `" + std::string(MIN_LEN_1TYPO) + "` must be an unsigned integer.");
+ }
+
+ if(req_params.count(MIN_LEN_2TYPO) == 0) {
+ req_params[MIN_LEN_2TYPO] = "7";
+ } else if(!StringUtils::is_uint32_t(req_params[MIN_LEN_2TYPO])) {
+ return Option<bool>(400, "Parameter `" + std::string(MIN_LEN_2TYPO) + "` must be an unsigned integer.");
+ }
+
if(req_params.count(PREFIX) == 0) {
req_params[PREFIX] = "true";
}
diff --git a/src/index.cpp b/src/index.cpp
index ba10642e..b603845f 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1644,7 +1644,9 @@ void Index::run_search(search_args* search_params) {
search_params->prioritize_exact_match,
search_params->exhaustive_search,
search_params->concurrency,
- search_params->search_cutoff_ms);
+ search_params->search_cutoff_ms,
+ search_params->min_len_1typo,
+ search_params->min_len_2typo);
}
void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,
@@ -1982,7 +1984,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&
search_field(0, window_tokens, search_tokens, nullptr, 0, num_toks_dropped, field_name,
nullptr, 0, {}, facets, {}, 2, searched_queries, topster, groups_processed,
&result_ids, result_ids_len, field_num_results, 0, group_by_fields,
- false, 4, query_hashes, token_order, false, 0, 1, false);
+ false, 4, query_hashes, token_order, false, 0, 1, false, 3, 7);
if(result_ids_len != 0) {
// remove window_tokens from `tokens`
@@ -2034,7 +2036,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
bool prioritize_exact_match,
const bool exhaustive_search,
const size_t concurrency,
- const size_t search_cutoff_ms) const {
+ const size_t search_cutoff_ms,
+ size_t min_len_1typo,
+ size_t min_len_2typo) const {
begin = std::chrono::high_resolution_clock::now();
search_stop_ms = search_cutoff_ms;
@@ -2170,7 +2174,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
query_hashes, token_order, field_prefix,
- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);
+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,
+ min_len_1typo, min_len_2typo);
bool syn_wildcard_filter_init_done = false;
@@ -2202,7 +2207,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
query_hashes, token_order, field_prefix,
- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search);
+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,
+ min_len_1typo, min_len_2typo);
}
}
@@ -2629,7 +2635,9 @@ void Index::search_field(const uint8_t & field_id,
const token_ordering token_order, const bool prefix,
const size_t drop_tokens_threshold,
const size_t typo_tokens_threshold,
- const bool exhaustive_search) const {
+ const bool exhaustive_search,
+ size_t min_len_1typo,
+ size_t min_len_2typo) const {
// NOTE: `query_tokens` preserve original tokens, while `search_tokens` could be a result of dropped tokens
@@ -2651,7 +2659,7 @@ void Index::search_field(const uint8_t & field_id,
std::vector<int> all_costs;
// This ensures that we don't end up doing a cost of 1 for a single char etc.
- int bounded_cost = get_bounded_typo_cost(max_cost, token.length());
+ int bounded_cost = get_bounded_typo_cost(max_cost, token.length(), min_len_1typo, min_len_2typo);
for(int cost = 0; cost <= bounded_cost; cost++) {
all_costs.push_back(cost);
@@ -2804,17 +2812,18 @@ void Index::search_field(const uint8_t & field_id,
all_result_ids_len, field_num_results, group_limit, group_by_fields,
prioritize_exact_match, concurrency, query_hashes,
token_order, prefix, drop_tokens_threshold, typo_tokens_threshold,
- exhaustive_search);
+ exhaustive_search, min_len_1typo, min_len_2typo);
}
}
-int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len) {
- if(token_len < 4) {
+int Index::get_bounded_typo_cost(const size_t max_cost, const size_t token_len,
+ const size_t min_len_1typo, const size_t min_len_2typo) {
+ if(token_len < min_len_1typo) {
// typo correction is disabled for small tokens
return 0;
}
- if(token_len < 7) {
+ if(token_len < min_len_2typo) {
// 2-typos are enabled only at token length of 7 chars
return std::min<int>(max_cost, 1);
}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 151bd3cf..ccc01647 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1577,3 +1577,67 @@ TEST_F(CollectionSpecificTest, VerbatimMatchShouldConsiderTokensMatchedAcrossAll
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionSpecificTest, CustomNumTyposConfiguration) {
+ // dropped tokens on a single field cannot be deemed as verbatim match
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("brand", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Hamburger";
+ doc1["brand"] = "Burger and King";
+ doc1["points"] = 10;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ // by default a typo on 3 char tokens are ignored (min 4 length is needed)
+
+ auto results = coll1->search("asd", {"brand"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false},
+ 2, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1},
+ 1000, true, false, true, "", false, 60000*100).get();
+
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = coll1->search("asd", {"brand"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false},
+ 2, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1},
+ 1000, true, false, true, "", false, 60000*100, 3, 7).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ // 2 typos are not tolerated by default on 6-len word
+
+ results = coll1->search("bixger", {"brand"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false},
+ 2, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1},
+ 1000, true, false, true, "", false, 60000*100).get();
+
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = coll1->search("bixger", {"brand"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false},
+ 2, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1},
+ 1000, true, false, true, "", false, 60000*100, 3, 6).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ collectionManager.drop_collection("coll1");
+}