Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix typo correction edge case.

Kishore Nallan1d0917dc41c1
diff --git a/src/art.cpp b/src/art.cpp
index 5ba4aea0..76dbddaf 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1304,25 +1304,29 @@ static inline int fuzzy_search_state(const bool prefix, int key_index, bool last
// b) term_len < iter_len: "pst" (term) on "pltninum" (key)
int cost = 0;
- int key_len = key_index + 1;
// a) because key's null character will appear first
if(last_key_char) {
+ int key_len = key_index;
+ cost = cost_row[term_len];
- if(key_len > 5 && term_len > key_len && (term_len - key_len) <= max_cost) {
- // used to handle some trailing edge cases, but limit to larger keys to prevent eager matches
- cost = std::min(cost_row[key_len], cost_row[term_len]);
- } else {
- cost = cost_row[term_len];
+ if(cost >= min_cost && cost <= max_cost) {
+ return 1;
}
- if(cost >= min_cost && cost <= max_cost) {
+ cost = cost_row[key_len];
+
+ // used to match q=strawberries on key=strawberry, but limit to larger keys to prevent eager matches
+ if(key_len > 5 && term_len > key_len && (term_len - key_len) <= max_cost &&
+ cost >= min_cost && cost <= max_cost) {
return 1;
}
return -1;
}
+ int key_len = key_index + 1;
+
// b) we might iterate past term_len to catch trailing typos
if(key_len >= term_len && prefix) {
cost = cost_row[term_len];
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 4f30b042..ae900fcc 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -655,6 +655,24 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf_non_prefix) {
ASSERT_TRUE(res == 0);
}
+TEST(ArtTest, test_art_prefix_larger_than_key) {
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
+
+ const char* key = "arvin";
+ art_document doc = get_document((uint32_t) 1);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));
+
+ std::string term = "earrings";
+ std::vector<art_leaf*> leaves;
+ art_fuzzy_search(&t, (const unsigned char *)(term.c_str()), term.size()+1, 0, 2, 10, FREQUENCY, false, nullptr, 0, leaves);
+ ASSERT_EQ(0, leaves.size());
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
+}
+
TEST(ArtTest, test_art_fuzzy_search_prefix_token_ordering) {
art_tree t;
int res = art_tree_init(&t);
@@ -732,9 +750,8 @@ TEST(ArtTest, test_art_fuzzy_search) {
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("zymosthenic", (const char *)leaves.at(0)->key);
- // transpose + missing -- temporarily ignored because too slow for the value!
-
- /*leaves.clear();
+ // transpose + missing
+ leaves.clear();
art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 0, 2, 10, FREQUENCY, false, nullptr, 0, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);
@@ -742,7 +759,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
leaves.clear();
art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 1, 2, 10, FREQUENCY, false, nullptr, 0, leaves);
ASSERT_EQ(1, leaves.size());
- ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);*/
+ ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);
// missing char
leaves.clear();
@@ -794,13 +811,12 @@ TEST(ArtTest, test_art_fuzzy_search) {
leaves.clear();
art_fuzzy_search(&t, (const unsigned char *) "antisocao", strlen("antisocao"), 0, 2, 10, FREQUENCY, true, nullptr, 0, leaves);
- ASSERT_EQ(7, leaves.size());
+ ASSERT_EQ(9, leaves.size());
long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
LOG(INFO) << "Time taken for: " << timeMillis << "ms";
-
res = art_tree_destroy(&t);
ASSERT_TRUE(res == 0);
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 20fa5dce..55d030f4 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -430,3 +430,25 @@ TEST_F(CollectionSpecificMoreTest, SortByStringEmptyValuesConfigThirdField) {
ASSERT_EQ(4, results["hits"].size());
ASSERT_EQ("2", results["hits"][3]["document"]["id"].get<std::string>());
}
+
+TEST_F(CollectionSpecificMoreTest, WrongTypoCorrection) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["title"] = "Gold plated arvin";
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ auto results = coll1->search("earrings", {"title"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 5, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(0, results["hits"].size());
+ collectionManager.drop_collection("coll1");
+}
\ No newline at end of file