Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Improve prefix typos in fuzzy search.

Kishore Nallan776730b5719e
diff --git a/src/art.cpp b/src/art.cpp
index 40b028a3..e5fb4c58 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1349,13 +1349,15 @@ static inline void rotate(int &i, int &j, int &k) {
}
// -1: return without adding, 0 : continue iteration, 1: return after adding
-static inline int fuzzy_search_state(const bool prefix, int key_index, bool last_key_char,
- const int query_len, const int* cost_row, int min_cost, int max_cost) {
+static inline int fuzzy_search_state(const bool prefix, int key_index, unsigned char p, unsigned char c,
+ const unsigned char* query, const int query_len,
+ const int* cost_row, int min_cost, int max_cost) {
// There are 2 scenarios:
// a) key_len < query_len: "pltninum" (query) on "pst" (key)
// b) query_len < key_len: "pst" (query) on "pltninum" (key)
+ bool last_key_char = (c == '\0');
int key_len = last_key_char ? key_index : key_index + 1;
if(last_key_char) {
@@ -1387,11 +1389,67 @@ static inline int fuzzy_search_state(const bool prefix, int key_index, bool last
}
}
- // Terminate the search early or continue iterating on the key?
- // We have to account for the case that `cost` could momentarily exceed max_cost but resolve later.
- // e.g. key=example, query=exZZample, after 5 chars, cost is 3 but drops to 2 at the end.
- // But we will limit this for longer keys for performance.
- return cost > max_cost && (key_len > 3 ? cost > (max_cost * 2) : true) ? -1 : 0;
+ /*
+ Terminate the search early or continue iterating on the key?
+ We have to account for the case that `cost` could momentarily exceed max_cost but resolve later.
+ In such cases, we will compare characters in the query with p and/or c to decide.
+ */
+
+ if(cost <= max_cost) {
+ return 0;
+ }
+
+ if(cost == 2 || cost == 3) {
+ /*
+ [1 letter extra]
+ exam ple
+ exZa mple
+
+ [1 letter missing]
+ exam ple
+ exmp le
+
+ [1 letter missing + transpose]
+ dacrycystal gia
+ dacrcyystlg ia
+ */
+ bool letter_more = (key_index+1 < query_len && query[key_index+1] == c);
+ bool letter_less = (key_index > 0 && query[key_index-1] == c);
+ if(letter_more || letter_less) {
+ return 0;
+ }
+ }
+
+ if(cost == 3 || cost == 4) {
+ /*
+ [2 letter extra]
+ exam ple
+ eTxT ample
+
+ abbviat ion
+ abbrevi ation
+ */
+
+ bool extra_matching_letters = (key_index + 1 < query_len && p == query[key_index + 1] &&
+ key_index + 2 < query_len && c == query[key_index + 2]);
+
+ if(extra_matching_letters) {
+ return 0;
+ }
+
+ /*
+ [2 letter missing]
+ exam ple
+ expl e
+ */
+
+ bool two_letter_less = (key_index > 1 && query[key_index-2] == c);
+ if(two_letter_less) {
+ return 0;
+ }
+ }
+
+ return -1;
}
static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *n, int depth, const unsigned char *term,
@@ -1420,10 +1478,9 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
if(!prefix || !last_key_char) {
levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
rotate(i, j, k);
- p = c;
}
- int action = fuzzy_search_state(prefix, depth, last_key_char, term_len, rows[j], min_cost, max_cost);
+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);
if(1 == action) {
results.push_back(n);
return;
@@ -1433,6 +1490,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
return;
}
+ p = c;
depth++;
}
@@ -1451,7 +1509,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
if(depth >= iter_len) {
// when a preceding partial node completely contains the whole leaf (e.g. "[raspberr]y" on "raspberries")
- int action = fuzzy_search_state(prefix, depth, true, term_len, rows[j], min_cost, max_cost);
+ int action = fuzzy_search_state(prefix, depth, '\0', '\0', term, term_len, rows[j], min_cost, max_cost);
if(action == 1) {
results.push_back(n);
}
@@ -1471,10 +1529,9 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
printf("cost: %d, depth: %d, term_len: %d\n", temp_cost, depth, term_len);
rotate(i, j, k);
- p = c;
}
- int action = fuzzy_search_state(prefix, depth, last_key_char, term_len, rows[j], min_cost, max_cost);
+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);
if(action == 1) {
results.push_back(n);
return;
@@ -1484,6 +1541,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
return;
}
+ p = c;
depth++;
}
@@ -1500,9 +1558,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
rotate(i, j, k);
- p = c;
- int action = fuzzy_search_state(prefix, depth, false, term_len, rows[j], min_cost, max_cost);
+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);
if(action == 1) {
results.push_back(n);
return;
@@ -1512,6 +1569,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
return;
}
+ p = c;
depth++;
}
@@ -1520,9 +1578,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
c = term[depth];
levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
rotate(i, j, k);
- p = c;
- int action = fuzzy_search_state(prefix, depth, false, term_len, rows[j], min_cost, max_cost);
+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);
if(action == 1) {
results.push_back(n);
return;
@@ -1532,6 +1589,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
return;
}
+ p = c;
depth++;
partial_len++;
}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index f8414653..daf9d4fd 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -653,6 +653,9 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf_qlen_greater_than_key) {
std::vector<art_leaf*> leaves;
art_fuzzy_search(&t, (const unsigned char *)(term.c_str()), term.size(), 0, 2, 10, FREQUENCY, true, false, "", nullptr, 0, leaves, exclude_leaves);
ASSERT_EQ(0, leaves.size());
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
}
TEST(ArtTest, test_art_fuzzy_search_single_leaf_non_prefix) {
@@ -816,7 +819,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, false, "", nullptr, 0, leaves, exclude_leaves);
ASSERT_EQ(10, leaves.size());
- std::set<std::string> expected_words = {"town", "sown", "mown", "lown", "howl", "howk", "howe", "how", "horn", "hoon"};
+ std::set<std::string> expected_words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};
for(size_t leaf_index = 0; leaf_index < leaves.size(); leaf_index++) {
art_leaf*& leaf = leaves.at(leaf_index);
@@ -1100,6 +1103,14 @@ TEST(ArtTest, test_art_search_roche_chews) {
ASSERT_EQ(1, leaves.size());
+ term = "xxroche";
+ leaves.clear();
+ exclude_leaves.clear();
+ art_fuzzy_search(&t, (const unsigned char*)term.c_str(), term.size()+1, 0, 2, 10,
+ FREQUENCY, false, false, "", nullptr, 0, leaves, exclude_leaves);
+
+ ASSERT_EQ(1, leaves.size());
+
res = art_tree_destroy(&t);
ASSERT_TRUE(res == 0);
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 13d1bdab..8e543ccd 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -397,7 +397,7 @@ TEST_F(CollectionTest, QueryWithTypo) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
- ids = {"1", "13", "8"};
+ ids = {"8", "1", "17"};
ASSERT_EQ(3, results["hits"].size());