Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Improve prefix typos in fuzzy search.
Kishore Nallan776730b5719e
diff --git a/src/art.cpp b/src/art.cppindex 40b028a3..e5fb4c58 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1349,13 +1349,15 @@ static inline void rotate(int &i, int &j, int &k) {}// -1: return without adding, 0 : continue iteration, 1: return after adding-static inline int fuzzy_search_state(const bool prefix, int key_index, bool last_key_char,- const int query_len, const int* cost_row, int min_cost, int max_cost) {+static inline int fuzzy_search_state(const bool prefix, int key_index, unsigned char p, unsigned char c,+ const unsigned char* query, const int query_len,+ const int* cost_row, int min_cost, int max_cost) {// There are 2 scenarios:// a) key_len < query_len: "pltninum" (query) on "pst" (key)// b) query_len < key_len: "pst" (query) on "pltninum" (key)+ bool last_key_char = (c == '\0');int key_len = last_key_char ? key_index : key_index + 1;if(last_key_char) {@@ -1387,11 +1389,67 @@ static inline int fuzzy_search_state(const bool prefix, int key_index, bool last}}- // Terminate the search early or continue iterating on the key?- // We have to account for the case that `cost` could momentarily exceed max_cost but resolve later.- // e.g. key=example, query=exZZample, after 5 chars, cost is 3 but drops to 2 at the end.- // But we will limit this for longer keys for performance.- return cost > max_cost && (key_len > 3 ? cost > (max_cost * 2) : true) ? -1 : 0;+ /*+ Terminate the search early or continue iterating on the key?+ We have to account for the case that `cost` could momentarily exceed max_cost but resolve later.+ In such cases, we will compare characters in the query with p and/or c to decide.+ */++ if(cost <= max_cost) {+ return 0;+ }++ if(cost == 2 || cost == 3) {+ /*+ [1 letter extra]+ exam ple+ exZa mple++ [1 letter missing]+ exam ple+ exmp le++ [1 letter missing + transpose]+ dacrycystal gia+ dacrcyystlg ia+ */+ bool letter_more = (key_index+1 < query_len && query[key_index+1] == c);+ bool letter_less = (key_index > 0 && query[key_index-1] == c);+ if(letter_more || letter_less) {+ return 0;+ }+ }++ if(cost == 3 || cost == 4) {+ /*+ [2 letter extra]+ exam ple+ eTxT ample++ abbviat ion+ abbrevi ation+ */++ bool extra_matching_letters = (key_index + 1 < query_len && p == query[key_index + 1] &&+ key_index + 2 < query_len && c == query[key_index + 2]);++ if(extra_matching_letters) {+ return 0;+ }++ /*+ [2 letter missing]+ exam ple+ expl e+ */++ bool two_letter_less = (key_index > 1 && query[key_index-2] == c);+ if(two_letter_less) {+ return 0;+ }+ }++ return -1;}static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *n, int depth, const unsigned char *term,@@ -1420,10 +1478,9 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *if(!prefix || !last_key_char) {levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);rotate(i, j, k);- p = c;}- int action = fuzzy_search_state(prefix, depth, last_key_char, term_len, rows[j], min_cost, max_cost);+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);if(1 == action) {results.push_back(n);return;@@ -1433,6 +1490,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *return;}+ p = c;depth++;}@@ -1451,7 +1509,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *if(depth >= iter_len) {// when a preceding partial node completely contains the whole leaf (e.g. "[raspberr]y" on "raspberries")- int action = fuzzy_search_state(prefix, depth, true, term_len, rows[j], min_cost, max_cost);+ int action = fuzzy_search_state(prefix, depth, '\0', '\0', term, term_len, rows[j], min_cost, max_cost);if(action == 1) {results.push_back(n);}@@ -1471,10 +1529,9 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *printf("cost: %d, depth: %d, term_len: %d\n", temp_cost, depth, term_len);rotate(i, j, k);- p = c;}- int action = fuzzy_search_state(prefix, depth, last_key_char, term_len, rows[j], min_cost, max_cost);+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);if(action == 1) {results.push_back(n);return;@@ -1484,6 +1541,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *return;}+ p = c;depth++;}@@ -1500,9 +1558,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);rotate(i, j, k);- p = c;- int action = fuzzy_search_state(prefix, depth, false, term_len, rows[j], min_cost, max_cost);+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);if(action == 1) {results.push_back(n);return;@@ -1512,6 +1569,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *return;}+ p = c;depth++;}@@ -1520,9 +1578,8 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *c = term[depth];levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);rotate(i, j, k);- p = c;- int action = fuzzy_search_state(prefix, depth, false, term_len, rows[j], min_cost, max_cost);+ int action = fuzzy_search_state(prefix, depth, p, c, term, term_len, rows[j], min_cost, max_cost);if(action == 1) {results.push_back(n);return;@@ -1532,6 +1589,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *return;}+ p = c;depth++;partial_len++;}diff --git a/test/art_test.cpp b/test/art_test.cppindex f8414653..daf9d4fd 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -653,6 +653,9 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf_qlen_greater_than_key) {std::vector<art_leaf*> leaves;art_fuzzy_search(&t, (const unsigned char *)(term.c_str()), term.size(), 0, 2, 10, FREQUENCY, true, false, "", nullptr, 0, leaves, exclude_leaves);ASSERT_EQ(0, leaves.size());++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);}TEST(ArtTest, test_art_fuzzy_search_single_leaf_non_prefix) {@@ -816,7 +819,7 @@ TEST(ArtTest, test_art_fuzzy_search) {art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, false, "", nullptr, 0, leaves, exclude_leaves);ASSERT_EQ(10, leaves.size());- std::set<std::string> expected_words = {"town", "sown", "mown", "lown", "howl", "howk", "howe", "how", "horn", "hoon"};+ std::set<std::string> expected_words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};for(size_t leaf_index = 0; leaf_index < leaves.size(); leaf_index++) {art_leaf*& leaf = leaves.at(leaf_index);@@ -1100,6 +1103,14 @@ TEST(ArtTest, test_art_search_roche_chews) {ASSERT_EQ(1, leaves.size());+ term = "xxroche";+ leaves.clear();+ exclude_leaves.clear();+ art_fuzzy_search(&t, (const unsigned char*)term.c_str(), term.size()+1, 0, 2, 10,+ FREQUENCY, false, false, "", nullptr, 0, leaves, exclude_leaves);++ ASSERT_EQ(1, leaves.size());+res = art_tree_destroy(&t);ASSERT_TRUE(res == 0);}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 13d1bdab..8e543ccd 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -397,7 +397,7 @@ TEST_F(CollectionTest, QueryWithTypo) {spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10).get();- ids = {"1", "13", "8"};+ ids = {"8", "1", "17"};ASSERT_EQ(3, results["hits"].size());