Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Adopt Damerau–Levenshtein distance, instead of plain Levenshtein.
Kishore Nallanc96a9d9b3580
diff --git a/include/art.h b/include/art.hindex 2e6b9fda..c96e072e 100644--- a/include/art.h+++ b/include/art.h@@ -234,8 +234,8 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, arint art_fuzzy_results(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,const int max_words, std::vector<art_leaf *> &results);-static void art_fuzzy_recurse(char c, const art_node *n, int depth, const unsigned char *term,- const int term_len, const int *previous_row, const int max_cost,+static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,+ const int term_len, const int* irow, const int* jrow, const int max_cost,std::vector<const art_node *> &results);static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results);diff --git a/src/art.cpp b/src/art.cppindex 0f0d0f36..e729e015 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1079,33 +1079,37 @@ static inline void copyIntArray2(const int *src, int *dest, const int len) {}}-static inline int levenshtein_dist(const char c, const unsigned char* term, const int term_len,- const int* previous_row, int* current_row) {+static inline int levenshtein_dist(const int depth, const char p, const char c, const unsigned char* term, const int term_len,+ const int* irow, const int* jrow, int* krow) {int row_min = std::numeric_limits<int>::max();const int columns = term_len+1;- current_row[0] = previous_row[0] + 1;+ krow[0] = jrow[0] + 1;++ // Calculate levenshtein distance incrementally (j => column, b => term):+ // https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance#Optimal_string_alignment_distance- // calculate levenshtein distance incrementallyfor(int column=1; column<columns; column++) {- int insert_cost = current_row[column - 1] + 1;- int delete_cost = previous_row[column] + 1;- int replace_cost = 0;+ int delete_cost = jrow[column] + 1;+ int insert_cost = krow[column - 1] + 1;- if(term[column-1] != c) {- replace_cost = previous_row[column - 1] + 1;- } else {- replace_cost = previous_row[column - 1];+ int cost = (c != term[column-1]) ? 1 : 0;+ int replace_cost = jrow[column - 1] + cost;++ krow[column] = min(min(insert_cost, delete_cost), replace_cost);++ if(depth > 1 && column > 1 && c == term[column-2] && p == term[column-1]) {+ krow[column] = std::min(krow[column], irow[column-2] + cost);}- current_row[column] = min(min(insert_cost, delete_cost), replace_cost);- if(current_row[column] < row_min) row_min = current_row[column];+ if(krow[column] < row_min) row_min = krow[column];}return row_min;}-static inline void art_fuzzy_children(const art_node *n, int depth, const unsigned char *term, const int term_len,- const int *previous_row, const int max_cost, std::vector<const art_node *> &results) {+static inline void art_fuzzy_children(char p, const art_node *n, int depth, const unsigned char *term, const int term_len,+ const int* irow, const int* jrow, const int max_cost,+ std::vector<const art_node *> &results) {char child_char;art_node* child;@@ -1116,7 +1120,7 @@ static inline void art_fuzzy_children(const art_node *n, int depth, const unsignchild_char = ((art_node4*)n)->keys[i];printf("\n4!child_char: %c, %d, depth: %d", child_char, child_char, depth);child = ((art_node4*)n)->children[i];- art_fuzzy_recurse(child_char, child, depth, term, term_len, previous_row, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);}break;case NODE16:@@ -1125,7 +1129,7 @@ static inline void art_fuzzy_children(const art_node *n, int depth, const unsignchild_char = ((art_node16*)n)->keys[i];printf("\n16!child_char: %c, depth: %d", child_char, depth);child = ((art_node16*)n)->children[i];- art_fuzzy_recurse(child_char, child, depth, term, term_len, previous_row, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);}break;case NODE48:@@ -1136,7 +1140,7 @@ static inline void art_fuzzy_children(const art_node *n, int depth, const unsignchild = ((art_node48*)n)->children[ix - 1];child_char = (char)i;printf("\n48!child_char: %c, depth: %d, ix: %d", child_char, depth, ix);- art_fuzzy_recurse(child_char, child, depth, term, term_len, previous_row, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);}break;case NODE256:@@ -1146,7 +1150,7 @@ static inline void art_fuzzy_children(const art_node *n, int depth, const unsignchild_char = (char) i;printf("\n256!child_char: %c, depth: %d", child_char, depth);child = ((art_node256*)n)->children[i];- art_fuzzy_recurse(child_char, child, depth, term, term_len, previous_row, max_cost, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, results);}break;default:@@ -1154,16 +1158,32 @@ static inline void art_fuzzy_children(const art_node *n, int depth, const unsign}}+static inline void rotate(int &i, int &j, int &k) {+ int old_i = i;+ i = j;+ j = k;+ k = old_i;+}+// e.g. catapult against coratapult-static void art_fuzzy_recurse(char c, const art_node *n, int depth, const unsigned char *term,- const int term_len, const int *previous_row, const int max_cost,+// e.g. microafot against microsoft+static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,+ const int term_len, const int* irow, const int* jrow, const int max_cost,std::vector<const art_node *> &results) {const int columns = term_len+1;- int current_row[columns];- int local_prev_row[columns];+ int i=0, j=1, k=2;+ int row0[columns];+ int row1[columns];+ int row2[columns];+ int* rows[3] = {row0, row1, row2};++ copyIntArray2(irow, rows[i], columns);+ copyIntArray2(jrow, rows[j], columns);++ int cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ rotate(i, j, k);+ p = c;- int cost = levenshtein_dist(c, term, term_len, previous_row, current_row);- copyIntArray2(current_row, local_prev_row, columns);depth++;printf("\nRecurse char: %c, cost: %d", c, cost);@@ -1181,15 +1201,17 @@ static void art_fuzzy_recurse(char c, const art_node *n, int depth, const unsignconst int end_index = min(l->key_len, term_len+max_cost);while(depth < end_index && cost <= 2*max_cost) {- cost = levenshtein_dist(l->key[depth], term, term_len, local_prev_row, current_row);- printf("leaf char: %c\n", l->key[idx]);+ c = l->key[depth];+ cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ printf("leaf char: %c\n", l->key[depth]);printf("cost: %d, depth: %d, term_len: %d\n", cost, depth, term_len);- copyIntArray2(current_row, local_prev_row, columns);+ rotate(i, j, k);+ p = c;depth++;}- // current_row[columns-1] holds the final cost- if(current_row[columns-1] <= max_cost) {+ // rows[j][columns-1] holds the final cost+ if(rows[j][columns-1] <= max_cost) {results.push_back(n);}@@ -1202,21 +1224,23 @@ static void art_fuzzy_recurse(char c, const art_node *n, int depth, const unsignprintf("\npartial_len: %d", partial_len);for(int idx=0; idx<end_index; idx++) {- printf("partial: %c ", n->partial[idx]);- current_row[0] = local_prev_row[0] + 1;- cost = levenshtein_dist(n->partial[idx], term, term_len, local_prev_row, current_row);- copyIntArray2(current_row, local_prev_row, columns);+ c = n->partial[idx];+ printf("partial: %c ", c);+ rows[k][0] = rows[j][0] + 1;+ cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ rotate(i, j, k);+ p = c;}depth += n->partial_len;printf("\ncost: %d", cost);- if(depth >= term_len-1 && current_row[columns-1] <= max_cost) {+ if(depth >= term_len-1 && rows[j][columns-1] <= max_cost) {results.push_back(n);return ;}- art_fuzzy_children(n, depth, term, term_len, current_row, max_cost, results);+ art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], max_cost, results);}/**@@ -1226,13 +1250,15 @@ int art_fuzzy_results(art_tree *t, const unsigned char *term, const int term_lenconst int max_cost, const int max_words, std::vector<art_leaf *> &results) {std::vector<const art_node*> nodes;- int previous_row[term_len + 1];+ int irow[term_len + 1];+ int jrow[term_len + 1];for (int i = 0; i <= term_len; i++){- previous_row[i] = i;+ jrow[i] = i;+ irow[i] = i;}auto begin = std::chrono::high_resolution_clock::now();- art_fuzzy_children(t->root, 0, term, term_len, previous_row, max_cost, nodes);+ art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, nodes);std::sort(nodes.begin(), nodes.end(), compare_art_node);long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl;diff --git a/src/main.cpp b/src/main.cppindex 84079b58..b434a2d1 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -26,7 +26,7 @@ int main() {cout << "FINISHED INDEXING!" << endl << flush;auto begin = std::chrono::high_resolution_clock::now();- collection->search("plant", 2, 100);+ collection->search("platn", 1, 100);long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();cout << "Time taken: " << timeMillis << "us" << endl;delete collection;