Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed an edge case in fuzzy search with SKU-like tokens.
kishorenca3024a73422b
diff --git a/src/art.cpp b/src/art.cppindex a500817b..7ac945c0 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1148,23 +1148,22 @@ static inline int levenshtein_dist(const int depth, const unsigned char p, constconst unsigned char* term, const int term_len,const int* irow, const int* jrow, int* krow) {int row_min = std::numeric_limits<int>::max();- const int columns = term_len+1;krow[0] = jrow[0] + 1;- // Calculate levenshtein distance incrementally (j => column, b => term):+ // Calculate levenshtein distance incrementally (term => b, column => j, c => a[i], p => a[i-1], irow => d[i-1]):// https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance#Optimal_string_alignment_distance- for(int column=1; column<columns; column++) {+ for(int column=1; column<=term_len; column++) {+ int cost = (c == term[column-1]) ? 0 : 1; // column-1 used because of zero-based char array+int delete_cost = jrow[column] + 1;int insert_cost = krow[column - 1] + 1;+ int substitution_cost = jrow[column - 1] + cost;- int cost = (c != term[column-1]) ? 1 : 0;- int replace_cost = jrow[column - 1] + cost;+ krow[column] = min(min(insert_cost, delete_cost), substitution_cost);- krow[column] = min(min(insert_cost, delete_cost), replace_cost);-- if(depth > 1 && column > 1 && c == term[column-2] && p == term[column-1]) {- krow[column] = std::min(krow[column], irow[column-2] + cost);+ if(depth > 1 && column > 1 && c == term[column-1-1] && p == term[column-1]) {+ krow[column] = std::min(krow[column], irow[column-2] + 1);}if(krow[column] < row_min) {@@ -1175,7 +1174,7 @@ static inline int levenshtein_dist(const int depth, const unsigned char p, constreturn row_min;}-static inline void art_fuzzy_children(char p, const art_node *n, int depth, const unsigned char *term, const int term_len,+static inline void art_fuzzy_children(unsigned char p, const art_node *n, int depth, const unsigned char *term, const int term_len,const int* irow, const int* jrow, const int min_cost, const int max_cost,const bool prefix, std::vector<const art_node *> &results) {char child_char;@@ -1238,6 +1237,8 @@ static inline void rotate(int &i, int &j, int &k) {static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *n, int depth, const unsigned char *term,const int term_len, const int* irow, const int* jrow, const int min_cost,const int max_cost, const bool prefix, std::vector<const art_node *> &results) {+ if (!n) return ;+const int columns = term_len+1;int i=0, j=1, k=2;int row0[columns];@@ -1248,7 +1249,15 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *copyIntArray2(irow, rows[i], columns);copyIntArray2(jrow, rows[j], columns);- int cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ int temp_cost = 0;++ if(depth == -1) {+ depth = 0;+ goto PARTIAL_CALC;+ }++ // First calculate cost with node char `c` and then leaf/partial related costs+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);rotate(i, j, k);p = c;@@ -1256,12 +1265,12 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *printf("Recurse char: %c, cost: %d, depth: %d\n", c, cost, depth);- if(cost > max_cost) {- // We do this to speed up things drastically, but at the cost of missing out on some genuine typos+ if(temp_cost > max_cost) {+ // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`return;}- if (!n) return ;+ // Cost is under control, let's check to see if we should proceed furtherif(IS_LEAF(n)) {art_leaf *l = (art_leaf *) LEAF_RAW(n);@@ -1276,10 +1285,10 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node **/const int end_index = prefix ? min(l->key_len, term_len) : l->key_len;- // If at any point, `cost > 2*max_cost` we can terminate immediately as we can never recover from that- while(depth < end_index && cost <= 2*max_cost) {+ // If at any point, `temp_cost > 2*max_cost` we can terminate immediately as we can never recover from that+ while(depth < end_index && temp_cost <= 2*max_cost) {c = l->key[depth];- cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);printf("leaf char: %c\n", l->key[depth]);printf("cost: %d, depth: %d, term_len: %d\n", cost, depth, term_len);rotate(i, j, k);@@ -1287,20 +1296,24 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *depth++;}- /* `rows[j][columns-1]` holds the final cost, `cost` holds the intermediate cost.- We will use the intermediate cost if the term is shorter than the key.+ /* `rows[j][columns-1]` holds the final cost, `temp_cost` holds the temporary cost.+ We will use the intermediate cost if the term is shorter than the key and if it's a prefix search.+ For non-prefix, we will only use final cost.*/- if(prefix && term_len < (int) l->key_len && cost >= min_cost && cost <= max_cost) {++ int final_cost = rows[j][columns-1];++ if(prefix && term_len < (int) l->key_len && temp_cost >= min_cost && temp_cost <= max_cost) {results.push_back(n);return;}- if(prefix && term_len >= (int) l->key_len && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {+ if(prefix && term_len >= (int) l->key_len && final_cost >= min_cost && final_cost <= max_cost) {results.push_back(n);return;}- if(!prefix && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {+ if(!prefix && final_cost >= min_cost && final_cost <= max_cost) {results.push_back(n);return;}@@ -1308,12 +1321,16 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *return ;}- if(prefix && depth >= term_len && rows[j][columns-1] <= max_cost) {- // For a prefix search, we store the node and not recurse further right now+ // For a prefix search whose depth has reached term length, we need not recurse further+ if(prefix && depth >= term_len) {results.push_back(n);return ;}+ PARTIAL_CALC:++ // For non-prefix search or if we have not reached term length, we will recurse further+const int partial_len = min(MAX_PREFIX_LEN, n->partial_len);const int end_index = min(partial_len, term_len+max_cost);@@ -1322,12 +1339,11 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *for(int idx=0; idx<end_index; idx++) {c = n->partial[idx];printf("partial: %c\n", c);- rows[k][0] = rows[j][0] + 1;- cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ temp_cost = levenshtein_dist(depth+idx, p, c, term, term_len, rows[i], rows[j], rows[k]);rotate(i, j, k);p = c;-- if(prefix && depth+idx+1 >= term_len && rows[j][columns-1] <= max_cost) {++ if(prefix && depth+idx+1 >= term_len && temp_cost <= max_cost) {// For a prefix search, we store the node and not recurse further right nowresults.push_back(n);return ;@@ -1351,8 +1367,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,int irow[term_len + 1];int jrow[term_len + 1];for (int i = 0; i <= term_len; i++){- jrow[i] = i;- irow[i] = i;+ irow[i] = jrow[i] = i;}//auto begin = std::chrono::high_resolution_clock::now();@@ -1361,12 +1376,16 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,art_leaf *l = (art_leaf *) LEAF_RAW(t->root);art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);} else {- if(t->root == NULL) {+ if(t->root == nullptr) {return 0;}- art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);++ // send depth as -1 to indicate that this is a root node+ art_fuzzy_recurse(0, 0, t->root, -1, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);}+ PROCESS_NODES:+if(token_order == FREQUENCY) {std::sort(nodes.begin(), nodes.end(), compare_art_node_frequency);} else {diff --git a/test/art_test.cpp b/test/art_test.cppindex ca690fb9..6ede5656 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -8,6 +8,7 @@#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()+#define skus_file_path std::string(std::string(ROOT_DIR)+"/test/skus.txt").c_str()art_document get_document(uint32_t id) {art_document document;@@ -717,7 +718,7 @@ TEST(ArtTest, test_art_fuzzy_search) {leaves.clear();art_fuzzy_search(&t, (const unsigned char *) "antisocao", strlen("antisocao"), 0, 2, 10, FREQUENCY, true, leaves);- ASSERT_EQ(8, leaves.size());+ ASSERT_EQ(10, leaves.size());res = art_tree_destroy(&t);ASSERT_TRUE(res == 0);@@ -742,8 +743,57 @@ TEST(ArtTest, test_art_fuzzy_search_unicode_chars) {EXPECT_EQ(1, l->values->ids.at(0));std::vector<art_leaf*> leaves;- art_fuzzy_search(&t, (unsigned char *)key, strlen(key), 0, 1, 10, FREQUENCY, true, leaves);+ art_fuzzy_search(&t, (unsigned char *)key, strlen(key), 0, 0, 10, FREQUENCY, true, leaves);+ ASSERT_EQ(1, leaves.size());+ }++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_search_sku_like_tokens) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);+++ std::vector<std::string> keys;+ int len;+ char buf[512];+ FILE *f = fopen(skus_file_path, "r");++ uintptr_t line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len - 1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));+ keys.push_back(std::string(buf, len-1));+ line++;+ }++ const char* key1 = "abc12345678217521";++ // exact search+ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key1, strlen(key1)+1);+ EXPECT_EQ(1, l->values->ids.getLength());++ // exact search all tokens via fuzzy API++ for (const auto &key : keys) {+ std::vector<art_leaf *> leaves;+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,+ FREQUENCY, true, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);++ leaves.clear();++ // non prefix+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,+ FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);}res = art_tree_destroy(&t);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 27dea864..23269f1f 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -524,8 +524,9 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_EQ(0, results["hits"].size());results = collection->search("xq", query_fields, "", facets, sort_fields, 2, 2, 1, FREQUENCY, true).get();- ASSERT_EQ(1, results["hits"].size());- ids = {"6"};++ ASSERT_EQ(2, results["hits"].size());+ ids = {"6", "12"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);diff --git a/test/skus.txt b/test/skus.txtnew file mode 100644index 00000000..103070a2--- /dev/null+++ b/test/skus.txt@@ -0,0 +1,24 @@+abc12345678282411+abc12345678278271+abc12345678975321+abc12345678274871+abc12345678246461+abc12345678647531+abc12345678272871+abc12345678217521+abc12345678782461+abc12345678715761+abc12345678878231+abc12345678878981+abc12345678387971+abc12345678278831+abc12345678879751+abc12345678978821+abc12345678288731+abc12345678287581+abc12345678378481+abc1234237831+abc1234878211+abc12345678428871+abc1234287821+abc1234287781\ No newline at end of file