Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed an edge case in fuzzy search with SKU-like tokens.

kishorenca3024a73422b
diff --git a/src/art.cpp b/src/art.cpp
index a500817b..7ac945c0 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1148,23 +1148,22 @@ static inline int levenshtein_dist(const int depth, const unsigned char p, const
const unsigned char* term, const int term_len,
const int* irow, const int* jrow, int* krow) {
int row_min = std::numeric_limits<int>::max();
- const int columns = term_len+1;
krow[0] = jrow[0] + 1;
- // Calculate levenshtein distance incrementally (j => column, b => term):
+ // Calculate levenshtein distance incrementally (term => b, column => j, c => a[i], p => a[i-1], irow => d[i-1]):
// https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance#Optimal_string_alignment_distance
- for(int column=1; column<columns; column++) {
+ for(int column=1; column<=term_len; column++) {
+ int cost = (c == term[column-1]) ? 0 : 1; // column-1 used because of zero-based char array
+
int delete_cost = jrow[column] + 1;
int insert_cost = krow[column - 1] + 1;
+ int substitution_cost = jrow[column - 1] + cost;
- int cost = (c != term[column-1]) ? 1 : 0;
- int replace_cost = jrow[column - 1] + cost;
+ krow[column] = min(min(insert_cost, delete_cost), substitution_cost);
- krow[column] = min(min(insert_cost, delete_cost), replace_cost);
-
- if(depth > 1 && column > 1 && c == term[column-2] && p == term[column-1]) {
- krow[column] = std::min(krow[column], irow[column-2] + cost);
+ if(depth > 1 && column > 1 && c == term[column-1-1] && p == term[column-1]) {
+ krow[column] = std::min(krow[column], irow[column-2] + 1);
}
if(krow[column] < row_min) {
@@ -1175,7 +1174,7 @@ static inline int levenshtein_dist(const int depth, const unsigned char p, const
return row_min;
}
-static inline void art_fuzzy_children(char p, const art_node *n, int depth, const unsigned char *term, const int term_len,
+static inline void art_fuzzy_children(unsigned char p, const art_node *n, int depth, const unsigned char *term, const int term_len,
const int* irow, const int* jrow, const int min_cost, const int max_cost,
const bool prefix, std::vector<const art_node *> &results) {
char child_char;
@@ -1238,6 +1237,8 @@ static inline void rotate(int &i, int &j, int &k) {
static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *n, int depth, const unsigned char *term,
const int term_len, const int* irow, const int* jrow, const int min_cost,
const int max_cost, const bool prefix, std::vector<const art_node *> &results) {
+ if (!n) return ;
+
const int columns = term_len+1;
int i=0, j=1, k=2;
int row0[columns];
@@ -1248,7 +1249,15 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
copyIntArray2(irow, rows[i], columns);
copyIntArray2(jrow, rows[j], columns);
- int cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
+ int temp_cost = 0;
+
+ if(depth == -1) {
+ depth = 0;
+ goto PARTIAL_CALC;
+ }
+
+ // First calculate cost with node char `c` and then leaf/partial related costs
+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
rotate(i, j, k);
p = c;
@@ -1256,12 +1265,12 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
printf("Recurse char: %c, cost: %d, depth: %d\n", c, cost, depth);
- if(cost > max_cost) {
- // We do this to speed up things drastically, but at the cost of missing out on some genuine typos
+ if(temp_cost > max_cost) {
+ // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`
return;
}
- if (!n) return ;
+ // Cost is under control, let's check to see if we should proceed further
if(IS_LEAF(n)) {
art_leaf *l = (art_leaf *) LEAF_RAW(n);
@@ -1276,10 +1285,10 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
*/
const int end_index = prefix ? min(l->key_len, term_len) : l->key_len;
- // If at any point, `cost > 2*max_cost` we can terminate immediately as we can never recover from that
- while(depth < end_index && cost <= 2*max_cost) {
+ // If at any point, `temp_cost > 2*max_cost` we can terminate immediately as we can never recover from that
+ while(depth < end_index && temp_cost <= 2*max_cost) {
c = l->key[depth];
- cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
printf("leaf char: %c\n", l->key[depth]);
printf("cost: %d, depth: %d, term_len: %d\n", cost, depth, term_len);
rotate(i, j, k);
@@ -1287,20 +1296,24 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
depth++;
}
- /* `rows[j][columns-1]` holds the final cost, `cost` holds the intermediate cost.
- We will use the intermediate cost if the term is shorter than the key.
+ /* `rows[j][columns-1]` holds the final cost, `temp_cost` holds the temporary cost.
+ We will use the intermediate cost if the term is shorter than the key and if it's a prefix search.
+ For non-prefix, we will only use final cost.
*/
- if(prefix && term_len < (int) l->key_len && cost >= min_cost && cost <= max_cost) {
+
+ int final_cost = rows[j][columns-1];
+
+ if(prefix && term_len < (int) l->key_len && temp_cost >= min_cost && temp_cost <= max_cost) {
results.push_back(n);
return;
}
- if(prefix && term_len >= (int) l->key_len && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {
+ if(prefix && term_len >= (int) l->key_len && final_cost >= min_cost && final_cost <= max_cost) {
results.push_back(n);
return;
}
- if(!prefix && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {
+ if(!prefix && final_cost >= min_cost && final_cost <= max_cost) {
results.push_back(n);
return;
}
@@ -1308,12 +1321,16 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
return ;
}
- if(prefix && depth >= term_len && rows[j][columns-1] <= max_cost) {
- // For a prefix search, we store the node and not recurse further right now
+ // For a prefix search whose depth has reached term length, we need not recurse further
+ if(prefix && depth >= term_len) {
results.push_back(n);
return ;
}
+ PARTIAL_CALC:
+
+ // For non-prefix search or if we have not reached term length, we will recurse further
+
const int partial_len = min(MAX_PREFIX_LEN, n->partial_len);
const int end_index = min(partial_len, term_len+max_cost);
@@ -1322,12 +1339,11 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
for(int idx=0; idx<end_index; idx++) {
c = n->partial[idx];
printf("partial: %c\n", c);
- rows[k][0] = rows[j][0] + 1;
- cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
+ temp_cost = levenshtein_dist(depth+idx, p, c, term, term_len, rows[i], rows[j], rows[k]);
rotate(i, j, k);
p = c;
-
- if(prefix && depth+idx+1 >= term_len && rows[j][columns-1] <= max_cost) {
+
+ if(prefix && depth+idx+1 >= term_len && temp_cost <= max_cost) {
// For a prefix search, we store the node and not recurse further right now
results.push_back(n);
return ;
@@ -1351,8 +1367,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
int irow[term_len + 1];
int jrow[term_len + 1];
for (int i = 0; i <= term_len; i++){
- jrow[i] = i;
- irow[i] = i;
+ irow[i] = jrow[i] = i;
}
//auto begin = std::chrono::high_resolution_clock::now();
@@ -1361,12 +1376,16 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
art_leaf *l = (art_leaf *) LEAF_RAW(t->root);
art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
} else {
- if(t->root == NULL) {
+ if(t->root == nullptr) {
return 0;
}
- art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
+
+ // send depth as -1 to indicate that this is a root node
+ art_fuzzy_recurse(0, 0, t->root, -1, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
}
+ PROCESS_NODES:
+
if(token_order == FREQUENCY) {
std::sort(nodes.begin(), nodes.end(), compare_art_node_frequency);
} else {
diff --git a/test/art_test.cpp b/test/art_test.cpp
index ca690fb9..6ede5656 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -8,6 +8,7 @@
#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()
#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()
+#define skus_file_path std::string(std::string(ROOT_DIR)+"/test/skus.txt").c_str()
art_document get_document(uint32_t id) {
art_document document;
@@ -717,7 +718,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
leaves.clear();
art_fuzzy_search(&t, (const unsigned char *) "antisocao", strlen("antisocao"), 0, 2, 10, FREQUENCY, true, leaves);
- ASSERT_EQ(8, leaves.size());
+ ASSERT_EQ(10, leaves.size());
res = art_tree_destroy(&t);
ASSERT_TRUE(res == 0);
@@ -742,8 +743,57 @@ TEST(ArtTest, test_art_fuzzy_search_unicode_chars) {
EXPECT_EQ(1, l->values->ids.at(0));
std::vector<art_leaf*> leaves;
- art_fuzzy_search(&t, (unsigned char *)key, strlen(key), 0, 1, 10, FREQUENCY, true, leaves);
+ art_fuzzy_search(&t, (unsigned char *)key, strlen(key), 0, 0, 10, FREQUENCY, true, leaves);
+ ASSERT_EQ(1, leaves.size());
+ }
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
+}
+
+TEST(ArtTest, test_art_search_sku_like_tokens) {
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
+
+
+ std::vector<std::string> keys;
+ int len;
+ char buf[512];
+ FILE *f = fopen(skus_file_path, "r");
+
+ uintptr_t line = 1;
+ while (fgets(buf, sizeof buf, f)) {
+ len = strlen(buf);
+ buf[len - 1] = '\0';
+ art_document doc = get_document((uint32_t) line);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));
+ keys.push_back(std::string(buf, len-1));
+ line++;
+ }
+
+ const char* key1 = "abc12345678217521";
+
+ // exact search
+ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key1, strlen(key1)+1);
+ EXPECT_EQ(1, l->values->ids.getLength());
+
+ // exact search all tokens via fuzzy API
+
+ for (const auto &key : keys) {
+ std::vector<art_leaf *> leaves;
+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,
+ FREQUENCY, true, leaves);
+ ASSERT_EQ(1, leaves.size());
+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);
+
+ leaves.clear();
+
+ // non prefix
+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,
+ FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);
}
res = art_tree_destroy(&t);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 27dea864..23269f1f 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -524,8 +524,9 @@ TEST_F(CollectionTest, PrefixSearching) {
ASSERT_EQ(0, results["hits"].size());
results = collection->search("xq", query_fields, "", facets, sort_fields, 2, 2, 1, FREQUENCY, true).get();
- ASSERT_EQ(1, results["hits"].size());
- ids = {"6"};
+
+ ASSERT_EQ(2, results["hits"].size());
+ ids = {"6", "12"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
diff --git a/test/skus.txt b/test/skus.txt
new file mode 100644
index 00000000..103070a2
--- /dev/null
+++ b/test/skus.txt
@@ -0,0 +1,24 @@
+abc12345678282411
+abc12345678278271
+abc12345678975321
+abc12345678274871
+abc12345678246461
+abc12345678647531
+abc12345678272871
+abc12345678217521
+abc12345678782461
+abc12345678715761
+abc12345678878231
+abc12345678878981
+abc12345678387971
+abc12345678278831
+abc12345678879751
+abc12345678978821
+abc12345678288731
+abc12345678287581
+abc12345678378481
+abc1234237831
+abc1234878211
+abc12345678428871
+abc1234287821
+abc1234287781
\ No newline at end of file