Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed an edge case in trie fuzzy search.

kishorencfe5bdf617ce0
diff --git a/src/art.cpp b/src/art.cpp
index a77e60cd..ad5d3db7 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1258,18 +1258,20 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
goto PARTIAL_CALC;
}
- // First calculate cost with node char `c` and then leaf/partial related costs
- temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
- rotate(i, j, k);
- p = c;
+ if (!((c == '\0' && depth == term_len))) {
+ // Calculate cost with node char `c`
+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
+ rotate(i, j, k);
+ p = c;
- depth++;
+ depth++;
- printf("Recurse char: %c, cost: %d, depth: %d\n", c, cost, depth);
+ printf("Recurse char: %c, cost: %d, depth: %d\n", c, cost, depth);
- if(temp_cost > max_cost) {
- // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`
- return;
+ if(temp_cost > max_cost) {
+ // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`
+ return;
+ }
}
// Cost is under control, let's check to see if we should proceed further
@@ -1292,7 +1294,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
c = l->key[depth];
temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
printf("leaf char: %c\n", l->key[depth]);
- printf("cost: %d, depth: %d, term_len: %d\n", cost, depth, term_len);
+ printf("cost: %d, depth: %d, term_len: %d\n", temp_cost, depth, term_len);
rotate(i, j, k);
p = c;
depth++;
@@ -1333,10 +1335,12 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
// For non-prefix search or if we have not reached term length, we will recurse further
- const int partial_len = min(MAX_PREFIX_LEN, n->partial_len);
+ int partial_len = min(MAX_PREFIX_LEN, n->partial_len);
const int end_index = min(partial_len, term_len+max_cost);
printf("partial_len: %d\n", partial_len);
+
+ // calculate partial related cost
for(int idx=0; idx<end_index; idx++) {
c = n->partial[idx];
@@ -1352,8 +1356,24 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *
}
}
- depth += n->partial_len;
- printf("cost: %d\n", cost);
+ depth += partial_len;
+ printf("cost: %d\n", temp_cost);
+
+ if(n->partial_len > MAX_PREFIX_LEN) {
+ // some intermediate path has been left out, so we have to "progress" the levenshtein matrix
+ while(partial_len++ < n->partial_len) {
+ c = term[depth];
+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);
+ rotate(i, j, k);
+ p = c;
+ depth++;
+ }
+ }
+
+ if(temp_cost > max_cost) {
+ // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`
+ return;
+ }
art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], min_cost, max_cost, prefix, results);
}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index e3ab87c9..093d8b5b 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -9,6 +9,7 @@
#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()
#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()
#define skus_file_path std::string(std::string(ROOT_DIR)+"/test/skus.txt").c_str()
+#define ill_file_path std::string(std::string(ROOT_DIR)+"/test/ill.txt").c_str()
art_document get_document(uint32_t id) {
art_document document;
@@ -800,6 +801,96 @@ TEST(ArtTest, test_art_search_sku_like_tokens) {
ASSERT_TRUE(res == 0);
}
+TEST(ArtTest, test_art_search_ill_like_tokens) {
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
+
+ std::vector<std::string> keys;
+
+ int len;
+ char buf[512];
+ FILE *f = fopen(ill_file_path, "r");
+
+ uintptr_t line = 1;
+ while (fgets(buf, sizeof buf, f)) {
+ len = strlen(buf);
+ buf[len - 1] = '\0';
+ art_document doc = get_document((uint32_t) line);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));
+ keys.push_back(std::string(buf, len-1));
+ line++;
+ }
+
+ for (const auto &key : keys) {
+ //LOG(INFO) << "Searching for " << key;
+ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);
+ ASSERT_FALSE(l == nullptr);
+ EXPECT_EQ(1, l->values->ids.getLength());
+
+ std::vector<art_leaf *> leaves;
+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,
+ FREQUENCY, true, leaves);
+
+ ASSERT_EQ(1, leaves.size());
+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);
+
+ leaves.clear();
+
+ // non prefix
+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,
+ FREQUENCY, false, leaves);
+ ASSERT_EQ(1, leaves.size());
+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);
+ }
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
+}
+
+TEST(ArtTest, test_art_search_ill_like_tokens2) {
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
+
+ std::vector<std::string> keys;
+ keys = {"input", "illustrations", "illustration"};
+
+ art_document doc = get_document((uint32_t) 1);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc, 1));
+
+ doc = get_document((uint32_t) 2);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc, 1));
+
+ doc = get_document((uint32_t) 3);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc, 1));
+
+ for (const auto &key : keys) {
+ //LOG(INFO) << "Searching for " << key;
+ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);
+ ASSERT_FALSE(l == nullptr);
+ EXPECT_EQ(1, l->values->ids.getLength());
+
+ std::vector<art_leaf *> leaves;
+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,
+ FREQUENCY, true, leaves);
+
+ ASSERT_EQ(1, leaves.size());
+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);
+
+ leaves.clear();
+
+ // non prefix
+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,
+ FREQUENCY, false, leaves);
+ ASSERT_EQ(1, leaves.size());
+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);
+ }
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
+}
+
TEST(ArtTest, test_encode_int32) {
unsigned char chars[8];
diff --git a/test/ill.txt b/test/ill.txt
new file mode 100644
index 00000000..20495e5e
--- /dev/null
+++ b/test/ill.txt
@@ -0,0 +1,68 @@
+input
+illustrations
+illustration
+internet
+image
+instrument
+italic
+interface
+in
+imagev
+issued
+identification
+ideograph
+info
+information
+inbox
+idea
+inspiration
+instagram
+isolation
+individual
+indent
+interrogation
+inventor
+ipad
+intellectual
+increasemoney
+interest
+inches
+ils
+instant
+imageresizelandscape
+imageresize
+inlove
+iot
+intelligent
+islam
+id
+identity
+intercardinal
+italian
+insurance
+insect
+imageflip
+ily
+ice
+icy
+industrial
+instruments
+invoice
+impairment
+index
+inventory
+institution
+insects
+imagealtslash
+inexpressive
+imageupload
+iceland
+ios
+integration
+inputdevice
+iran
+indifference
+ignorance
+islands
+indonesia
+incredulous
\ No newline at end of file