Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed an edge case in trie fuzzy search.
kishorencfe5bdf617ce0
diff --git a/src/art.cpp b/src/art.cppindex a77e60cd..ad5d3db7 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1258,18 +1258,20 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *goto PARTIAL_CALC;}- // First calculate cost with node char `c` and then leaf/partial related costs- temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);- rotate(i, j, k);- p = c;+ if (!((c == '\0' && depth == term_len))) {+ // Calculate cost with node char `c`+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ rotate(i, j, k);+ p = c;- depth++;+ depth++;- printf("Recurse char: %c, cost: %d, depth: %d\n", c, cost, depth);+ printf("Recurse char: %c, cost: %d, depth: %d\n", c, cost, depth);- if(temp_cost > max_cost) {- // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`- return;+ if(temp_cost > max_cost) {+ // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`+ return;+ }}// Cost is under control, let's check to see if we should proceed further@@ -1292,7 +1294,7 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *c = l->key[depth];temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);printf("leaf char: %c\n", l->key[depth]);- printf("cost: %d, depth: %d, term_len: %d\n", cost, depth, term_len);+ printf("cost: %d, depth: %d, term_len: %d\n", temp_cost, depth, term_len);rotate(i, j, k);p = c;depth++;@@ -1333,10 +1335,12 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *// For non-prefix search or if we have not reached term length, we will recurse further- const int partial_len = min(MAX_PREFIX_LEN, n->partial_len);+ int partial_len = min(MAX_PREFIX_LEN, n->partial_len);const int end_index = min(partial_len, term_len+max_cost);printf("partial_len: %d\n", partial_len);++ // calculate partial related costfor(int idx=0; idx<end_index; idx++) {c = n->partial[idx];@@ -1352,8 +1356,24 @@ static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *}}- depth += n->partial_len;- printf("cost: %d\n", cost);+ depth += partial_len;+ printf("cost: %d\n", temp_cost);++ if(n->partial_len > MAX_PREFIX_LEN) {+ // some intermediate path has been left out, so we have to "progress" the levenshtein matrix+ while(partial_len++ < n->partial_len) {+ c = term[depth];+ temp_cost = levenshtein_dist(depth, p, c, term, term_len, rows[i], rows[j], rows[k]);+ rotate(i, j, k);+ p = c;+ depth++;+ }+ }++ if(temp_cost > max_cost) {+ // Speeds up things drastically, but can miss out on "front-loaded" typos like `kumputer`+ return;+ }art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], min_cost, max_cost, prefix, results);}diff --git a/test/art_test.cpp b/test/art_test.cppindex e3ab87c9..093d8b5b 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -9,6 +9,7 @@#define words_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/words.txt").c_str()#define uuid_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/uuid.txt").c_str()#define skus_file_path std::string(std::string(ROOT_DIR)+"/test/skus.txt").c_str()+#define ill_file_path std::string(std::string(ROOT_DIR)+"/test/ill.txt").c_str()art_document get_document(uint32_t id) {art_document document;@@ -800,6 +801,96 @@ TEST(ArtTest, test_art_search_sku_like_tokens) {ASSERT_TRUE(res == 0);}+TEST(ArtTest, test_art_search_ill_like_tokens) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ std::vector<std::string> keys;++ int len;+ char buf[512];+ FILE *f = fopen(ill_file_path, "r");++ uintptr_t line = 1;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len - 1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) buf, len, &doc, 1));+ keys.push_back(std::string(buf, len-1));+ line++;+ }++ for (const auto &key : keys) {+ //LOG(INFO) << "Searching for " << key;+ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);+ ASSERT_FALSE(l == nullptr);+ EXPECT_EQ(1, l->values->ids.getLength());++ std::vector<art_leaf *> leaves;+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,+ FREQUENCY, true, leaves);++ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);++ leaves.clear();++ // non prefix+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,+ FREQUENCY, false, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);+ }++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_search_ill_like_tokens2) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ std::vector<std::string> keys;+ keys = {"input", "illustrations", "illustration"};++ art_document doc = get_document((uint32_t) 1);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[0].c_str(), keys[0].size()+1, &doc, 1));++ doc = get_document((uint32_t) 2);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[1].c_str(), keys[1].size()+1, &doc, 1));++ doc = get_document((uint32_t) 3);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char *) keys[2].c_str(), keys[2].size()+1, &doc, 1));++ for (const auto &key : keys) {+ //LOG(INFO) << "Searching for " << key;+ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key.c_str(), key.size()+1);+ ASSERT_FALSE(l == nullptr);+ EXPECT_EQ(1, l->values->ids.getLength());++ std::vector<art_leaf *> leaves;+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,+ FREQUENCY, true, leaves);++ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);++ leaves.clear();++ // non prefix+ art_fuzzy_search(&t, (const unsigned char*)key.c_str(), key.size()+1, 0, 0, 10,+ FREQUENCY, false, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ(key.c_str(), (const char *) leaves.at(0)->key);+ }++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}+TEST(ArtTest, test_encode_int32) {unsigned char chars[8];diff --git a/test/ill.txt b/test/ill.txtnew file mode 100644index 00000000..20495e5e--- /dev/null+++ b/test/ill.txt@@ -0,0 +1,68 @@+input+illustrations+illustration+internet+image+instrument+italic+interface+in+imagev+issued+identification+ideograph+info+information+inbox+idea+inspiration+isolation+individual+indent+interrogation+inventor+ipad+intellectual+increasemoney+interest+inches+ils+instant+imageresizelandscape+imageresize+inlove+iot+intelligent+islam+id+identity+intercardinal+italian+insurance+insect+imageflip+ily+ice+icy+industrial+instruments+invoice+impairment+index+inventory+institution+insects+imagealtslash+inexpressive+imageupload+iceland+ios+integration+inputdevice+iran+indifference+ignorance+islands+indonesia+incredulous\ No newline at end of file