Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fuzzy search tests.
Kishore Nallanaab59121104a
diff --git a/include/art.h b/include/art.hindex a01418a2..35f0c0df 100644--- a/include/art.h+++ b/include/art.h@@ -231,12 +231,8 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, ar/*** Returns leaves that match a given string within a fuzzy distance of max_cost.*/-int art_fuzzy_results(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,- const int max_words, std::vector<art_leaf *> &results);--static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,- const int term_len, const int* irow, const int* jrow, const int max_cost,- std::vector<const art_node *> &results);+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,+ const int max_words, std::vector<art_leaf *> &results);static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results);diff --git a/src/art.cpp b/src/art.cppindex 3e6bd7e5..9674634c 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -28,6 +28,10 @@#define microseconds std::chrono::duration_cast<std::chrono::microseconds>+static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,+ const int term_len, const int* irow, const int* jrow, const int max_cost,+ std::vector<const art_node *> &results);+/** Comparator for art_leaf struct.* We order the leaves descending based on the score.@@ -1263,8 +1267,8 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, cons/*** Returns leaves that match a given string within a fuzzy distance of max_cost.*/-int art_fuzzy_results(art_tree *t, const unsigned char *term, const int term_len,- const int max_cost, const int max_words, std::vector<art_leaf *> &results) {+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,+ const int max_cost, const int max_words, std::vector<art_leaf *> &results) {std::vector<const art_node*> nodes;int irow[term_len + 1];@@ -1275,7 +1279,14 @@ int art_fuzzy_results(art_tree *t, const unsigned char *term, const int term_len}auto begin = std::chrono::high_resolution_clock::now();- art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, nodes);++ if(IS_LEAF(t->root)) {+ art_leaf *l = (art_leaf *) LEAF_RAW(t->root);+ art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, max_cost, nodes);+ } else {+ art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, nodes);+ }+std::sort(nodes.begin(), nodes.end(), compare_art_node);long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl;diff --git a/src/collection.cpp b/src/collection.cppindex c8a32c5c..112fa851 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -111,7 +111,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_std::transform(token.begin(), token.end(), token.begin(), ::tolower);std::vector<art_leaf*> leaves;- art_fuzzy_results(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1, cost, 3, leaves);+ art_fuzzy_search(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1, cost, 3, leaves);if(!leaves.empty()) {for(auto i=0; i<leaves.size(); i++) {printf("%s - ", token.c_str());diff --git a/test/art_test.cpp b/test/art_test.cppindex c19d89a1..47294bd8 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -203,6 +203,7 @@ TEST(ArtTest, test_art_insert_delete) {// Check the minimum and maximumASSERT_TRUE(!art_minimum(&t));ASSERT_TRUE(!art_maximum(&t));+ ASSERT_TRUE(art_size(&t) == 0);res = art_tree_destroy(&t);ASSERT_TRUE(res == 0);@@ -544,4 +545,123 @@ TEST(ArtTest, test_art_delete_out_of_bounds) {res = art_tree_destroy(&t);ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_insert_multiple_ids_for_same_token) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ char* key1 = "implement";+ art_document doc = get_document((uint32_t) 1);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 1));++ doc = get_document((uint32_t) 2);+ art_values* value = (art_values*) art_insert(&t, (unsigned char*)key1, strlen(key1) + 1, &doc, 2);+ ASSERT_TRUE(value != NULL);++ doc = get_document((uint32_t) 3);+ ASSERT_TRUE(value == (art_values*) art_insert(&t, (unsigned char*)key1, strlen(key1)+1, &doc, 3));++ ASSERT_TRUE(art_size(&t) == 1);+ ASSERT_EQ(value->ids.getLength(), 3);+ ASSERT_EQ(value->ids.at(0), 1);+ ASSERT_EQ(value->ids.at(1), 2);+ ASSERT_EQ(value->ids.at(2), 3);++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}++TEST(ArtTest, test_art_fuzzy_search_single_leaf) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ char* implement_key = "implement";+ art_document doc = get_document((uint32_t) 1);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)implement_key, strlen(implement_key)+1, &doc, 1));++ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)implement_key, strlen(implement_key)+1);+ EXPECT_EQ(1, l->values->ids.at(0));++ std::vector<art_leaf*> leaves;+ art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 10, leaves);+ ASSERT_EQ(1, leaves.size());++ char* implement_key_typo1 = "implment";+ char* implement_key_typo2 = "implwnent";++ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 10, leaves);+ ASSERT_EQ(0, leaves.size());++ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 1, 10, leaves);+ ASSERT_EQ(1, leaves.size());++ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 2, 10, leaves);+ ASSERT_EQ(1, leaves.size());+}++TEST(ArtTest, test_art_fuzzy_search) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ int len;+ char buf[512];+ FILE *f = fopen("/tmp/typesense_test/words.txt", "r");++ uint64_t xor_mask = 0;+ uintptr_t line = 1, nlines;+ while (fgets(buf, sizeof buf, f)) {+ len = strlen(buf);+ buf[len-1] = '\0';+ art_document doc = get_document((uint32_t) line);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)buf, len, &doc, 1));++ xor_mask ^= (line * (buf[0] + len));+ line++;+ }++ nlines = line - 1;++ std::vector<art_leaf*> leaves;++ // transpose+ art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 1, 10, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ("zymosthenic", (const char *)leaves.at(0)->key);++ // transpose + missing+ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 2, 10, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);++ // missing char+ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 1, 10, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ("gaberlunzie", (const char *)leaves.at(0)->key);++ // extra char+ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 1, 10, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ("highliving", (const char *)leaves.at(0)->key);++ // substituted char+ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 1, 10, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ("racemiferous", (const char *)leaves.at(0)->key);++ // missing char + extra char+ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 2, 10, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ("Saarbrucken", (const char *)leaves.at(0)->key);}\ No newline at end of file