Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fuzzy prefix searching should rely on the intermediate cost rather than the final cost.
Kishore Nalland4d1a9e3bd46
diff --git a/src/art.cpp b/src/art.cppindex daa53a0a..541a3c99 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1267,8 +1267,8 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, consprintf("\nIS_LEAF\nLEAF KEY: %s, depth: %d\n", l->key, depth);/*- For prefix search, when key is longer than term, we could potentially iterate till `term_len+max_cost` for:- term = `th`, leaf = `mathematics` - if we compared only first 2 chars, exceeds max_cost+ For prefix search, when key is longer than term, we could potentially iterate till `term_len+max_cost`. E.g:+ term = `th`, key = `mathematics` - if we compared only first 2 chars, it will exceed max_costHowever, we refrain from doing so for performance reasons, or atleast until we hear strong objections.Also, for prefix searches we don't compare with full leaf key.@@ -1286,9 +1286,22 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, consdepth++;}- // rows[j][columns-1] holds the final cost- if(rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {+ /* `rows[j][columns-1]` holds the final cost, `cost` holds the intermediate cost.+ We will use the intermediate cost if the term is shorter than the key.+ */+ if(prefix && term_len < (int) l->key_len && cost >= min_cost && cost <= max_cost) {+ results.push_back(n);+ return;+ }++ if(prefix && term_len >= (int) l->key_len && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {results.push_back(n);+ return;+ }++ if(!prefix && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {+ results.push_back(n);+ return;}return ;@@ -1341,7 +1354,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,irow[i] = i;}- auto begin = std::chrono::high_resolution_clock::now();+ //auto begin = std::chrono::high_resolution_clock::now();if(IS_LEAF(t->root)) {art_leaf *l = (art_leaf *) LEAF_RAW(t->root);@@ -1359,10 +1372,10 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,std::sort(nodes.begin(), nodes.end(), compare_art_node_score);}- long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();+ //long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();//!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();- begin = std::chrono::high_resolution_clock::now();+ //begin = std::chrono::high_resolution_clock::now();for(auto node: nodes) {art_topk_iter(node, token_order, max_words, results);@@ -1374,7 +1387,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,std::sort(results.begin(), results.end(), compare_art_leaf_score);}- time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();+ //time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();//!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us";return 0;}diff --git a/test/art_test.cpp b/test/art_test.cppindex 8491f745..926dd838 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -608,6 +608,26 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {ASSERT_TRUE(res == 0);}+TEST(ArtTest, test_art_fuzzy_search_single_leaf_prefix) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ const char* key = "application";+ art_document doc = get_document((uint32_t) 1);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc, 1));++ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key, strlen(key)+1);+ EXPECT_EQ(1, l->values->ids.at(0));++ std::vector<art_leaf*> leaves;+ art_fuzzy_search(&t, (const unsigned char *) "aplication", strlen(key), 0, 1, 10, FREQUENCY, true, leaves);+ ASSERT_EQ(1, leaves.size());++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}+TEST(ArtTest, test_art_fuzzy_search) {art_tree t;int res = art_tree_init(&t);@@ -697,7 +717,7 @@ TEST(ArtTest, test_art_fuzzy_search) {leaves.clear();art_fuzzy_search(&t, (const unsigned char *) "antisocao", strlen("antisocao"), 0, 2, 10, FREQUENCY, true, leaves);- ASSERT_EQ(6, leaves.size());+ ASSERT_EQ(8, leaves.size());res = art_tree_destroy(&t);ASSERT_TRUE(res == 0);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 219a4c83..6c614b8f 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -479,7 +479,15 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_EQ(0, results["hits"].size());results = collection->search("xq", query_fields, "", facets, sort_fields, 2, 2, 1, FREQUENCY, true).get();- ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(1, results["hits"].size());+ ids = {"6"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }// prefix with a typoresults = collection->search("late propx", query_fields, "", facets, sort_fields, 2, 1, 1, FREQUENCY, true).get();