Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fuzzy prefix searching should rely on the intermediate cost rather than the final cost.

Kishore Nalland4d1a9e3bd46
diff --git a/src/art.cpp b/src/art.cpp
index daa53a0a..541a3c99 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1267,8 +1267,8 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, cons
printf("\nIS_LEAF\nLEAF KEY: %s, depth: %d\n", l->key, depth);
/*
- For prefix search, when key is longer than term, we could potentially iterate till `term_len+max_cost` for:
- term = `th`, leaf = `mathematics` - if we compared only first 2 chars, exceeds max_cost
+ For prefix search, when key is longer than term, we could potentially iterate till `term_len+max_cost`. E.g:
+ term = `th`, key = `mathematics` - if we compared only first 2 chars, it will exceed max_cost
However, we refrain from doing so for performance reasons, or atleast until we hear strong objections.
Also, for prefix searches we don't compare with full leaf key.
@@ -1286,9 +1286,22 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, cons
depth++;
}
- // rows[j][columns-1] holds the final cost
- if(rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {
+ /* `rows[j][columns-1]` holds the final cost, `cost` holds the intermediate cost.
+ We will use the intermediate cost if the term is shorter than the key.
+ */
+ if(prefix && term_len < (int) l->key_len && cost >= min_cost && cost <= max_cost) {
+ results.push_back(n);
+ return;
+ }
+
+ if(prefix && term_len >= (int) l->key_len && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {
results.push_back(n);
+ return;
+ }
+
+ if(!prefix && rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {
+ results.push_back(n);
+ return;
}
return ;
@@ -1341,7 +1354,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
irow[i] = i;
}
- auto begin = std::chrono::high_resolution_clock::now();
+ //auto begin = std::chrono::high_resolution_clock::now();
if(IS_LEAF(t->root)) {
art_leaf *l = (art_leaf *) LEAF_RAW(t->root);
@@ -1359,10 +1372,10 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
std::sort(nodes.begin(), nodes.end(), compare_art_node_score);
}
- long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
+ //long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
//!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();
- begin = std::chrono::high_resolution_clock::now();
+ //begin = std::chrono::high_resolution_clock::now();
for(auto node: nodes) {
art_topk_iter(node, token_order, max_words, results);
@@ -1374,7 +1387,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
std::sort(results.begin(), results.end(), compare_art_leaf_score);
}
- time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
+ //time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
//!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us";
return 0;
}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 8491f745..926dd838 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -608,6 +608,26 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {
ASSERT_TRUE(res == 0);
}
+TEST(ArtTest, test_art_fuzzy_search_single_leaf_prefix) {
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
+
+ const char* key = "application";
+ art_document doc = get_document((uint32_t) 1);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc, 1));
+
+ art_leaf* l = (art_leaf *) art_search(&t, (const unsigned char *)key, strlen(key)+1);
+ EXPECT_EQ(1, l->values->ids.at(0));
+
+ std::vector<art_leaf*> leaves;
+ art_fuzzy_search(&t, (const unsigned char *) "aplication", strlen(key), 0, 1, 10, FREQUENCY, true, leaves);
+ ASSERT_EQ(1, leaves.size());
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
+}
+
TEST(ArtTest, test_art_fuzzy_search) {
art_tree t;
int res = art_tree_init(&t);
@@ -697,7 +717,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
leaves.clear();
art_fuzzy_search(&t, (const unsigned char *) "antisocao", strlen("antisocao"), 0, 2, 10, FREQUENCY, true, leaves);
- ASSERT_EQ(6, leaves.size());
+ ASSERT_EQ(8, leaves.size());
res = art_tree_destroy(&t);
ASSERT_TRUE(res == 0);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 219a4c83..6c614b8f 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -479,7 +479,15 @@ TEST_F(CollectionTest, PrefixSearching) {
ASSERT_EQ(0, results["hits"].size());
results = collection->search("xq", query_fields, "", facets, sort_fields, 2, 2, 1, FREQUENCY, true).get();
- ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(1, results["hits"].size());
+ ids = {"6"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["document"]["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
// prefix with a typo
results = collection->search("late propx", query_fields, "", facets, sort_fields, 2, 1, 1, FREQUENCY, true).get();