Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor fuzzy search state transition.

Kishore Nallan7a5c0eafda18
diff --git a/src/art.cpp b/src/art.cpp
index 2a6c99e9..f778eace 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1298,48 +1298,48 @@ static inline void rotate(int &i, int &j, int &k) {
// -1: return without adding, 0 : continue iteration, 1: return after adding
static inline int fuzzy_search_state(const bool prefix, int key_index, bool last_key_char,
- int term_len, const int* cost_row, int min_cost, int max_cost) {
+ const int query_len, const int* cost_row, int min_cost, int max_cost) {
- // a) iter_len < term_len: "pltninum" (term) on "pst" (key)
- // b) term_len < iter_len: "pst" (term) on "pltninum" (key)
+ // There are 2 scenarios:
+ // a) key_len < query_len: "pltninum" (query) on "pst" (key)
+ // b) query_len < key_len: "pst" (query) on "pltninum" (key)
- int cost = 0;
+ int key_len = last_key_char ? key_index : key_index + 1;
- // a) because key's null character will appear first
if(last_key_char) {
- int key_len = key_index;
- cost = cost_row[term_len];
-
- if(cost >= min_cost && cost <= max_cost) {
+ // Last char, so have to return 1 or -1
+ if(cost_row[query_len] >= min_cost && cost_row[query_len] <= max_cost) {
return 1;
}
- cost = cost_row[key_len];
-
- // used to match q=strawberries on key=strawberry, but limit to larger keys to prevent eager matches
- if(key_len > 5 && term_len > key_len && (term_len - key_len) <= max_cost &&
- cost >= min_cost-1 && cost <= max_cost-1) {
+ // Special case used to match q=strawberries on key=strawberry (query_len > key_len)
+ // but limit to larger keys to prevent eager matches
+ if(key_len > 5 && query_len > key_len && (query_len - key_len) <= max_cost &&
+ cost_row[key_len] >= min_cost && cost_row[key_len] <= max_cost-1) {
return 1;
}
return -1;
}
- int key_len = key_index + 1;
+ // `key_len` can't exceed `query_len` since length of `cost_row` is `query_len + 1`
+ int cost = cost_row[std::min(key_len, query_len)];
- // b) we might iterate past term_len to catch trailing typos
- if(key_len >= term_len && prefix) {
- cost = cost_row[term_len];
+ if(key_len >= query_len && prefix) {
+ // Case b)
+ // For prefix queries
+ // - we can return early if key_len reaches query_len and cost is within bounds.
+ // - might have to iterate past prefix query length to catch trailing typos.
if(cost >= min_cost && cost <= max_cost) {
return 1;
}
- } else {
- // `key_len` can't exceed `term_len` since length of `cost_row` is `term_len + 1`
- cost = cost_row[std::min(key_len, term_len)];
}
- int bounded_cost = (max_cost == 0) ? max_cost : (max_cost + 1);
- return (cost > bounded_cost) ? -1 : 0;
+ // Terminate the search early or continue iterating on the key?
+ // We have to account for the case that `cost` could momentarily exceed max_cost but resolve later.
+ // e.g. key=example, query=exZZample, after 5 chars, cost is 3 but drops to 2 at the end.
+ // But we will limit this for longer keys for performance.
+ return cost > max_cost && (key_len > 3 ? cost > (max_cost * 2) : true) ? -1 : 0;
}
static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *n, int depth, const unsigned char *term,
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 38c962db..c1030972 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -744,10 +744,9 @@ TEST(ArtTest, test_art_fuzzy_search) {
}
std::vector<art_leaf*> leaves;
-
- leaves.clear();
auto begin = std::chrono::high_resolution_clock::now();
+ leaves.clear();
art_fuzzy_search(&t, (const unsigned char *) "pltinum", strlen("pltinum"), 0, 1, 10, FREQUENCY, true, nullptr, 0, leaves);
ASSERT_EQ(2, leaves.size());
ASSERT_STREQ("platinumsmith", (const char *)leaves.at(0)->key);
@@ -800,7 +799,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, nullptr, 0, leaves);
ASSERT_EQ(10, leaves.size());
- std::set<std::string> expected_words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};
+ std::set<std::string> expected_words = {"town", "sown", "mown", "lown", "howl", "howk", "howe", "how", "horn", "hoon"};
for(size_t leaf_index = 0; leaf_index < leaves.size(); leaf_index++) {
art_leaf*& leaf = leaves.at(leaf_index);
@@ -864,6 +863,29 @@ TEST(ArtTest, test_art_fuzzy_search_unicode_chars) {
ASSERT_TRUE(res == 0);
}
+TEST(ArtTest, test_art_fuzzy_search_extra_chars) {
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
+
+ std::vector<const char*> keys = {
+ "abbviation"
+ };
+
+ for(const char* key: keys) {
+ art_document doc = get_document((uint32_t) 1);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));
+ }
+
+ const char* query = "abbreviation";
+ std::vector<art_leaf*> leaves;
+ art_fuzzy_search(&t, (unsigned char *)query, strlen(query), 0, 2, 10, FREQUENCY, true, nullptr, 0, leaves);
+ ASSERT_EQ(1, leaves.size());
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
+}
+
TEST(ArtTest, test_art_search_sku_like_tokens) {
art_tree t;
int res = art_tree_init(&t);
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index 11ce2ed8..b12e68cb 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -774,6 +774,28 @@ TEST_F(CollectionLocaleTest, SearchOnCyrillicLargeText) {
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
}
+TEST_F(CollectionLocaleTest, SearchOnArabicText) {
+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ std::string data = "جهينة";
+ std::string q = "جوهينة";
+
+ auto dchars = data.c_str();
+ auto qchars = q.c_str();
+
+ nlohmann::json doc;
+ doc["title"] = "جهينة";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("جوهينة", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();
+ LOG(INFO) << results;
+
+ ASSERT_STREQ("<mark>جهينة</mark>",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+}
+
/*
TEST_F(CollectionLocaleTest, TranslitPad) {
UErrorCode translit_status = U_ZERO_ERROR;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 93bf05a4..b72573a3 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -392,7 +392,7 @@ TEST_F(CollectionTest, QueryWithTypo) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
- ids = {"8", "1", "17"};
+ ids = {"1", "13", "8"};
ASSERT_EQ(3, results["hits"].size());
@@ -667,20 +667,20 @@ TEST_F(CollectionTest, PrefixSearching) {
}
TEST_F(CollectionTest, TypoTokensThreshold) {
- // Query expansion should happen only based on the `typo_tokens_threshold` value
- auto results = collection->search("launch", {"title"}, "", {}, sort_fields, {2}, 10, 1,
+ // Typo correction should happen only based on the `typo_tokens_threshold` value
+ auto results = collection->search("redundant", {"title"}, "", {}, sort_fields, {2}, 10, 1,
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "", 0).get();
- ASSERT_EQ(5, results["hits"].size());
- ASSERT_EQ(5, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["found"].get<size_t>());
- results = collection->search("launch", {"title"}, "", {}, sort_fields, {2}, 10, 1,
+ results = collection->search("redundant", {"title"}, "", {}, sort_fields, {2}, 10, 1,
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "", 10).get();
- ASSERT_EQ(7, results["hits"].size());
- ASSERT_EQ(7, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["found"].get<size_t>());
}
TEST_F(CollectionTest, MultiOccurrenceString) {
diff --git a/test/documents.jsonl b/test/documents.jsonl
index 8caaa289..2c68f2d2 100644
--- a/test/documents.jsonl
+++ b/test/documents.jsonl
@@ -21,4 +21,4 @@
{"points":7,"title":"What kinds of things have been tossed out of ISSS in space?"}
{"points":17,"title":"What does triple redundant closed loop digital avionics system mean?"}
{"points":11,"title":"How are rockets guided to follow specific loop trajectory?"}
-{"points":8,"title":"What do remotely controlled bolts look like?"}
\ No newline at end of file
+{"points":8,"title":"What do remotely controlled redundent bolts look like?"}
\ No newline at end of file