Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor fuzzy search state transition.
Kishore Nallan7a5c0eafda18
diff --git a/src/art.cpp b/src/art.cppindex 2a6c99e9..f778eace 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -1298,48 +1298,48 @@ static inline void rotate(int &i, int &j, int &k) {// -1: return without adding, 0 : continue iteration, 1: return after addingstatic inline int fuzzy_search_state(const bool prefix, int key_index, bool last_key_char,- int term_len, const int* cost_row, int min_cost, int max_cost) {+ const int query_len, const int* cost_row, int min_cost, int max_cost) {- // a) iter_len < term_len: "pltninum" (term) on "pst" (key)- // b) term_len < iter_len: "pst" (term) on "pltninum" (key)+ // There are 2 scenarios:+ // a) key_len < query_len: "pltninum" (query) on "pst" (key)+ // b) query_len < key_len: "pst" (query) on "pltninum" (key)- int cost = 0;+ int key_len = last_key_char ? key_index : key_index + 1;- // a) because key's null character will appear firstif(last_key_char) {- int key_len = key_index;- cost = cost_row[term_len];-- if(cost >= min_cost && cost <= max_cost) {+ // Last char, so have to return 1 or -1+ if(cost_row[query_len] >= min_cost && cost_row[query_len] <= max_cost) {return 1;}- cost = cost_row[key_len];-- // used to match q=strawberries on key=strawberry, but limit to larger keys to prevent eager matches- if(key_len > 5 && term_len > key_len && (term_len - key_len) <= max_cost &&- cost >= min_cost-1 && cost <= max_cost-1) {+ // Special case used to match q=strawberries on key=strawberry (query_len > key_len)+ // but limit to larger keys to prevent eager matches+ if(key_len > 5 && query_len > key_len && (query_len - key_len) <= max_cost &&+ cost_row[key_len] >= min_cost && cost_row[key_len] <= max_cost-1) {return 1;}return -1;}- int key_len = key_index + 1;+ // `key_len` can't exceed `query_len` since length of `cost_row` is `query_len + 1`+ int cost = cost_row[std::min(key_len, query_len)];- // b) we might iterate past term_len to catch trailing typos- if(key_len >= term_len && prefix) {- cost = cost_row[term_len];+ if(key_len >= query_len && prefix) {+ // Case b)+ // For prefix queries+ // - we can return early if key_len reaches query_len and cost is within bounds.+ // - might have to iterate past prefix query length to catch trailing typos.if(cost >= min_cost && cost <= max_cost) {return 1;}- } else {- // `key_len` can't exceed `term_len` since length of `cost_row` is `term_len + 1`- cost = cost_row[std::min(key_len, term_len)];}- int bounded_cost = (max_cost == 0) ? max_cost : (max_cost + 1);- return (cost > bounded_cost) ? -1 : 0;+ // Terminate the search early or continue iterating on the key?+ // We have to account for the case that `cost` could momentarily exceed max_cost but resolve later.+ // e.g. key=example, query=exZZample, after 5 chars, cost is 3 but drops to 2 at the end.+ // But we will limit this for longer keys for performance.+ return cost > max_cost && (key_len > 3 ? cost > (max_cost * 2) : true) ? -1 : 0;}static void art_fuzzy_recurse(unsigned char p, unsigned char c, const art_node *n, int depth, const unsigned char *term,diff --git a/test/art_test.cpp b/test/art_test.cppindex 38c962db..c1030972 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -744,10 +744,9 @@ TEST(ArtTest, test_art_fuzzy_search) {}std::vector<art_leaf*> leaves;-- leaves.clear();auto begin = std::chrono::high_resolution_clock::now();+ leaves.clear();art_fuzzy_search(&t, (const unsigned char *) "pltinum", strlen("pltinum"), 0, 1, 10, FREQUENCY, true, nullptr, 0, leaves);ASSERT_EQ(2, leaves.size());ASSERT_STREQ("platinumsmith", (const char *)leaves.at(0)->key);@@ -800,7 +799,7 @@ TEST(ArtTest, test_art_fuzzy_search) {art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, nullptr, 0, leaves);ASSERT_EQ(10, leaves.size());- std::set<std::string> expected_words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};+ std::set<std::string> expected_words = {"town", "sown", "mown", "lown", "howl", "howk", "howe", "how", "horn", "hoon"};for(size_t leaf_index = 0; leaf_index < leaves.size(); leaf_index++) {art_leaf*& leaf = leaves.at(leaf_index);@@ -864,6 +863,29 @@ TEST(ArtTest, test_art_fuzzy_search_unicode_chars) {ASSERT_TRUE(res == 0);}+TEST(ArtTest, test_art_fuzzy_search_extra_chars) {+ art_tree t;+ int res = art_tree_init(&t);+ ASSERT_TRUE(res == 0);++ std::vector<const char*> keys = {+ "abbviation"+ };++ for(const char* key: keys) {+ art_document doc = get_document((uint32_t) 1);+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));+ }++ const char* query = "abbreviation";+ std::vector<art_leaf*> leaves;+ art_fuzzy_search(&t, (unsigned char *)query, strlen(query), 0, 2, 10, FREQUENCY, true, nullptr, 0, leaves);+ ASSERT_EQ(1, leaves.size());++ res = art_tree_destroy(&t);+ ASSERT_TRUE(res == 0);+}+TEST(ArtTest, test_art_search_sku_like_tokens) {art_tree t;int res = art_tree_init(&t);diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex 11ce2ed8..b12e68cb 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -774,6 +774,28 @@ TEST_F(CollectionLocaleTest, SearchOnCyrillicLargeText) {results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());}+TEST_F(CollectionLocaleTest, SearchOnArabicText) {+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ std::string data = "جهينة";+ std::string q = "جوهينة";++ auto dchars = data.c_str();+ auto qchars = q.c_str();++ nlohmann::json doc;+ doc["title"] = "جهينة";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("جوهينة", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}).get();+ LOG(INFO) << results;++ ASSERT_STREQ("<mark>جهينة</mark>",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+}+/*TEST_F(CollectionLocaleTest, TranslitPad) {UErrorCode translit_status = U_ZERO_ERROR;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 93bf05a4..b72573a3 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -392,7 +392,7 @@ TEST_F(CollectionTest, QueryWithTypo) {spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10).get();- ids = {"8", "1", "17"};+ ids = {"1", "13", "8"};ASSERT_EQ(3, results["hits"].size());@@ -667,20 +667,20 @@ TEST_F(CollectionTest, PrefixSearching) {}TEST_F(CollectionTest, TypoTokensThreshold) {- // Query expansion should happen only based on the `typo_tokens_threshold` value- auto results = collection->search("launch", {"title"}, "", {}, sort_fields, {2}, 10, 1,+ // Typo correction should happen only based on the `typo_tokens_threshold` value+ auto results = collection->search("redundant", {"title"}, "", {}, sort_fields, {2}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "", 0).get();- ASSERT_EQ(5, results["hits"].size());- ASSERT_EQ(5, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<size_t>());- results = collection->search("launch", {"title"}, "", {}, sort_fields, {2}, 10, 1,+ results = collection->search("redundant", {"title"}, "", {}, sort_fields, {2}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "", 10).get();- ASSERT_EQ(7, results["hits"].size());- ASSERT_EQ(7, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(2, results["found"].get<size_t>());}TEST_F(CollectionTest, MultiOccurrenceString) {diff --git a/test/documents.jsonl b/test/documents.jsonlindex 8caaa289..2c68f2d2 100644--- a/test/documents.jsonl+++ b/test/documents.jsonl@@ -21,4 +21,4 @@{"points":7,"title":"What kinds of things have been tossed out of ISSS in space?"}{"points":17,"title":"What does triple redundant closed loop digital avionics system mean?"}{"points":11,"title":"How are rockets guided to follow specific loop trajectory?"}-{"points":8,"title":"What do remotely controlled bolts look like?"}\ No newline at end of file+{"points":8,"title":"What do remotely controlled redundent bolts look like?"}\ No newline at end of file