Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Make ART fuzzy_search take min_cost and max_cost instead of only max_cost.
Kishore Nallan0b88e669f677
diff --git a/TODO.md b/TODO.mdindex dfb7fdc9..a9ffd0f6 100644--- a/TODO.md+++ b/TODO.md@@ -21,6 +21,8 @@- storage key prefix should include collection name- storage key prefix should include collection name- use art for indexing score as well+- ISX what (score based on typo matches)+- Mininum results should be a variable instead of blindly going with max_results**API**diff --git a/include/art.h b/include/art.hindex 96742748..ee7057b1 100644--- a/include/art.h+++ b/include/art.h@@ -236,7 +236,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, ar/*** Returns leaves that match a given string within a fuzzy distance of max_cost.*/-int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,const int max_words, const token_ordering token_order, const bool prefix, std::vector<art_leaf *> &results);static int art_topk_iter(const art_node *root, token_ordering token_order, const int max_results,diff --git a/include/collection.h b/include/collection.hindex 9e49d485..3eb10476 100644--- a/include/collection.h+++ b/include/collection.h@@ -40,7 +40,7 @@ private:void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, std::vector<nlohmann::json> & results,- size_t & total_results, const size_t & max_results);+ spp::sparse_hash_set<uint64_t> & dedup_seq_ids, size_t & total_results, const size_t & max_results);void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;diff --git a/src/art.cpp b/src/art.cppindex f0b932fd..156fc857 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -31,8 +31,8 @@enum recurse_progress { CONTINUE, ABORT, ITERATE };static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,- const int term_len, const int* irow, const int* jrow, const int max_cost,- const bool prefix, std::vector<const art_node *> &results);+ const int term_len, const int* irow, const int* jrow, const int min_cost,+ const int max_cost, const bool prefix, std::vector<const art_node *> &results);void art_int_fuzzy_recurse(art_node *n, int depth, unsigned char* int_str, int int_str_len,uint32_t compare, std::vector<const art_leaf *> &results);@@ -1163,7 +1163,7 @@ static inline int levenshtein_dist(const int depth, const char p, const char c,}static inline void art_fuzzy_children(char p, const art_node *n, int depth, const unsigned char *term, const int term_len,- const int* irow, const int* jrow, const int max_cost,+ const int* irow, const int* jrow, const int min_cost, const int max_cost,const bool prefix, std::vector<const art_node *> &results) {char child_char;art_node* child;@@ -1175,7 +1175,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild_char = ((art_node4*)n)->keys[i];printf("\n4!child_char: %c, %d, depth: %d", child_char, child_char, depth);child = ((art_node4*)n)->children[i];- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);}break;case NODE16:@@ -1184,7 +1184,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild_char = ((art_node16*)n)->keys[i];printf("\n16!child_char: %c, depth: %d", child_char, depth);child = ((art_node16*)n)->children[i];- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);}break;case NODE48:@@ -1195,7 +1195,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild = ((art_node48*)n)->children[ix - 1];child_char = (char)i;printf("\n48!child_char: %c, depth: %d, ix: %d", child_char, depth, ix);- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);}break;case NODE256:@@ -1205,7 +1205,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, conschild_char = (char) i;printf("\n256!child_char: %c, depth: %d", child_char, depth);child = ((art_node256*)n)->children[i];- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);}break;default:@@ -1223,8 +1223,8 @@ static inline void rotate(int &i, int &j, int &k) {// e.g. catapult against coratapult// e.g. microafot against microsoftstatic void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,- const int term_len, const int* irow, const int* jrow, const int max_cost,- const bool prefix, std::vector<const art_node *> &results) {+ const int term_len, const int* irow, const int* jrow, const int min_cost,+ const int max_cost, const bool prefix, std::vector<const art_node *> &results) {const int columns = term_len+1;int i=0, j=1, k=2;int row0[columns];@@ -1266,7 +1266,7 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, cons}// rows[j][columns-1] holds the final cost- if(rows[j][columns-1] <= max_cost) {+ if(rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {results.push_back(n);}@@ -1296,13 +1296,13 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, consreturn ;}- art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], max_cost, prefix, results);+ art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], min_cost, max_cost, prefix, results);}/*** Returns leaves that match a given string within a fuzzy distance of max_cost.*/-int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,const int max_words, const token_ordering token_order, const bool prefix,std::vector<art_leaf *> &results) {@@ -1318,9 +1318,9 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,if(IS_LEAF(t->root)) {art_leaf *l = (art_leaf *) LEAF_RAW(t->root);- art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);+ art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);} else {- art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);+ art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);}if(token_order == FREQUENCY) {diff --git a/src/collection.cpp b/src/collection.cppindex 42245767..94c0291d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -137,15 +137,12 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,}void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves,- std::vector<nlohmann::json> & results, size_t & total_results,- const size_t & max_results) {+ std::vector<nlohmann::json> & results, spp::sparse_hash_set<uint64_t> & dedup_seq_ids,+ size_t & total_results, const size_t & max_results) {const size_t combination_limit = 10;auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);- // For deduplication. If 2 query suggestions give a document as result, ensure that only one is returned- spp::sparse_hash_set<uint64_t> dedup_seq_ids;-for(long long n=0; n<N && n<combination_limit; ++n) {// every element in `query_suggestion` contains a token and its associated hitsstd::vector<art_leaf *> query_suggestion = next_suggestion(token_leaves, n);@@ -217,7 +214,10 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_std::vector<nlohmann::json> results;// To prevent us from doing ART search repeatedly as we iterate through possible corrections- spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cache;+ spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;++ // To prevent duplicate results, while preserving order of result vector+ spp::sparse_hash_set<uint64_t> result_set;// Used to drop the least occurring token(s) for partial searchesspp::sparse_hash_map<std::string, uint32_t> token_to_count;@@ -260,15 +260,16 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_const std::string token_cost_hash = token + std::to_string(costs[token_index]);std::vector<art_leaf*> leaves;+ std::cout << "\n**Searching for: " << token << " - cost: " << costs[token_index] << std::endl;- if(token_cache.count(token_cost_hash) != 0) {- leaves = token_cache[token_cost_hash];+ if(token_cost_cache.count(token_cost_hash) != 0) {+ leaves = token_cost_cache[token_cost_hash];} else {int token_len = prefix ? (int) token.length() : (int) token.length() + 1;art_fuzzy_search(index_map.at("title"), (const unsigned char *) token.c_str(), token_len,- costs[token_index], 3, token_order, prefix, leaves);+ costs[token_index], costs[token_index], 3, token_order, prefix, leaves);if(!leaves.empty()) {- token_cache.emplace(token_cost_hash, leaves);+ token_cost_cache.emplace(token_cost_hash, leaves);}}@@ -295,7 +296,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_// Unless we're already at max_cost for this token, don't look at remaining tokens since we would// see them again in a future iteration when we retry with a larger cost- if(costs[token_index] != max_cost) {+ if(token_index == -1 || costs[token_index] != max_cost) {retry_with_larger_cost = true;break;}@@ -307,9 +308,13 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_if(token_leaves.size() != 0 && !retry_with_larger_cost) {// If a) all tokens were found, or b) Some were skipped because they don't exist within max_cost,// go ahead and search for candidates with what we have so far- search_candidates(token_leaves, results, total_results, max_results);+ search_candidates(token_leaves, results, result_set, total_results, max_results);+ std::cout << "total_results = " << total_results << std::endl;+ for(auto res: results) {+ std::cout << "RES: " << res << std::endl;+ }- if (total_results > 0) {+ if (total_results >= max_results) {// Unless there are results, we continue outerloop (looking at tokens with greater cost)break;}diff --git a/test/art_test.cpp b/test/art_test.cppindex e1c0867d..9c1f63c1 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -584,22 +584,22 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {EXPECT_EQ(1, l->values->ids.at(0));std::vector<art_leaf*> leaves;- art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 0, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());const char* implement_key_typo1 = "implment";const char* implement_key_typo2 = "implwnent";leaves.clear();- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 0, 10, FREQUENCY, false, leaves);ASSERT_EQ(0, leaves.size());leaves.clear();- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 1, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());leaves.clear();- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 2, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 0, 2, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());res = art_tree_destroy(&t);@@ -632,43 +632,48 @@ TEST(ArtTest, test_art_fuzzy_search) {std::vector<art_leaf*> leaves;// transpose- art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 1, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 0, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("zymosthenic", (const char *)leaves.at(0)->key);// transpose + missingleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 2, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 0, 2, 10, FREQUENCY, false, leaves);+ ASSERT_EQ(1, leaves.size());+ ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);++ leaves.clear();+ art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 1, 2, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);// missing charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 1, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 0, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("gaberlunzie", (const char *)leaves.at(0)->key);// extra charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 1, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 0, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("highliving", (const char *)leaves.at(0)->key);// substituted charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 1, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 0, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("racemiferous", (const char *)leaves.at(0)->key);// missing char + extra charleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 2, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 0, 2, 10, FREQUENCY, false, leaves);ASSERT_EQ(1, leaves.size());ASSERT_STREQ("Saarbrucken", (const char *)leaves.at(0)->key);// multiple matching resultsleaves.clear();- art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 1, 10, FREQUENCY, false, leaves);+ art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, leaves);ASSERT_EQ(10, leaves.size());std::vector<const char*> words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};diff --git a/test/collection_test.cpp b/test/collection_test.cppindex fb7334e2..0be7ac76 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -89,7 +89,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}// with non-zero cost- results = collection->search("DoesNotExist from", 2, 10);+ results = collection->search("DoesNotExist from", 1, 10);ASSERT_EQ(2, results.size());for(size_t i = 0; i < results.size(); i++) {@@ -100,11 +100,16 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}// with 2 indexed words- results = collection->search("from DoesNotExist insTruments", 2, 10);- ASSERT_EQ(1, results.size());- nlohmann::json result = results.at(0);- std::string result_id = result["id"];- ASSERT_STREQ("2", result_id.c_str());+ results = collection->search("from DoesNotExist insTruments", 1, 10);+ ASSERT_EQ(2, results.size());+ ids = {"2", "17"};++ for(size_t i = 0; i < results.size(); i++) {+ nlohmann::json result = results.at(i);+ std::string id = ids.at(i);+ std::string result_id = result["id"];+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }results.clear();results = collection->search("DoesNotExist1 DoesNotExist2", 0, 10);@@ -208,9 +213,9 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {TEST_F(CollectionTest, TextContainingAnActualTypo) {// A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happensstd::vector<nlohmann::json> results = collection->search("ISX what", 1, 10, FREQUENCY, false);- ASSERT_EQ(4, results.size());+ ASSERT_EQ(5, results.size());- std::vector<std::string> ids = {"19", "6", "21", "8"};+ std::vector<std::string> ids = {"20", "19", "6", "21", "8"};for(size_t i = 0; i < results.size(); i++) {nlohmann::json result = results.at(i);