Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Make ART fuzzy_search take min_cost and max_cost instead of only max_cost.

Kishore Nallan0b88e669f677
diff --git a/TODO.md b/TODO.md
index dfb7fdc9..a9ffd0f6 100644
--- a/TODO.md
+++ b/TODO.md
@@ -21,6 +21,8 @@
- storage key prefix should include collection name
- storage key prefix should include collection name
- use art for indexing score as well
+- ISX what (score based on typo matches)
+- Mininum results should be a variable instead of blindly going with max_results
**API**
diff --git a/include/art.h b/include/art.h
index 96742748..ee7057b1 100644
--- a/include/art.h
+++ b/include/art.h
@@ -236,7 +236,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, ar
/**
* Returns leaves that match a given string within a fuzzy distance of max_cost.
*/
-int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,
+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,
const int max_words, const token_ordering token_order, const bool prefix, std::vector<art_leaf *> &results);
static int art_topk_iter(const art_node *root, token_ordering token_order, const int max_results,
diff --git a/include/collection.h b/include/collection.h
index 9e49d485..3eb10476 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -40,7 +40,7 @@ private:
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
void search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves, std::vector<nlohmann::json> & results,
- size_t & total_results, const size_t & max_results);
+ spp::sparse_hash_set<uint64_t> & dedup_seq_ids, size_t & total_results, const size_t & max_results);
void index_string_field(const std::string &field_name, art_tree *t, const nlohmann::json &document, uint32_t seq_id) const;
diff --git a/src/art.cpp b/src/art.cpp
index f0b932fd..156fc857 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -31,8 +31,8 @@
enum recurse_progress { CONTINUE, ABORT, ITERATE };
static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,
- const int term_len, const int* irow, const int* jrow, const int max_cost,
- const bool prefix, std::vector<const art_node *> &results);
+ const int term_len, const int* irow, const int* jrow, const int min_cost,
+ const int max_cost, const bool prefix, std::vector<const art_node *> &results);
void art_int_fuzzy_recurse(art_node *n, int depth, unsigned char* int_str, int int_str_len,
uint32_t compare, std::vector<const art_leaf *> &results);
@@ -1163,7 +1163,7 @@ static inline int levenshtein_dist(const int depth, const char p, const char c,
}
static inline void art_fuzzy_children(char p, const art_node *n, int depth, const unsigned char *term, const int term_len,
- const int* irow, const int* jrow, const int max_cost,
+ const int* irow, const int* jrow, const int min_cost, const int max_cost,
const bool prefix, std::vector<const art_node *> &results) {
char child_char;
art_node* child;
@@ -1175,7 +1175,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child_char = ((art_node4*)n)->keys[i];
printf("\n4!child_char: %c, %d, depth: %d", child_char, child_char, depth);
child = ((art_node4*)n)->children[i];
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);
}
break;
case NODE16:
@@ -1184,7 +1184,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child_char = ((art_node16*)n)->keys[i];
printf("\n16!child_char: %c, depth: %d", child_char, depth);
child = ((art_node16*)n)->children[i];
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);
}
break;
case NODE48:
@@ -1195,7 +1195,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child = ((art_node48*)n)->children[ix - 1];
child_char = (char)i;
printf("\n48!child_char: %c, depth: %d, ix: %d", child_char, depth, ix);
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);
}
break;
case NODE256:
@@ -1205,7 +1205,7 @@ static inline void art_fuzzy_children(char p, const art_node *n, int depth, cons
child_char = (char) i;
printf("\n256!child_char: %c, depth: %d", child_char, depth);
child = ((art_node256*)n)->children[i];
- art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, max_cost, prefix, results);
+ art_fuzzy_recurse(p, child_char, child, depth, term, term_len, irow, jrow, min_cost, max_cost, prefix, results);
}
break;
default:
@@ -1223,8 +1223,8 @@ static inline void rotate(int &i, int &j, int &k) {
// e.g. catapult against coratapult
// e.g. microafot against microsoft
static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, const unsigned char *term,
- const int term_len, const int* irow, const int* jrow, const int max_cost,
- const bool prefix, std::vector<const art_node *> &results) {
+ const int term_len, const int* irow, const int* jrow, const int min_cost,
+ const int max_cost, const bool prefix, std::vector<const art_node *> &results) {
const int columns = term_len+1;
int i=0, j=1, k=2;
int row0[columns];
@@ -1266,7 +1266,7 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, cons
}
// rows[j][columns-1] holds the final cost
- if(rows[j][columns-1] <= max_cost) {
+ if(rows[j][columns-1] >= min_cost && rows[j][columns-1] <= max_cost) {
results.push_back(n);
}
@@ -1296,13 +1296,13 @@ static void art_fuzzy_recurse(char p, char c, const art_node *n, int depth, cons
return ;
}
- art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], max_cost, prefix, results);
+ art_fuzzy_children(c, n, depth, term, term_len, rows[i], rows[j], min_cost, max_cost, prefix, results);
}
/**
* Returns leaves that match a given string within a fuzzy distance of max_cost.
*/
-int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int max_cost,
+int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, const int min_cost, const int max_cost,
const int max_words, const token_ordering token_order, const bool prefix,
std::vector<art_leaf *> &results) {
@@ -1318,9 +1318,9 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
if(IS_LEAF(t->root)) {
art_leaf *l = (art_leaf *) LEAF_RAW(t->root);
- art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);
+ art_fuzzy_recurse(0, l->key[0], t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
} else {
- art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, max_cost, prefix, nodes);
+ art_fuzzy_children(0, t->root, 0, term, term_len, irow, jrow, min_cost, max_cost, prefix, nodes);
}
if(token_order == FREQUENCY) {
diff --git a/src/collection.cpp b/src/collection.cpp
index 42245767..94c0291d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -137,15 +137,12 @@ void Collection::index_string_field(const std::string &field_name, art_tree *t,
}
void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_leaves,
- std::vector<nlohmann::json> & results, size_t & total_results,
- const size_t & max_results) {
+ std::vector<nlohmann::json> & results, spp::sparse_hash_set<uint64_t> & dedup_seq_ids,
+ size_t & total_results, const size_t & max_results) {
const size_t combination_limit = 10;
auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);
- // For deduplication. If 2 query suggestions give a document as result, ensure that only one is returned
- spp::sparse_hash_set<uint64_t> dedup_seq_ids;
-
for(long long n=0; n<N && n<combination_limit; ++n) {
// every element in `query_suggestion` contains a token and its associated hits
std::vector<art_leaf *> query_suggestion = next_suggestion(token_leaves, n);
@@ -217,7 +214,10 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
std::vector<nlohmann::json> results;
// To prevent us from doing ART search repeatedly as we iterate through possible corrections
- spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cache;
+ spp::sparse_hash_map<std::string, std::vector<art_leaf*>> token_cost_cache;
+
+ // To prevent duplicate results, while preserving order of result vector
+ spp::sparse_hash_set<uint64_t> result_set;
// Used to drop the least occurring token(s) for partial searches
spp::sparse_hash_map<std::string, uint32_t> token_to_count;
@@ -260,15 +260,16 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
const std::string token_cost_hash = token + std::to_string(costs[token_index]);
std::vector<art_leaf*> leaves;
+ std::cout << "\n**Searching for: " << token << " - cost: " << costs[token_index] << std::endl;
- if(token_cache.count(token_cost_hash) != 0) {
- leaves = token_cache[token_cost_hash];
+ if(token_cost_cache.count(token_cost_hash) != 0) {
+ leaves = token_cost_cache[token_cost_hash];
} else {
int token_len = prefix ? (int) token.length() : (int) token.length() + 1;
art_fuzzy_search(index_map.at("title"), (const unsigned char *) token.c_str(), token_len,
- costs[token_index], 3, token_order, prefix, leaves);
+ costs[token_index], costs[token_index], 3, token_order, prefix, leaves);
if(!leaves.empty()) {
- token_cache.emplace(token_cost_hash, leaves);
+ token_cost_cache.emplace(token_cost_hash, leaves);
}
}
@@ -295,7 +296,7 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
// Unless we're already at max_cost for this token, don't look at remaining tokens since we would
// see them again in a future iteration when we retry with a larger cost
- if(costs[token_index] != max_cost) {
+ if(token_index == -1 || costs[token_index] != max_cost) {
retry_with_larger_cost = true;
break;
}
@@ -307,9 +308,13 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
if(token_leaves.size() != 0 && !retry_with_larger_cost) {
// If a) all tokens were found, or b) Some were skipped because they don't exist within max_cost,
// go ahead and search for candidates with what we have so far
- search_candidates(token_leaves, results, total_results, max_results);
+ search_candidates(token_leaves, results, result_set, total_results, max_results);
+ std::cout << "total_results = " << total_results << std::endl;
+ for(auto res: results) {
+ std::cout << "RES: " << res << std::endl;
+ }
- if (total_results > 0) {
+ if (total_results >= max_results) {
// Unless there are results, we continue outerloop (looking at tokens with greater cost)
break;
}
diff --git a/test/art_test.cpp b/test/art_test.cpp
index e1c0867d..9c1f63c1 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -584,22 +584,22 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf) {
EXPECT_EQ(1, l->values->ids.at(0));
std::vector<art_leaf*> leaves;
- art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key, strlen(implement_key) + 1, 0, 0, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
const char* implement_key_typo1 = "implment";
const char* implement_key_typo2 = "implwnent";
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 0, 10, FREQUENCY, false, leaves);
ASSERT_EQ(0, leaves.size());
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 1, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo1, strlen(implement_key_typo1) + 1, 0, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 2, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) implement_key_typo2, strlen(implement_key_typo2) + 1, 0, 2, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
res = art_tree_destroy(&t);
@@ -632,43 +632,48 @@ TEST(ArtTest, test_art_fuzzy_search) {
std::vector<art_leaf*> leaves;
// transpose
- art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 1, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "zymosthneic", strlen("zymosthneic") + 1, 0, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("zymosthenic", (const char *)leaves.at(0)->key);
// transpose + missing
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 2, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 0, 2, 10, FREQUENCY, false, leaves);
+ ASSERT_EQ(1, leaves.size());
+ ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);
+
+ leaves.clear();
+ art_fuzzy_search(&t, (const unsigned char *) "dacrcyystlgia", strlen("dacrcyystlgia") + 1, 1, 2, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("dacrycystalgia", (const char *)leaves.at(0)->key);
// missing char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 1, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "gaberlunze", strlen("gaberlunze") + 1, 0, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("gaberlunzie", (const char *)leaves.at(0)->key);
// extra char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 1, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "higghliving", strlen("higghliving") + 1, 0, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("highliving", (const char *)leaves.at(0)->key);
// substituted char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 1, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "eacemiferous", strlen("eacemiferous") + 1, 0, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("racemiferous", (const char *)leaves.at(0)->key);
// missing char + extra char
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 2, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "Sarbruckken", strlen("Sarbruckken") + 1, 0, 2, 10, FREQUENCY, false, leaves);
ASSERT_EQ(1, leaves.size());
ASSERT_STREQ("Saarbrucken", (const char *)leaves.at(0)->key);
// multiple matching results
leaves.clear();
- art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 1, 10, FREQUENCY, false, leaves);
+ art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, leaves);
ASSERT_EQ(10, leaves.size());
std::vector<const char*> words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index fb7334e2..0be7ac76 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -89,7 +89,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
}
// with non-zero cost
- results = collection->search("DoesNotExist from", 2, 10);
+ results = collection->search("DoesNotExist from", 1, 10);
ASSERT_EQ(2, results.size());
for(size_t i = 0; i < results.size(); i++) {
@@ -100,11 +100,16 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
}
// with 2 indexed words
- results = collection->search("from DoesNotExist insTruments", 2, 10);
- ASSERT_EQ(1, results.size());
- nlohmann::json result = results.at(0);
- std::string result_id = result["id"];
- ASSERT_STREQ("2", result_id.c_str());
+ results = collection->search("from DoesNotExist insTruments", 1, 10);
+ ASSERT_EQ(2, results.size());
+ ids = {"2", "17"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string id = ids.at(i);
+ std::string result_id = result["id"];
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
results.clear();
results = collection->search("DoesNotExist1 DoesNotExist2", 0, 10);
@@ -208,9 +213,9 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {
TEST_F(CollectionTest, TextContainingAnActualTypo) {
// A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens
std::vector<nlohmann::json> results = collection->search("ISX what", 1, 10, FREQUENCY, false);
- ASSERT_EQ(4, results.size());
+ ASSERT_EQ(5, results.size());
- std::vector<std::string> ids = {"19", "6", "21", "8"};
+ std::vector<std::string> ids = {"20", "19", "6", "21", "8"};
for(size_t i = 0; i < results.size(); i++) {
nlohmann::json result = results.at(i);