Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix prefix based searches.

Kishore Nallan17a7297384e1
diff --git a/src/art.cpp b/src/art.cpp
index 0b5064cc..2b652fe4 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -883,27 +883,21 @@ static uint32_t get_score(art_node* child) {
return child->max_token_count;
}
-static int topk_iter(const art_node *root, int term_len, int k, std::vector<art_leaf*> & results) {
+static int topk_iter(const art_node *root, int max_results, std::vector<art_leaf*> & results) {
printf("INSIDE topk_iter: root->type: %d\n", root->type);
std::priority_queue<art_node *, std::vector<const art_node *>,
std::function<bool(const art_node*, const art_node*)>> q(compare_art_node_frequency);
q.push(root);
- while(!q.empty() && results.size() < k) {
+ while(!q.empty() && results.size() < max_results) {
art_node *n = (art_node *) q.top();
q.pop();
if (!n) continue;
if (IS_LEAF(n)) {
art_leaf *l = (art_leaf *) LEAF_RAW(n);
- //printf("\nTOPK LEAF: %.*s", l->key_len, l->key);
- //std::cout << ", SCORE: " << l->token_count << std::endl;
- // This basically ignores tokens whose prefix match perfectly, but exceeding overall threshold
- int diff = term_len - (int) l->key_len;
- if(diff >= -2 && diff <= 2) {
- results.push_back(l);
- }
+ results.push_back(l);
continue;
}
@@ -1326,7 +1320,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
begin = std::chrono::high_resolution_clock::now();
for(auto node: nodes) {
- topk_iter(node, term_len, max_words, results);
+ topk_iter(node, max_words, results);
}
if(token_order == FREQUENCY) {
diff --git a/src/collection.cpp b/src/collection.cpp
index ecc746ea..a3508ab1 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -87,8 +87,11 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_l
auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };
long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);
+ // For deduplication. If 2 query suggestions give a document as result, ensure that only one is returned
+ spp::sparse_hash_set<uint64_t> dedup_seq_ids;
+
for(long long n=0; n<N && n<combination_limit; ++n) {
- // every element in `query_suggestion` represents a token and its associated hits
+ // every element in `query_suggestion` contains a token and its associated hits
std::vector<art_leaf *> query_suggestion = next_suggestion(token_leaves, n);
/*std:: cout << "\nSuggestion: ";
@@ -122,10 +125,13 @@ void Collection::search_candidates(std::vector<std::vector<art_leaf*>> & token_l
for (uint32_t i = 0; i < topster.size; i++) {
uint64_t seq_id = topster.getKeyAt(i);
- std::string value;
- store->get(get_seq_id_key((uint32_t) seq_id), value);
- nlohmann::json document = nlohmann::json::parse(value);
- results.push_back(document);
+ if(dedup_seq_ids.count(seq_id) == 0) {
+ std::string value;
+ store->get(get_seq_id_key((uint32_t) seq_id), value);
+ nlohmann::json document = nlohmann::json::parse(value);
+ results.push_back(document);
+ dedup_seq_ids.emplace(seq_id);
+ }
}
if(total_results >= max_results) break;
@@ -202,7 +208,8 @@ std::vector<nlohmann::json> Collection::search(std::string query, const int num_
if(token_cache.count(token_cost_hash) != 0) {
leaves = token_cache[token_cost_hash];
} else {
- art_fuzzy_search(&t, (const unsigned char *) token.c_str(), (int) token.length() + 1,
+ int token_len = prefix ? (int) token.length() : (int) token.length() + 1;
+ art_fuzzy_search(&t, (const unsigned char *) token.c_str(), token_len,
costs[token_index], 3, token_order, prefix, leaves);
if(!leaves.empty()) {
token_cache.emplace(token_cost_hash, leaves);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 81c4c65e..a6d9f351 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -192,6 +192,30 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {
std::vector<std::string> ids = {"19", "6", "21", "8"};
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+}
+
+TEST_F(CollectionTest, PrefixSearching) {
+ std::vector<nlohmann::json> results = collection->search("ex", 0, 10, FREQUENCY, true);
+ ASSERT_EQ(2, results.size());
+ std::vector<std::string> ids = {"12", "6"};
+
+ for(size_t i = 0; i < results.size(); i++) {
+ nlohmann::json result = results.at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ results = collection->search("ex", 0, 10, MAX_SCORE, true);
+ ASSERT_EQ(2, results.size());
+ ids = {"6", "12"};
+
for(size_t i = 0; i < results.size(); i++) {
nlohmann::json result = results.at(i);
std::string result_id = result["id"];