Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Respect drop_tokens_threshold even when that token does not exist in the index.

Kishore Nallanb186816ca57e
diff --git a/src/art.cpp b/src/art.cpp
index 80a0cd28..daa53a0a 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -905,11 +905,11 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
std::vector<art_leaf *> &results) {
printf("INSIDE art_topk_iter: root->type: %d\n", root->type);
- std::priority_queue<art_node *, std::vector<const art_node *>,
+ std::priority_queue<const art_node *, std::vector<const art_node *>,
decltype(&compare_art_node_score_pq)> q(compare_art_node_score_pq);
if(token_order == FREQUENCY) {
- q = std::priority_queue<art_node *, std::vector<const art_node *>,
+ q = std::priority_queue<const art_node *, std::vector<const art_node *>,
decltype(&compare_art_node_frequency_pq)>(compare_art_node_frequency_pq);
}
diff --git a/src/index.cpp b/src/index.cpp
index 08bf7b64..2a73a6fe 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -331,7 +331,7 @@ void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, st
// every element in `query_suggestion` contains a token and its associated hits
std::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);
- /*for(auto i=0; i < query_suggestion.size(); i++) {
+ /*for(size_t i=0; i < query_suggestion.size(); i++) {
LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key;
}*/
@@ -715,7 +715,12 @@ void Index::search_field(std::string & query, const std::string & field, uint32_
if(it != token_to_costs[token_index].end()) {
token_to_costs[token_index].erase(it);
- // no more costs left for this token, clean up
+ // when no more costs are left for this token and `drop_tokens_threshold` is breached
+ if(token_to_costs[token_index].empty() && topster.size >= drop_tokens_threshold) {
+ break;
+ }
+
+ // otherwise, we try to drop the token and search with remaining tokens
if(token_to_costs[token_index].empty()) {
token_to_costs.erase(token_to_costs.begin()+token_index);
tokens.erase(tokens.begin()+token_index);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 5e4ccbd1..c503916d 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -223,6 +223,10 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
+ results.clear();
+ results = collection->search("the a DoesNotExist", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false, 0).get();
+ ASSERT_EQ(0, results["hits"].size());
+
// with no indexed word
results.clear();
results = collection->search("DoesNotExist1 DoesNotExist2", query_fields, "", facets, sort_fields, 0, 10).get();