Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support excluding tokens from query.

kishorenc1bed2d7d8082
diff --git a/include/collection.h b/include/collection.h
index eac329c0..d381ea9b 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -328,5 +328,8 @@ public:
void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,
nlohmann::json &new_doc,
nlohmann::json &del_doc);
+
+ void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
+ std::vector<std::string>& q_exclude_tokens) const;
};
diff --git a/include/index.h b/include/index.h
index 97a8b06c..33913276 100644
--- a/include/index.h
+++ b/include/index.h
@@ -23,7 +23,8 @@ struct token_candidates {
};
struct search_args {
- std::string query;
+ std::vector<std::string> q_include_tokens;
+ std::vector<std::string> q_exclude_tokens;
std::vector<std::string> search_fields;
std::vector<filter> filters;
std::vector<facet> facets;
@@ -54,13 +55,16 @@ struct search_args {
}
- search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,
+ search_args(std::vector<std::string> q_include_tokens,
+ std::vector<std::string> q_exclude_tokens,
+ std::vector<std::string> search_fields, std::vector<filter> filters,
std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,
std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,
size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,
size_t drop_tokens_threshold, size_t typo_tokens_threshold,
const std::vector<std::string>& group_by_fields, size_t group_limit):
- query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
+ q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens),
+ search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),
max_facet_values(max_facet_values), per_page(per_page),
page(page), token_order(token_order), prefix(prefix),
@@ -152,6 +156,8 @@ private:
void search_field(const uint8_t & field_id,
std::vector<std::string>& query_tokens,
std::vector<std::string>& search_tokens,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
size_t& num_tokens_dropped,
const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,
const std::vector<uint32_t>& curated_ids,
@@ -163,7 +169,9 @@ private:
const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD);
- void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,
+ void search_candidates(const uint8_t & field_id,
+ uint32_t* filter_ids, size_t filter_ids_length,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
const std::vector<uint32_t>& curated_ids,
const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,
std::vector<std::vector<art_leaf*>> & searched_queries,
@@ -202,7 +210,8 @@ private:
uint32_t* collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const;
- void collate_included_ids(const std::string & query, const std::string & field, const uint8_t field_id,
+ void collate_included_ids(const std::vector<std::string>& q_included_tokens,
+ const std::string & field, const uint8_t field_id,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);
@@ -226,7 +235,10 @@ public:
void run_search();
- void search(Option<uint32_t> & outcome, const std::string & query, const std::vector<std::string> & search_fields,
+ void search(Option<uint32_t> & outcome,
+ const std::vector<std::string> & q_include_tokens,
+ const std::vector<std::string> & q_exclude_tokens,
+ const std::vector<std::string> & search_fields,
const std::vector<filter> & filters, std::vector<facet> & facets,
facet_query_t & facet_query,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
diff --git a/src/collection.cpp b/src/collection.cpp
index a7836dea..cfe5d908 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -733,10 +733,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
//LOG(INFO) << "Num indices used for querying: " << indices.size();
+ std::vector<std::string> q_include_tokens;
+ std::vector<std::string> q_exclude_tokens;
+
+ parse_search_query(query, q_include_tokens, q_exclude_tokens);
+
// send data to individual index threads
size_t index_id = 0;
for(Index* index: indices) {
- index->search_params = new search_args(query, search_fields, filters, facets,
+ index->search_params = new search_args(q_include_tokens, q_exclude_tokens, search_fields, filters, facets,
index_to_included_ids[index_id], index_to_excluded_ids[index_id],
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
per_page, page, token_order, prefix,
@@ -1124,6 +1129,31 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
return result;
}
+void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
+ std::vector<std::string>& q_exclude_tokens) const {
+ if(query == "*") {
+ q_exclude_tokens = {};
+ q_include_tokens = {query};
+ } else {
+ std::vector<std::string> tokens;
+ StringUtils::split(query, tokens, " ");
+
+ for(std::string& token: tokens) {
+ if(token[0] == '-') {
+ std::string&& just_token = token.substr(1);
+ Tokenizer(just_token, false, true).tokenize(just_token);
+ if(!just_token.empty()) {
+ q_exclude_tokens.push_back(just_token);
+ }
+ } else {
+ Tokenizer(token, false, true).tokenize(token);
+ if(!token.empty()) {
+ q_include_tokens.push_back(token);
+ }
+ }
+ }
+ }
+}
void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) const {
if(topster->distinct) {
diff --git a/src/index.cpp b/src/index.cpp
index b28656a2..f1726a23 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -883,7 +883,9 @@ void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to
}
}
-void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,
+void Index::search_candidates(const uint8_t & field_id,
+ uint32_t* filter_ids, size_t filter_ids_length,
+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
const std::vector<uint32_t>& curated_ids,
const std::vector<sort_by> & sort_fields,
std::vector<token_candidates> & token_candidates_vec,
@@ -939,6 +941,15 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
continue;
}
+ // Exclude document IDs associated with excluded tokens from the result set
+ if(exclude_token_ids_size != 0) {
+ uint32_t *excluded_result_ids = nullptr;
+ result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,
+ &excluded_result_ids);
+ delete[] result_ids;
+ result_ids = excluded_result_ids;
+ }
+
if(!curated_ids.empty()) {
uint32_t *excluded_result_ids = nullptr;
result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],
@@ -1266,7 +1277,8 @@ void Index::run_search() {
}
// after the wait, we own the lock.
- search(search_params->outcome, search_params->query, search_params->search_fields,
+ search(search_params->outcome, search_params->q_include_tokens, search_params->q_exclude_tokens,
+ search_params->search_fields,
search_params->filters, search_params->facets, search_params->facet_query,
search_params->included_ids, search_params->excluded_ids,
search_params->sort_fields_std, search_params->num_typos,
@@ -1288,7 +1300,8 @@ void Index::run_search() {
}
}
-void Index::collate_included_ids(const std::string & query, const std::string & field, const uint8_t field_id,
+void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,
+ const std::string & field, const uint8_t field_id,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
Topster* curated_topster,
std::vector<std::vector<art_leaf*>> & searched_queries) {
@@ -1301,11 +1314,7 @@ void Index::collate_included_ids(const std::string & query, const std::string &
std::vector<art_leaf *> override_query;
- Tokenizer tokenizer(query, false, true);
- std::string token;
- size_t token_index = 0;
-
- while(tokenizer.next(token, token_index)) {
+ for(const std::string& token: q_included_tokens) {
const size_t token_len = token.length();
std::vector<art_leaf*> leaves;
@@ -1343,7 +1352,8 @@ void Index::collate_included_ids(const std::string & query, const std::string &
}
void Index::search(Option<uint32_t> & outcome,
- const std::string & query,
+ const std::vector<std::string> & q_include_tokens,
+ const std::vector<std::string> & q_exclude_tokens,
const std::vector<std::string> & search_fields,
const std::vector<filter> & filters,
std::vector<facet> & facets, facet_query_t & facet_query,
@@ -1392,7 +1402,7 @@ void Index::search(Option<uint32_t> & outcome,
//auto begin = std::chrono::high_resolution_clock::now();
uint32_t* all_result_ids = nullptr;
- if(query == "*") {
+ if(!q_include_tokens.empty() && q_include_tokens[0] == "*") {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
const std::string & field = search_fields[0];
@@ -1433,16 +1443,36 @@ void Index::search(Option<uint32_t> & outcome,
score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
groups_processed, filter_ids, filter_ids_length);
- collate_included_ids(query, field, field_id, included_ids_map, curated_topster, searched_queries);
+ collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
all_result_ids_len = filter_ids_length;
all_result_ids = filter_ids;
filter_ids = nullptr;
} else {
const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);
+ uint32_t *exclude_token_ids = nullptr;
+ size_t exclude_token_ids_size = 0;
- std::vector<std::string> tokens;
- Tokenizer(query, false, true).tokenize(tokens);
+ // find documents that contain the excluded tokens to exclude them from results later
+ for(size_t i = 0; i < num_search_fields; i++) {
+ const std::string & field_name = search_fields[i];
+ for(const std::string& exclude_token: q_exclude_tokens) {
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
+ (const unsigned char *) exclude_token.c_str(),
+ exclude_token.size() + 1);
+
+ if(leaf) {
+ uint32_t *ids = leaf->values->ids.uncompress();
+ uint32_t *exclude_token_ids_merged = nullptr;
+ exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size, ids,
+ leaf->values->ids.getLength(),
+ &exclude_token_ids_merged);
+ delete[] ids;
+ delete[] exclude_token_ids;
+ exclude_token_ids = exclude_token_ids_merged;
+ }
+ }
+ }
for(size_t i = 0; i < num_search_fields; i++) {
// proceed to query search only when no filters are provided or when filtering produces results
@@ -1450,14 +1480,15 @@ void Index::search(Option<uint32_t> & outcome,
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort results
const std::string & field = search_fields[i];
- std::vector<std::string> query_tokens = tokens;
- std::vector<std::string> search_tokens = tokens;
+ std::vector<std::string> query_tokens = q_include_tokens;
+ std::vector<std::string> search_tokens = q_include_tokens;
size_t num_tokens_dropped = 0;
- search_field(field_id, query_tokens, search_tokens, num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
+ field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,
token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
- collate_included_ids(query, field, field_id, included_ids_map, curated_topster, searched_queries);
+ collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
}
}
}
@@ -1492,6 +1523,8 @@ void Index::search(Option<uint32_t> & outcome,
void Index::search_field(const uint8_t & field_id,
std::vector<std::string>& query_tokens,
std::vector<std::string>& search_tokens,
+ const uint32_t* exclude_token_ids,
+ size_t exclude_token_ids_size,
size_t& num_tokens_dropped,
const std::string & field,
uint32_t *filter_ids, size_t filter_ids_length,
@@ -1605,9 +1638,9 @@ void Index::search_field(const uint8_t & field_id,
if(!token_candidates_vec.empty() && token_candidates_vec.size() == search_tokens.size()) {
// If all tokens were found, go ahead and search for candidates with what we have so far
- search_candidates(field_id, filter_ids, filter_ids_length, curated_ids, sort_fields, token_candidates_vec,
- searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,
- typo_tokens_threshold);
+ search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,
+ curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,
+ groups_processed, all_result_ids, all_result_ids_len, typo_tokens_threshold);
}
if (all_result_ids_len >= typo_tokens_threshold) {
@@ -1640,12 +1673,10 @@ void Index::search_field(const uint8_t & field_id,
}
}
-
- return search_field(field_id, query_tokens, truncated_tokens, num_tokens_dropped, field,
- filter_ids, filter_ids_length, curated_ids,
- facets, sort_fields, num_typos,
- searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,
- token_order, prefix);
+ return search_field(field_id, query_tokens, truncated_tokens, exclude_token_ids, exclude_token_ids_size,
+ num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids,facets,
+ sort_fields, num_typos,searched_queries, topster, groups_processed, all_result_ids,
+ all_result_ids_len, token_order, prefix);
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 3f546efa..63ce3ac0 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -144,7 +144,7 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {
ASSERT_EQ(0, results["found"].get<int>());
}
-TEST_F(CollectionTest, ExactPhraseSearch) {
+TEST_F(CollectionTest, PhraseSearch) {
std::vector<std::string> facets;
nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, sort_fields, 0, 10).get();
ASSERT_EQ(5, results["hits"].size());
@@ -205,6 +205,23 @@ TEST_F(CollectionTest, ExactPhraseSearch) {
}
}
+TEST_F(CollectionTest, SearchWithExcludedTokens) {
+ std::vector<std::string> facets;
+ nlohmann::json results = collection->search("how -propellants -are", query_fields, "", facets, sort_fields, 0, 10).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ(2, results["found"].get<uint32_t>());
+
+ std::vector<std::string> ids = {"9", "17"};
+
+ for (size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string id = ids.at(i);
+ std::string result_id = result["document"]["id"];
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+}
+
TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
// Tokens that are not found in the index should be skipped
std::vector<std::string> facets;
@@ -2596,5 +2613,18 @@ TEST_F(CollectionTest, MultiFieldRelevance) {
ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));
}
+ // with exclude token syntax
+ results = coll1->search("-downie dustin kensrue down there by the train",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ expected_ids = {2, 0};
+
+ for(size_t i=0; i<expected_ids.size(); i++) {
+ ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));
+ }
+
collectionManager.drop_collection("coll1");
}