Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support excluding tokens from query.
kishorenc1bed2d7d8082
diff --git a/include/collection.h b/include/collection.hindex eac329c0..d381ea9b 100644--- a/include/collection.h+++ b/include/collection.h@@ -328,5 +328,8 @@ public:void get_doc_changes(const nlohmann::json &document, nlohmann::json &old_doc,nlohmann::json &new_doc,nlohmann::json &del_doc);++ void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,+ std::vector<std::string>& q_exclude_tokens) const;};diff --git a/include/index.h b/include/index.hindex 97a8b06c..33913276 100644--- a/include/index.h+++ b/include/index.h@@ -23,7 +23,8 @@ struct token_candidates {};struct search_args {- std::string query;+ std::vector<std::string> q_include_tokens;+ std::vector<std::string> q_exclude_tokens;std::vector<std::string> search_fields;std::vector<filter> filters;std::vector<facet> facets;@@ -54,13 +55,16 @@ struct search_args {}- search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,+ search_args(std::vector<std::string> q_include_tokens,+ std::vector<std::string> q_exclude_tokens,+ std::vector<std::string> search_fields, std::vector<filter> filters,std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,size_t drop_tokens_threshold, size_t typo_tokens_threshold,const std::vector<std::string>& group_by_fields, size_t group_limit):- query(query), search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),+ q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens),+ search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),max_facet_values(max_facet_values), per_page(per_page),page(page), token_order(token_order), prefix(prefix),@@ -152,6 +156,8 @@ private:void search_field(const uint8_t & field_id,std::vector<std::string>& query_tokens,std::vector<std::string>& search_tokens,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,size_t& num_tokens_dropped,const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,const std::vector<uint32_t>& curated_ids,@@ -163,7 +169,9 @@ private:const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD);- void search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,+ void search_candidates(const uint8_t & field_id,+ uint32_t* filter_ids, size_t filter_ids_length,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,const std::vector<uint32_t>& curated_ids,const std::vector<sort_by> & sort_fields, std::vector<token_candidates> & token_to_candidates,std::vector<std::vector<art_leaf*>> & searched_queries,@@ -202,7 +210,8 @@ private:uint32_t* collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const;- void collate_included_ids(const std::string & query, const std::string & field, const uint8_t field_id,+ void collate_included_ids(const std::vector<std::string>& q_included_tokens,+ const std::string & field, const uint8_t field_id,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries);@@ -226,7 +235,10 @@ public:void run_search();- void search(Option<uint32_t> & outcome, const std::string & query, const std::vector<std::string> & search_fields,+ void search(Option<uint32_t> & outcome,+ const std::vector<std::string> & q_include_tokens,+ const std::vector<std::string> & q_exclude_tokens,+ const std::vector<std::string> & search_fields,const std::vector<filter> & filters, std::vector<facet> & facets,facet_query_t & facet_query,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,diff --git a/src/collection.cpp b/src/collection.cppindex a7836dea..cfe5d908 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -733,10 +733,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std:://LOG(INFO) << "Num indices used for querying: " << indices.size();+ std::vector<std::string> q_include_tokens;+ std::vector<std::string> q_exclude_tokens;++ parse_search_query(query, q_include_tokens, q_exclude_tokens);+// send data to individual index threadssize_t index_id = 0;for(Index* index: indices) {- index->search_params = new search_args(query, search_fields, filters, facets,+ index->search_params = new search_args(q_include_tokens, q_exclude_tokens, search_fields, filters, facets,index_to_included_ids[index_id], index_to_excluded_ids[index_id],sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,per_page, page, token_order, prefix,@@ -1124,6 +1129,31 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::return result;}+void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,+ std::vector<std::string>& q_exclude_tokens) const {+ if(query == "*") {+ q_exclude_tokens = {};+ q_include_tokens = {query};+ } else {+ std::vector<std::string> tokens;+ StringUtils::split(query, tokens, " ");++ for(std::string& token: tokens) {+ if(token[0] == '-') {+ std::string&& just_token = token.substr(1);+ Tokenizer(just_token, false, true).tokenize(just_token);+ if(!just_token.empty()) {+ q_exclude_tokens.push_back(just_token);+ }+ } else {+ Tokenizer(token, false, true).tokenize(token);+ if(!token.empty()) {+ q_include_tokens.push_back(token);+ }+ }+ }+ }+}void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs) const {if(topster->distinct) {diff --git a/src/index.cpp b/src/index.cppindex b28656a2..f1726a23 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -883,7 +883,9 @@ void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to}}-void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,+void Index::search_candidates(const uint8_t & field_id,+ uint32_t* filter_ids, size_t filter_ids_length,+ const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,const std::vector<uint32_t>& curated_ids,const std::vector<sort_by> & sort_fields,std::vector<token_candidates> & token_candidates_vec,@@ -939,6 +941,15 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, sicontinue;}+ // Exclude document IDs associated with excluded tokens from the result set+ if(exclude_token_ids_size != 0) {+ uint32_t *excluded_result_ids = nullptr;+ result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,+ &excluded_result_ids);+ delete[] result_ids;+ result_ids = excluded_result_ids;+ }+if(!curated_ids.empty()) {uint32_t *excluded_result_ids = nullptr;result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],@@ -1266,7 +1277,8 @@ void Index::run_search() {}// after the wait, we own the lock.- search(search_params->outcome, search_params->query, search_params->search_fields,+ search(search_params->outcome, search_params->q_include_tokens, search_params->q_exclude_tokens,+ search_params->search_fields,search_params->filters, search_params->facets, search_params->facet_query,search_params->included_ids, search_params->excluded_ids,search_params->sort_fields_std, search_params->num_typos,@@ -1288,7 +1300,8 @@ void Index::run_search() {}}-void Index::collate_included_ids(const std::string & query, const std::string & field, const uint8_t field_id,+void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,+ const std::string & field, const uint8_t field_id,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,Topster* curated_topster,std::vector<std::vector<art_leaf*>> & searched_queries) {@@ -1301,11 +1314,7 @@ void Index::collate_included_ids(const std::string & query, const std::string &std::vector<art_leaf *> override_query;- Tokenizer tokenizer(query, false, true);- std::string token;- size_t token_index = 0;-- while(tokenizer.next(token, token_index)) {+ for(const std::string& token: q_included_tokens) {const size_t token_len = token.length();std::vector<art_leaf*> leaves;@@ -1343,7 +1352,8 @@ void Index::collate_included_ids(const std::string & query, const std::string &}void Index::search(Option<uint32_t> & outcome,- const std::string & query,+ const std::vector<std::string> & q_include_tokens,+ const std::vector<std::string> & q_exclude_tokens,const std::vector<std::string> & search_fields,const std::vector<filter> & filters,std::vector<facet> & facets, facet_query_t & facet_query,@@ -1392,7 +1402,7 @@ void Index::search(Option<uint32_t> & outcome,//auto begin = std::chrono::high_resolution_clock::now();uint32_t* all_result_ids = nullptr;- if(query == "*") {+ if(!q_include_tokens.empty() && q_include_tokens[0] == "*") {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);const std::string & field = search_fields[0];@@ -1433,16 +1443,36 @@ void Index::search(Option<uint32_t> & outcome,score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},groups_processed, filter_ids, filter_ids_length);- collate_included_ids(query, field, field_id, included_ids_map, curated_topster, searched_queries);+ collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);all_result_ids_len = filter_ids_length;all_result_ids = filter_ids;filter_ids = nullptr;} else {const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);+ uint32_t *exclude_token_ids = nullptr;+ size_t exclude_token_ids_size = 0;- std::vector<std::string> tokens;- Tokenizer(query, false, true).tokenize(tokens);+ // find documents that contain the excluded tokens to exclude them from results later+ for(size_t i = 0; i < num_search_fields; i++) {+ const std::string & field_name = search_fields[i];+ for(const std::string& exclude_token: q_exclude_tokens) {+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),+ (const unsigned char *) exclude_token.c_str(),+ exclude_token.size() + 1);++ if(leaf) {+ uint32_t *ids = leaf->values->ids.uncompress();+ uint32_t *exclude_token_ids_merged = nullptr;+ exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size, ids,+ leaf->values->ids.getLength(),+ &exclude_token_ids_merged);+ delete[] ids;+ delete[] exclude_token_ids;+ exclude_token_ids = exclude_token_ids_merged;+ }+ }+ }for(size_t i = 0; i < num_search_fields; i++) {// proceed to query search only when no filters are provided or when filtering produces results@@ -1450,14 +1480,15 @@ void Index::search(Option<uint32_t> & outcome,const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i); // Order of `fields` are used to sort resultsconst std::string & field = search_fields[i];- std::vector<std::string> query_tokens = tokens;- std::vector<std::string> search_tokens = tokens;+ std::vector<std::string> query_tokens = q_include_tokens;+ std::vector<std::string> search_tokens = q_include_tokens;size_t num_tokens_dropped = 0;- search_field(field_id, query_tokens, search_tokens, num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,+ field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, topster, groups_processed, &all_result_ids, all_result_ids_len,token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);- collate_included_ids(query, field, field_id, included_ids_map, curated_topster, searched_queries);+ collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);}}}@@ -1492,6 +1523,8 @@ void Index::search(Option<uint32_t> & outcome,void Index::search_field(const uint8_t & field_id,std::vector<std::string>& query_tokens,std::vector<std::string>& search_tokens,+ const uint32_t* exclude_token_ids,+ size_t exclude_token_ids_size,size_t& num_tokens_dropped,const std::string & field,uint32_t *filter_ids, size_t filter_ids_length,@@ -1605,9 +1638,9 @@ void Index::search_field(const uint8_t & field_id,if(!token_candidates_vec.empty() && token_candidates_vec.size() == search_tokens.size()) {// If all tokens were found, go ahead and search for candidates with what we have so far- search_candidates(field_id, filter_ids, filter_ids_length, curated_ids, sort_fields, token_candidates_vec,- searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,- typo_tokens_threshold);+ search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,+ curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,+ groups_processed, all_result_ids, all_result_ids_len, typo_tokens_threshold);}if (all_result_ids_len >= typo_tokens_threshold) {@@ -1640,12 +1673,10 @@ void Index::search_field(const uint8_t & field_id,}}-- return search_field(field_id, query_tokens, truncated_tokens, num_tokens_dropped, field,- filter_ids, filter_ids_length, curated_ids,- facets, sort_fields, num_typos,- searched_queries, topster, groups_processed, all_result_ids, all_result_ids_len,- token_order, prefix);+ return search_field(field_id, query_tokens, truncated_tokens, exclude_token_ids, exclude_token_ids_size,+ num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids,facets,+ sort_fields, num_typos,searched_queries, topster, groups_processed, all_result_ids,+ all_result_ids_len, token_order, prefix);}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 3f546efa..63ce3ac0 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -144,7 +144,7 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {ASSERT_EQ(0, results["found"].get<int>());}-TEST_F(CollectionTest, ExactPhraseSearch) {+TEST_F(CollectionTest, PhraseSearch) {std::vector<std::string> facets;nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, sort_fields, 0, 10).get();ASSERT_EQ(5, results["hits"].size());@@ -205,6 +205,23 @@ TEST_F(CollectionTest, ExactPhraseSearch) {}}+TEST_F(CollectionTest, SearchWithExcludedTokens) {+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("how -propellants -are", query_fields, "", facets, sort_fields, 0, 10).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ(2, results["found"].get<uint32_t>());++ std::vector<std::string> ids = {"9", "17"};++ for (size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string id = ids.at(i);+ std::string result_id = result["document"]["id"];+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+}+TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// Tokens that are not found in the index should be skippedstd::vector<std::string> facets;@@ -2596,5 +2613,18 @@ TEST_F(CollectionTest, MultiFieldRelevance) {ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));}+ // with exclude token syntax+ results = coll1->search("-downie dustin kensrue down there by the train",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ expected_ids = {2, 0};++ for(size_t i=0; i<expected_ids.size(); i++) {+ ASSERT_EQ(expected_ids[i], std::stoi(results["hits"][i]["document"]["id"].get<std::string>()));+ }+collectionManager.drop_collection("coll1");}