Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle case when all query tokens are exclusions.
kishorencfc989f1ac292
diff --git a/src/collection.cpp b/src/collection.cppindex 2d7c9873..f1f79e97 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1152,6 +1152,11 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s}}}++ if(q_include_tokens.empty()) {+ // this can happen if the only query token is an exclusion token+ q_include_tokens.emplace_back("*");+ }}}@@ -1284,6 +1289,10 @@ void Collection::highlight_result(const field &search_field,std::vector<uint32_t*> leaf_to_indices;std::vector<art_leaf*> query_suggestion;+ if(searched_queries.size() <= field_order_kv->query_index) {+ return ;+ }+for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {// Must search for the token string fresh on that field for the given document since `token_leaf`// is from the best matched field and need not be present in other fields of a document.diff --git a/src/index.cpp b/src/index.cppindex 7db552ad..ed18b108 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1325,6 +1325,31 @@ void Index::search(Option<uint32_t> & outcome,//auto begin = std::chrono::high_resolution_clock::now();uint32_t* all_result_ids = nullptr;+ const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);+ uint32_t *exclude_token_ids = nullptr;+ size_t exclude_token_ids_size = 0;++ // find documents that contain the excluded tokens to exclude them from results later+ for(size_t i = 0; i < num_search_fields; i++) {+ const std::string & field_name = search_fields[i];+ for(const std::string& exclude_token: q_exclude_tokens) {+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),+ (const unsigned char *) exclude_token.c_str(),+ exclude_token.size() + 1);++ if(leaf) {+ uint32_t *ids = leaf->values->ids.uncompress();+ uint32_t *exclude_token_ids_merged = nullptr;+ exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size, ids,+ leaf->values->ids.getLength(),+ &exclude_token_ids_merged);+ delete[] ids;+ delete[] exclude_token_ids;+ exclude_token_ids = exclude_token_ids_merged;+ }+ }+ }+if(!q_include_tokens.empty() && q_include_tokens[0] == "*") {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);const std::string & field = search_fields[0];@@ -1364,6 +1389,20 @@ void Index::search(Option<uint32_t> & outcome,filter_ids = excluded_result_ids;}+ // Exclude document IDs associated with excluded tokens from the result set+ if(exclude_token_ids_size != 0) {+ if(filters.empty()) {+ // filter ids populated from hash map will not be sorted, but sorting is required for intersection+ std::sort(filter_ids, filter_ids+filter_ids_length);+ }++ uint32_t *excluded_result_ids = nullptr;+ filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, exclude_token_ids,+ exclude_token_ids_size, &excluded_result_ids);+ delete[] filter_ids;+ filter_ids = excluded_result_ids;+ }+score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},groups_processed, filter_ids, filter_ids_length);collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);@@ -1372,31 +1411,6 @@ void Index::search(Option<uint32_t> & outcome,all_result_ids = filter_ids;filter_ids = nullptr;} else {- const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);- uint32_t *exclude_token_ids = nullptr;- size_t exclude_token_ids_size = 0;-- // find documents that contain the excluded tokens to exclude them from results later- for(size_t i = 0; i < num_search_fields; i++) {- const std::string & field_name = search_fields[i];- for(const std::string& exclude_token: q_exclude_tokens) {- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),- (const unsigned char *) exclude_token.c_str(),- exclude_token.size() + 1);-- if(leaf) {- uint32_t *ids = leaf->values->ids.uncompress();- uint32_t *exclude_token_ids_merged = nullptr;- exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size, ids,- leaf->values->ids.getLength(),- &exclude_token_ids_merged);- delete[] ids;- delete[] exclude_token_ids;- exclude_token_ids = exclude_token_ids_merged;- }- }- }-for(size_t i = 0; i < num_search_fields; i++) {// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.empty() || filter_ids_length > 0) {@@ -1416,10 +1430,10 @@ void Index::search(Option<uint32_t> & outcome,collate_included_ids(q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);}}-- delete [] exclude_token_ids;}+ delete [] exclude_token_ids;+do_facets(facets, facet_query, all_result_ids, all_result_ids_len);do_facets(facets, facet_query, &included_ids[0], included_ids.size());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 63b2701b..7ecdcbf5 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -220,6 +220,15 @@ TEST_F(CollectionTest, SearchWithExcludedTokens) {std::string result_id = result["document"]["id"];ASSERT_STREQ(id.c_str(), result_id.c_str());}++ results = collection->search("-rocket", query_fields, "", facets, sort_fields, 0, 50).get();++ ASSERT_EQ(21, results["found"].get<uint32_t>());+ ASSERT_EQ(21, results["hits"].size());++ results = collection->search("-rocket -cryovolcanism", query_fields, "", facets, sort_fields, 0, 50).get();++ ASSERT_EQ(20, results["found"].get<uint32_t>());}TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {