Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix multi-field phrase search.
Kishore Nallan880b7d218582
diff --git a/include/index.h b/include/index.hindex cc214d28..6b5edc19 100644--- a/include/index.h+++ b/include/index.h@@ -738,7 +738,7 @@ public:const std::string& fallback_field_type,const DIRTY_VALUES& dirty_values);- void search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,+ void search_wildcard(const std::vector<filter>& filters,const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,spp::sparse_hash_set<uint64_t>& groups_processed,diff --git a/src/index.cpp b/src/index.cppindex f7de9758..ee2c5cc6 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2071,65 +2071,11 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,uint32_t* exclude_token_ids = nullptr;size_t exclude_token_ids_size = 0;- uint32_t* phrase_match_ids = nullptr;- size_t phrase_match_ids_size = 0;-// handle exclude tokens and phrase search if neededfor(size_t i = 0; i < num_search_fields; i++) {const std::string & field_name = search_fields[i].name;bool is_array = search_schema.at(field_name).is_array();- // do AND of `q_phrases` to add to filters- for(const auto& phrase: field_query_tokens[i].q_phrases) {- std::vector<void*> posting_lists;- for(const std::string& token: phrase) {- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),- (const unsigned char *) token.c_str(),- token.size() + 1);-- if(leaf) {- posting_lists.push_back(leaf->values);- }- }-- if(posting_lists.size() != phrase.size()) {- // unmatched length should mean zero results should be returned- delete [] filter_ids;- filter_ids = nullptr;- filter_ids_length = 0;- return;- }-- std::vector<uint32_t> contains_ids;- posting_t::intersect(posting_lists, contains_ids);-- uint32_t* phrase_ids = new uint32_t[contains_ids.size()];- size_t phrase_ids_size = 0;- posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),- phrase_ids, phrase_ids_size);-- if(phrase_ids_size != 0) {- if(filter_ids_length == 0) {- filter_ids_length = phrase_ids_size;- filter_ids = phrase_ids;- } else {- uint32_t *filter_ids_merged = nullptr;- filter_ids_length = ArrayUtils::and_scalar(phrase_ids, phrase_ids_size, filter_ids, filter_ids_length,- &filter_ids_merged);- delete [] phrase_ids;- delete [] filter_ids;- filter_ids = filter_ids_merged;- }- } else {- // no results should be returned- delete [] phrase_ids;- delete [] filter_ids;- filter_ids = nullptr;- filter_ids_length = 0;- return;- }- }-for(const auto& q_exclude_phrase: field_query_tokens[i].q_exclude_tokens) {// if phrase has multiple words, then we have to do exclusion of phrase match resultsstd::vector<void*> posting_lists;@@ -2167,15 +2113,18 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,std::vector<Topster*> ftopsters;- if (!field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&- field_query_tokens[0].q_include_tokens[0] == "*") {+ auto is_wildcard_query = !field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&+ field_query_tokens[0].q_include_tokens[0] == "*";++ // for phrase query, parser will set field_query_tokens to "*", need to handle that+ if (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);const std::string& field = search_fields[0].name;curate_filtered_ids(filters, curated_ids, exclude_token_ids,exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);- search_wildcard(field_query_tokens[0].q_include_tokens, filters, included_ids_map, sort_fields_std, topster,+ search_wildcard(filters, included_ids_map, sort_fields_std, topster,curated_topster, groups_processed, searched_queries, group_limit, group_by_fields,curated_ids, curated_ids_sorted,exclude_token_ids, exclude_token_ids_size, field_id, field,@@ -2204,13 +2153,80 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,q_pos_synonyms.emplace_back(q_pos_syn);}+ // handle phrase search by ANDing `q_phrases` to add to filters+ const std::string & field_name = search_fields[i].name;+ bool is_array = search_schema.at(field_name).is_array();++ uint32_t* phrase_match_ids = nullptr;+ size_t phrase_match_ids_size = 0;++ for(const auto& phrase: field_query_tokens[i].q_phrases) {+ std::vector<void*> posting_lists;++ for(const std::string& token: phrase) {+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),+ (const unsigned char *) token.c_str(),+ token.size() + 1);++ if(leaf) {+ posting_lists.push_back(leaf->values);+ }+ }++ if(posting_lists.size() != phrase.size()) {+ // unmatched length means no matches will be found+ continue;+ }++ std::vector<uint32_t> contains_ids;+ posting_t::intersect(posting_lists, contains_ids);++ uint32_t* phrase_ids = new uint32_t[contains_ids.size()];+ size_t phrase_ids_size = 0;+ posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),+ phrase_ids, phrase_ids_size);++ if(phrase_ids_size != 0) {+ if(phrase_match_ids_size == 0) {+ phrase_match_ids_size = phrase_ids_size;+ phrase_match_ids = phrase_ids;+ } else {+ uint32_t *phrase_ids_merged = nullptr;+ phrase_match_ids_size = ArrayUtils::and_scalar(phrase_ids, phrase_ids_size, phrase_match_ids,+ phrase_match_ids_size, &phrase_ids_merged);+ delete [] phrase_match_ids;+ delete [] phrase_ids;+ phrase_match_ids = phrase_ids_merged;+ }+ } else {+ delete [] phrase_ids;+ continue;+ }+ }++ auto actual_filter_ids = filter_ids;+ auto actual_filter_ids_length = filter_ids_length;++ if(phrase_match_ids_size != 0) {+ if(filter_ids_length == 0) {+ actual_filter_ids_length = phrase_match_ids_size;+ actual_filter_ids = phrase_match_ids;+ } else {+ actual_filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, phrase_match_ids,+ phrase_match_ids_size, &actual_filter_ids);++ delete [] phrase_match_ids;+ phrase_match_ids = nullptr;+ }+ }+// num_typos is already validated upstream, but still playing safeint field_num_typos = (i < num_typos.size()) ? num_typos[i] : num_typos[0];bool field_prefix = (i < prefixes.size()) ? prefixes[i] : prefixes[0];// proceed to query search only when no filters are provided or when filtering produces results- if(filters.empty() || filter_ids_length > 0) {+ if(filters.empty() || actual_filter_ids_length > 0) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);const std::string& field_name = search_fields[i].name;@@ -2234,14 +2250,28 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,size_t field_num_results = 0;std::set<uint64> query_hashes;- search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size,- num_tokens_dropped, field_it->second, field_name,- filter_ids, filter_ids_length, curated_ids_sorted, sort_fields_std,- field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,- field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,- query_hashes, token_order, field_prefix,- drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,- min_len_1typo, min_len_2typo);+ if(!is_wildcard_query) {+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size,+ num_tokens_dropped, field_it->second, field_name,+ actual_filter_ids, actual_filter_ids_length, curated_ids_sorted, sort_fields_std,+ field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,+ field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,+ query_hashes, token_order, field_prefix,+ drop_tokens_threshold, typo_tokens_threshold, exhaustive_search,+ min_len_1typo, min_len_2typo);+ } else if(actual_filter_ids_length != 0) {+ // indicates exact match query+ curate_filtered_ids(filters, curated_ids, exclude_token_ids,+ exclude_token_ids_size, actual_filter_ids, actual_filter_ids_length, curated_ids_sorted);++ search_wildcard(filters, included_ids_map, sort_fields_std, actual_topster,+ curated_topster,+ groups_processed, searched_queries, group_limit, group_by_fields,+ curated_ids, curated_ids_sorted,+ exclude_token_ids, exclude_token_ids_size, field_id, field_name,+ all_result_ids, all_result_ids_len,+ actual_filter_ids, actual_filter_ids_length, concurrency);+ }bool syn_wildcard_filter_init_done = false;@@ -2258,19 +2288,20 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,if(!syn_wildcard_filter_init_done) {curate_filtered_ids(filters, curated_ids, exclude_token_ids, exclude_token_ids_size,- filter_ids, filter_ids_length, curated_ids_sorted);+ actual_filter_ids, actual_filter_ids_length, curated_ids_sorted);syn_wildcard_filter_init_done = true;}- search_wildcard({"*"}, filters, included_ids_map, sort_fields_std, actual_topster,+ search_wildcard(filters, included_ids_map, sort_fields_std, actual_topster,curated_topster,groups_processed, searched_queries, group_limit, group_by_fields,curated_ids, curated_ids_sorted,exclude_token_ids, exclude_token_ids_size, field_id, field_name,- all_result_ids, all_result_ids_len, filter_ids, filter_ids_length, concurrency);+ all_result_ids, all_result_ids_len,+ actual_filter_ids, actual_filter_ids_length, concurrency);} else {search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,- field_it->second, field_name, filter_ids, filter_ids_length, curated_ids_sorted, sort_fields_std,+ field_it->second, field_name, actual_filter_ids, actual_filter_ids_length, curated_ids_sorted, sort_fields_std,field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,query_hashes, token_order, field_prefix,@@ -2284,6 +2315,10 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,collate_included_ids(field_query_tokens[i].q_include_tokens, field_name, field_id, included_ids_map, curated_topster, searched_queries);//LOG(INFO) << "topster_ids.size: " << topster_ids.size();}++ if(phrase_match_ids_size != 0) {+ delete actual_filter_ids;+ }}//auto begin0 = std::chrono::high_resolution_clock::now();@@ -2768,7 +2803,7 @@ void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::s}}-void Index::search_wildcard(const std::vector<std::string>& qtokens, const std::vector<filter>& filters,+void Index::search_wildcard(const std::vector<filter>& filters,const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,const std::vector<sort_by>& sort_fields_std, Topster* topster, Topster* curated_topster,spp::sparse_hash_set<uint64_t>& groups_processed,@@ -2869,7 +2904,7 @@ void Index::search_wildcard(const std::vector<std::string>& qtokens, const std::std::chrono::high_resolution_clock::now() - beginF).count();LOG(INFO) << "Time for raw scoring: " << timeMillisF;*/- collate_included_ids(qtokens, field, field_id, included_ids_map, curated_topster, searched_queries);+ collate_included_ids({}, field, field_id, included_ids_map, curated_topster, searched_queries);uint32_t* new_all_result_ids = nullptr;all_result_ids_len = ArrayUtils::or_scalar(all_result_ids, all_result_ids_len, filter_ids,diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 6972717e..271bb1db 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -2212,6 +2212,38 @@ TEST_F(CollectionSpecificTest, PhraseSearch) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionSpecificTest, PhraseSearchMultipleFields) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("description", field_types::STRING, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "Dog Shoemaker";+ doc1["description"] = "A book about a dog and a shoemaker";++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["title"] = "Dog and cat";+ doc2["description"] = "A book about two animals.";++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search(R"("dog shoemaker")", {"title", "description"},+ "", {}, {}, {2, 2}, 10, 1, FREQUENCY, {true, true}, 10).get();++ ASSERT_EQ(1, results["hits"].size());++ results = coll1->search(R"("dog and cat")", {"title", "description"},+ "", {}, {}, {2, 2}, 10, 1, FREQUENCY, {true, true}, 10).get();++ ASSERT_EQ(1, results["hits"].size());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionSpecificTest, HandleLargeWeights) {std::vector<field> fields = {field("title", field_types::STRING, false),field("description", field_types::STRING, false),