Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Phrase search skeleton.
Kishore Nallanab7a3f53bb3c
diff --git a/include/collection.h b/include/collection.hindex 9b4531f0..606f2ce7 100644--- a/include/collection.h+++ b/include/collection.h@@ -340,7 +340,8 @@ public:bool is_exceeding_memory_threshold() const;void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,- std::vector<std::string>& q_exclude_tokens,+ std::vector<std::vector<std::string>>& q_exclude_tokens,+ std::vector<std::vector<std::string>>& q_phrases,const std::string& locale, const bool already_segmented) const;// PUBLIC OPERATIONSdiff --git a/include/index.h b/include/index.hindex 7dbb3da3..cc214d28 100644--- a/include/index.h+++ b/include/index.h@@ -52,7 +52,8 @@ struct search_field_t {struct query_tokens_t {std::vector<std::string> q_include_tokens;- std::vector<std::string> q_exclude_tokens;+ std::vector<std::vector<std::string>> q_exclude_tokens;+ std::vector<std::vector<std::string>> q_phrases;std::vector<std::vector<std::string>> q_synonyms;};diff --git a/include/posting.h b/include/posting.hindex e8ad4be3..d2685131 100644--- a/include/posting.h+++ b/include/posting.h@@ -112,6 +112,10 @@ public:const uint32_t* ids, uint32_t num_ids,uint32_t*& exact_ids, size_t& num_exact_ids);+ static void get_phrase_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,+ const uint32_t* ids, uint32_t num_ids,+ uint32_t*& phrase_ids, size_t& num_phrase_ids);+static void get_matching_array_indices(const std::vector<void*>& raw_posting_lists,uint32_t id, std::vector<size_t>& indices);};diff --git a/include/posting_list.h b/include/posting_list.hindex 8d39d92b..a477ee42 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -169,6 +169,10 @@ public:const uint32_t* ids, const uint32_t num_ids,uint32_t*& exact_ids, size_t& num_exact_ids);+ static void get_phrase_matches(std::vector<iterator_t>& its, bool field_is_array,+ const uint32_t* ids, const uint32_t num_ids,+ uint32_t*& phrase_ids, size_t& num_phrase_ids);+static void get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,std::vector<size_t>& indices);};diff --git a/src/collection.cpp b/src/collection.cppindex e93e1c2b..4adbe23b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -896,7 +896,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sif(search_fields.size() == 0) {// has to be a wildcard queryfield_query_tokens.emplace_back(query_tokens_t{});- parse_search_query(query, field_query_tokens[0].q_include_tokens, field_query_tokens[0].q_exclude_tokens, "",+ parse_search_query(query, field_query_tokens[0].q_include_tokens,+ field_query_tokens[0].q_exclude_tokens, field_query_tokens[0].q_phrases, "",false);} else {for(size_t i = 0; i < search_fields.size(); i++) {@@ -904,7 +905,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sfield_query_tokens.emplace_back(query_tokens_t{});const std::string & field_locale = search_schema.at(search_field).locale;- parse_search_query(query, field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_exclude_tokens,+ parse_search_query(query, field_query_tokens[i].q_include_tokens,+ field_query_tokens[i].q_exclude_tokens,+ field_query_tokens[i].q_phrases,field_locale, pre_segmented_query);// get synonyms@@ -1367,7 +1370,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s}void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,- std::vector<std::string>& q_exclude_tokens,+ std::vector<std::vector<std::string>>& q_exclude_tokens,+ std::vector<std::vector<std::string>>& q_phrases,const std::string& locale, const bool already_segmented) const {if(query == "*") {q_exclude_tokens = {};@@ -1380,13 +1384,18 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s} else {std::vector<char> custom_symbols = symbols_to_index;custom_symbols.push_back('-');+ custom_symbols.push_back('"');Tokenizer(query, true, false, locale, custom_symbols, token_separators).tokenize(tokens);}bool exclude_operator_prior = false;+ bool phrase_search_op_prior = false;+ std::vector<std::string> phrase;for(auto& token: tokens) {+ bool end_of_phrase = false;+if(token == "-") {continue;} else if(token[0] == '-') {@@ -1394,7 +1403,18 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::stoken = token.substr(1);}- // retokenize using collection config+ if(token[0] == '"' && token.size() > 1) {+ phrase_search_op_prior = true;+ token = token.substr(1);+ }++ if(!token.empty() && (token.back() == '"' || (token[0] == '"' && token.size() == 1))) {+ // handles single token phrase and a phrase with padded space, like: "some query " here+ end_of_phrase = true;+ token = token.substr(0, token.size()-1);+ }++ // retokenize using collection config (handles hyphens being part of the query)std::vector<std::string> sub_tokens;if(already_segmented) {@@ -1405,12 +1425,38 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sfor(auto& sub_token: sub_tokens) {if(exclude_operator_prior) {- q_exclude_tokens.push_back(sub_token);- exclude_operator_prior = false;+ if(phrase_search_op_prior) {+ phrase.push_back(sub_token);+ } else {+ q_exclude_tokens.push_back({sub_token});+ exclude_operator_prior = false;+ }+ } else if(phrase_search_op_prior) {+ phrase.push_back(sub_token);} else {q_include_tokens.push_back(sub_token);}}++ if(end_of_phrase && phrase_search_op_prior) {+ if(exclude_operator_prior) {+ q_exclude_tokens.push_back(phrase);+ } else {+ q_phrases.push_back(phrase);+ }++ phrase_search_op_prior = false;+ exclude_operator_prior = false;+ phrase.clear();+ }+ }++ if(!phrase.empty()) {+ if(exclude_operator_prior) {+ q_exclude_tokens.push_back(phrase);+ } else {+ q_phrases.push_back(phrase);+ }}if(q_include_tokens.empty()) {diff --git a/src/index.cpp b/src/index.cppindex 8f857611..fc692d69 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2060,36 +2060,101 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,uint32_t* all_result_ids = nullptr;const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);- uint32_t *exclude_token_ids = nullptr;+ uint32_t* exclude_token_ids = nullptr;size_t exclude_token_ids_size = 0;- // find documents that contain the excluded tokens to exclude them from results later+ uint32_t* phrase_match_ids = nullptr;+ size_t phrase_match_ids_size = 0;++ // handle exclude tokens and phrase search if neededfor(size_t i = 0; i < num_search_fields; i++) {const std::string & field_name = search_fields[i].name;+ bool is_array = search_schema.at(field_name).is_array();- std::vector<void*> posting_lists;+ // do AND of `q_phrases` to add to filters+ for(const auto& phrase: field_query_tokens[i].q_phrases) {+ std::vector<void*> posting_lists;+ for(const std::string& token: phrase) {+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),+ (const unsigned char *) token.c_str(),+ token.size() + 1);- for(const std::string& exclude_token: field_query_tokens[i].q_exclude_tokens) {- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),- (const unsigned char *) exclude_token.c_str(),- exclude_token.size() + 1);+ if(leaf) {+ posting_lists.push_back(leaf->values);+ }+ }- if(leaf) {- posting_lists.push_back(leaf->values);+ if(posting_lists.size() != phrase.size()) {+ // unmatched length should mean zero results should be returned+ delete [] filter_ids;+ filter_ids = nullptr;+ filter_ids_length = 0;+ return;}- }- std::vector<uint32_t> exclude_token_id_vec;- if(!posting_lists.empty()) {- posting_t::merge(posting_lists, exclude_token_id_vec);+ std::vector<uint32_t> contains_ids;+ posting_t::intersect(posting_lists, contains_ids);++ uint32_t* phrase_ids = new uint32_t[contains_ids.size()];+ size_t phrase_ids_size = 0;+ posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),+ phrase_ids, phrase_ids_size);++ if(phrase_ids_size != 0) {+ if(filter_ids_length == 0) {+ filter_ids_length = phrase_ids_size;+ filter_ids = phrase_ids;+ } else {+ uint32_t *filter_ids_merged = nullptr;+ filter_ids_length = ArrayUtils::and_scalar(phrase_ids, phrase_ids_size, filter_ids, filter_ids_length,+ &filter_ids_merged);+ delete [] phrase_ids;+ delete [] filter_ids;+ filter_ids = filter_ids_merged;+ }+ } else {+ // no results should be returned+ delete [] phrase_ids;+ delete [] filter_ids;+ filter_ids = nullptr;+ filter_ids_length = 0;+ return;+ }}- uint32_t *exclude_token_ids_merged = nullptr;- exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,- &exclude_token_id_vec[0], exclude_token_id_vec.size(),- &exclude_token_ids_merged);- delete[] exclude_token_ids;- exclude_token_ids = exclude_token_ids_merged;+ for(const auto& q_exclude_phrase: field_query_tokens[i].q_exclude_tokens) {+ // if phrase has multiple words, then we have to do exclusion of phrase match results+ std::vector<void*> posting_lists;+ for(const std::string& exclude_token: q_exclude_phrase) {+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),+ (const unsigned char *) exclude_token.c_str(),+ exclude_token.size() + 1);+ if(leaf) {+ posting_lists.push_back(leaf->values);+ }+ }++ if(posting_lists.size() != q_exclude_phrase.size()) {+ continue;+ }++ std::vector<uint32_t> contains_ids;+ posting_t::intersect(posting_lists, contains_ids);++ uint32_t* phrase_ids = new uint32_t[contains_ids.size()];+ size_t phrase_ids_size = 0;++ posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),+ phrase_ids, phrase_ids_size);++ uint32_t *exclude_token_ids_merged = nullptr;+ exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,+ phrase_ids, phrase_ids_size,+ &exclude_token_ids_merged);+ delete [] phrase_ids;+ delete [] exclude_token_ids;+ exclude_token_ids = exclude_token_ids_merged;+ }}std::vector<Topster*> ftopsters;diff --git a/src/posting.cpp b/src/posting.cppindex 14484af0..aa38f09c 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -493,6 +493,26 @@ void posting_t::get_matching_array_indices(const std::vector<void*>& raw_posting}}+void posting_t::get_phrase_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,+ const uint32_t* ids, uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {++ std::vector<posting_list_t*> plists;+ std::vector<posting_list_t*> expanded_plists;+ to_expanded_plists(raw_posting_lists, plists, expanded_plists);++ std::vector<posting_list_t::iterator_t> its;++ for(posting_list_t* pl: plists) {+ its.push_back(pl->new_iterator());+ }++ posting_list_t::get_phrase_matches(its, field_is_array, ids, num_ids, phrase_ids, num_phrase_ids);++ for(posting_list_t* expanded_plist: expanded_plists) {+ delete expanded_plist;+ }+}+void posting_t::block_intersector_t::split_lists(size_t concurrency,std::vector<std::vector<posting_list_t::iterator_t>>& partial_its_vec) {const size_t num_blocks = this->plists[0]->num_blocks();diff --git a/src/posting_list.cpp b/src/posting_list.cppindex a3dc949a..7ec3fe08 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -1148,6 +1148,42 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const boolnum_exact_ids = exact_id_index;}+void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool field_is_array, const uint32_t* ids,+ const uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {++ size_t phrase_id_index = 0;++ if(its.size() == 1) {+ for(size_t i = 0; i < num_ids; i++) {+ phrase_ids[phrase_id_index] = ids[i];+ phrase_id_index++;+ }+ } else {+ for(size_t i = 0; i < num_ids; i++) {+ uint32_t id = ids[i];++ for (int j = its.size() - 1; j >= 0; j--) {+ posting_list_t::iterator_t& it = its[j];+ it.skip_to(id);+ }++ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;+ get_offsets(its, array_token_positions);++ for(auto& kv: array_token_positions) {+ auto dist = Match(id, kv.second, false, false).distance;+ if(dist == its.size() - 1) {+ phrase_ids[phrase_id_index] = ids[i];+ phrase_id_index++;+ break;+ }+ }+ }+ }++ num_phrase_ids = phrase_id_index;+}+void posting_list_t::get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,std::vector<size_t>& indices) {std::map<size_t, std::bitset<32>> array_index_to_token_index;diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 624811a4..528485ea 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -2042,3 +2042,102 @@ TEST_F(CollectionSpecificTest, EmptyArrayShouldBeAcceptedAsFirstValue) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionSpecificTest, PhraseSearch) {+ Collection *coll1;+ std::vector<field> fields = {field("title", field_types::STRING, false, true),+ field("points", field_types::INT32, false, true)};+ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ auto op = collectionManager.create_collection("coll1", 1, fields, "");+ ASSERT_TRUE(op.ok());+ coll1 = op.get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Then and there by the down"},+ {"Down There by the Train"},+ {"The State Trooper"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;+ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["points"] = i;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ // without phrase search++ auto results = coll1->search(R"(down there by)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ // with phrase search+ results = coll1->search(R"("down there by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ // phrase search with exclusion+ results = coll1->search(R"("by the" -train)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ // exclusion of an entire phrase+ results = coll1->search(R"(-"by the down")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ results = coll1->search(R"(-"by the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ // phrase search with token with no matching doc+ results = coll1->search(R"("by the dinosaur")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(0, results["hits"].size());++ // phrase search with no matching document+ results = coll1->search(R"("by the state")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(0, results["hits"].size());++ // phrase search with filter condition+ results = coll1->search(R"("there by the")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ // exclude phrase with tokens that don't have any document matched+ results = coll1->search(R"(-"by the dinosaur")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(3, results["hits"].size());++ // phrase with normal non-matching token+ results = coll1->search(R"("by the" state)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(0, results["hits"].size());++ // phrase with normal matching token+ results = coll1->search(R"("by the" and)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ // two phrases+ results = coll1->search(R"("by the" "then and")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ results = coll1->search(R"("by the" "there by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(2, results["hits"].size());++ // two phrases with filter+ results = coll1->search(R"("by the" "there by")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ // single token phrase+ results = coll1->search(R"("trooper")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 159ecec9..ab1a5e1c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4083,3 +4083,134 @@ TEST_F(CollectionTest, FieldLevelPrefixConfiguration) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionTest, QueryParsingForPhraseSearch) {+ Collection* coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::string> q_include_tokens;+ std::vector<std::vector<std::string>> q_exclude_tokens;+ std::vector<std::vector<std::string>> q_phrases;++ std::string q = R"(the "phrase search" query)";+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);++ ASSERT_EQ(2, q_include_tokens.size());+ ASSERT_EQ("the", q_include_tokens[0]);+ ASSERT_EQ("query", q_include_tokens[1]);+ ASSERT_EQ(1, q_phrases.size());+ ASSERT_EQ(2, q_phrases[0].size());+ ASSERT_EQ("phrase", q_phrases[0][0]);+ ASSERT_EQ("search", q_phrases[0][1]);++ // quoted string has trailing padded space++ q = R"("space padded " query)";+ q_include_tokens.clear();+ q_exclude_tokens.clear();+ q_phrases.clear();++ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);+ ASSERT_EQ(1, q_include_tokens.size());+ ASSERT_EQ("query", q_include_tokens[0]);+ ASSERT_EQ(1, q_phrases.size());+ ASSERT_EQ(2, q_phrases[0].size());+ ASSERT_EQ("space", q_phrases[0][0]);+ ASSERT_EQ("padded", q_phrases[0][1]);++ // multiple quoted strings++ q = R"("first phrase" "second phrase")";+ q_include_tokens.clear();+ q_exclude_tokens.clear();+ q_phrases.clear();++ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);+ ASSERT_EQ(1, q_include_tokens.size());+ ASSERT_EQ("*", q_include_tokens[0]);+ ASSERT_EQ(2, q_phrases.size());+ ASSERT_EQ(2, q_phrases[0].size());+ ASSERT_EQ("first", q_phrases[0][0]);+ ASSERT_EQ("phrase", q_phrases[0][1]);+ ASSERT_EQ("second", q_phrases[1][0]);+ ASSERT_EQ("phrase", q_phrases[1][1]);++ // single quoted string++ q = R"("hello")";+ q_include_tokens.clear();+ q_exclude_tokens.clear();+ q_phrases.clear();++ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);+ ASSERT_EQ(1, q_include_tokens.size());+ ASSERT_EQ("*", q_include_tokens[0]);+ ASSERT_EQ(1, q_phrases.size());+ ASSERT_EQ(1, q_phrases[0].size());+ ASSERT_EQ("hello", q_phrases[0][0]);++ // stray trailing quote++ q = R"(hello")";+ q_include_tokens.clear();+ q_exclude_tokens.clear();+ q_phrases.clear();++ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);+ ASSERT_EQ(1, q_include_tokens.size());+ ASSERT_EQ("hello", q_include_tokens[0]);+ ASSERT_EQ(0, q_phrases.size());++ // padded space one either side of quote+ q = R"("some query " here)";+ q_include_tokens.clear();+ q_exclude_tokens.clear();+ q_phrases.clear();++ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);+ ASSERT_EQ(1, q_include_tokens.size());+ ASSERT_EQ("here", q_include_tokens[0]);+ ASSERT_EQ(1, q_phrases.size());+ ASSERT_EQ(2, q_phrases[0].size());+ ASSERT_EQ("some", q_phrases[0][0]);+ ASSERT_EQ("query", q_phrases[0][1]);++ // with exclude operator+ q = R"(-"some phrase" here)";+ q_include_tokens.clear();+ q_exclude_tokens.clear();+ q_phrases.clear();+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);+ ASSERT_EQ(1, q_include_tokens.size());+ ASSERT_EQ("here", q_include_tokens[0]);+ ASSERT_EQ(0, q_phrases.size());+ ASSERT_EQ(1, q_exclude_tokens.size());+ ASSERT_EQ(2, q_exclude_tokens[0].size());+ ASSERT_EQ("some", q_exclude_tokens[0][0]);+ ASSERT_EQ("phrase", q_exclude_tokens[0][1]);++ // with multiple exclude operators+ q = R"(-"some phrase" here -token)";+ q_include_tokens.clear();+ q_exclude_tokens.clear();+ q_phrases.clear();+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);+ ASSERT_EQ(1, q_include_tokens.size());+ ASSERT_EQ("here", q_include_tokens[0]);+ ASSERT_EQ(0, q_phrases.size());+ ASSERT_EQ(2, q_exclude_tokens.size());+ ASSERT_EQ(2, q_exclude_tokens[0].size());+ ASSERT_EQ("some", q_exclude_tokens[0][0]);+ ASSERT_EQ("phrase", q_exclude_tokens[0][1]);+ ASSERT_EQ(1, q_exclude_tokens[1].size());+ ASSERT_EQ("token", q_exclude_tokens[1][0]);++ collectionManager.drop_collection("coll1");+}