Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Phrase search skeleton.

Kishore Nallanab7a3f53bb3c
diff --git a/include/collection.h b/include/collection.h
index 9b4531f0..606f2ce7 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -340,7 +340,8 @@ public:
bool is_exceeding_memory_threshold() const;
void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
- std::vector<std::string>& q_exclude_tokens,
+ std::vector<std::vector<std::string>>& q_exclude_tokens,
+ std::vector<std::vector<std::string>>& q_phrases,
const std::string& locale, const bool already_segmented) const;
// PUBLIC OPERATIONS
diff --git a/include/index.h b/include/index.h
index 7dbb3da3..cc214d28 100644
--- a/include/index.h
+++ b/include/index.h
@@ -52,7 +52,8 @@ struct search_field_t {
struct query_tokens_t {
std::vector<std::string> q_include_tokens;
- std::vector<std::string> q_exclude_tokens;
+ std::vector<std::vector<std::string>> q_exclude_tokens;
+ std::vector<std::vector<std::string>> q_phrases;
std::vector<std::vector<std::string>> q_synonyms;
};
diff --git a/include/posting.h b/include/posting.h
index e8ad4be3..d2685131 100644
--- a/include/posting.h
+++ b/include/posting.h
@@ -112,6 +112,10 @@ public:
const uint32_t* ids, uint32_t num_ids,
uint32_t*& exact_ids, size_t& num_exact_ids);
+ static void get_phrase_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,
+ const uint32_t* ids, uint32_t num_ids,
+ uint32_t*& phrase_ids, size_t& num_phrase_ids);
+
static void get_matching_array_indices(const std::vector<void*>& raw_posting_lists,
uint32_t id, std::vector<size_t>& indices);
};
diff --git a/include/posting_list.h b/include/posting_list.h
index 8d39d92b..a477ee42 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -169,6 +169,10 @@ public:
const uint32_t* ids, const uint32_t num_ids,
uint32_t*& exact_ids, size_t& num_exact_ids);
+ static void get_phrase_matches(std::vector<iterator_t>& its, bool field_is_array,
+ const uint32_t* ids, const uint32_t num_ids,
+ uint32_t*& phrase_ids, size_t& num_phrase_ids);
+
static void get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,
std::vector<size_t>& indices);
};
diff --git a/src/collection.cpp b/src/collection.cpp
index e93e1c2b..4adbe23b 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -896,7 +896,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
if(search_fields.size() == 0) {
// has to be a wildcard query
field_query_tokens.emplace_back(query_tokens_t{});
- parse_search_query(query, field_query_tokens[0].q_include_tokens, field_query_tokens[0].q_exclude_tokens, "",
+ parse_search_query(query, field_query_tokens[0].q_include_tokens,
+ field_query_tokens[0].q_exclude_tokens, field_query_tokens[0].q_phrases, "",
false);
} else {
for(size_t i = 0; i < search_fields.size(); i++) {
@@ -904,7 +905,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
field_query_tokens.emplace_back(query_tokens_t{});
const std::string & field_locale = search_schema.at(search_field).locale;
- parse_search_query(query, field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_exclude_tokens,
+ parse_search_query(query, field_query_tokens[i].q_include_tokens,
+ field_query_tokens[i].q_exclude_tokens,
+ field_query_tokens[i].q_phrases,
field_locale, pre_segmented_query);
// get synonyms
@@ -1367,7 +1370,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
}
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
- std::vector<std::string>& q_exclude_tokens,
+ std::vector<std::vector<std::string>>& q_exclude_tokens,
+ std::vector<std::vector<std::string>>& q_phrases,
const std::string& locale, const bool already_segmented) const {
if(query == "*") {
q_exclude_tokens = {};
@@ -1380,13 +1384,18 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
} else {
std::vector<char> custom_symbols = symbols_to_index;
custom_symbols.push_back('-');
+ custom_symbols.push_back('"');
Tokenizer(query, true, false, locale, custom_symbols, token_separators).tokenize(tokens);
}
bool exclude_operator_prior = false;
+ bool phrase_search_op_prior = false;
+ std::vector<std::string> phrase;
for(auto& token: tokens) {
+ bool end_of_phrase = false;
+
if(token == "-") {
continue;
} else if(token[0] == '-') {
@@ -1394,7 +1403,18 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
token = token.substr(1);
}
- // retokenize using collection config
+ if(token[0] == '"' && token.size() > 1) {
+ phrase_search_op_prior = true;
+ token = token.substr(1);
+ }
+
+ if(!token.empty() && (token.back() == '"' || (token[0] == '"' && token.size() == 1))) {
+ // handles single token phrase and a phrase with padded space, like: "some query " here
+ end_of_phrase = true;
+ token = token.substr(0, token.size()-1);
+ }
+
+ // retokenize using collection config (handles hyphens being part of the query)
std::vector<std::string> sub_tokens;
if(already_segmented) {
@@ -1405,12 +1425,38 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
for(auto& sub_token: sub_tokens) {
if(exclude_operator_prior) {
- q_exclude_tokens.push_back(sub_token);
- exclude_operator_prior = false;
+ if(phrase_search_op_prior) {
+ phrase.push_back(sub_token);
+ } else {
+ q_exclude_tokens.push_back({sub_token});
+ exclude_operator_prior = false;
+ }
+ } else if(phrase_search_op_prior) {
+ phrase.push_back(sub_token);
} else {
q_include_tokens.push_back(sub_token);
}
}
+
+ if(end_of_phrase && phrase_search_op_prior) {
+ if(exclude_operator_prior) {
+ q_exclude_tokens.push_back(phrase);
+ } else {
+ q_phrases.push_back(phrase);
+ }
+
+ phrase_search_op_prior = false;
+ exclude_operator_prior = false;
+ phrase.clear();
+ }
+ }
+
+ if(!phrase.empty()) {
+ if(exclude_operator_prior) {
+ q_exclude_tokens.push_back(phrase);
+ } else {
+ q_phrases.push_back(phrase);
+ }
}
if(q_include_tokens.empty()) {
diff --git a/src/index.cpp b/src/index.cpp
index 8f857611..fc692d69 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2060,36 +2060,101 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,
uint32_t* all_result_ids = nullptr;
const size_t num_search_fields = std::min(search_fields.size(), (size_t) FIELD_LIMIT_NUM);
- uint32_t *exclude_token_ids = nullptr;
+ uint32_t* exclude_token_ids = nullptr;
size_t exclude_token_ids_size = 0;
- // find documents that contain the excluded tokens to exclude them from results later
+ uint32_t* phrase_match_ids = nullptr;
+ size_t phrase_match_ids_size = 0;
+
+ // handle exclude tokens and phrase search if needed
for(size_t i = 0; i < num_search_fields; i++) {
const std::string & field_name = search_fields[i].name;
+ bool is_array = search_schema.at(field_name).is_array();
- std::vector<void*> posting_lists;
+ // do AND of `q_phrases` to add to filters
+ for(const auto& phrase: field_query_tokens[i].q_phrases) {
+ std::vector<void*> posting_lists;
+ for(const std::string& token: phrase) {
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
+ (const unsigned char *) token.c_str(),
+ token.size() + 1);
- for(const std::string& exclude_token: field_query_tokens[i].q_exclude_tokens) {
- art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
- (const unsigned char *) exclude_token.c_str(),
- exclude_token.size() + 1);
+ if(leaf) {
+ posting_lists.push_back(leaf->values);
+ }
+ }
- if(leaf) {
- posting_lists.push_back(leaf->values);
+ if(posting_lists.size() != phrase.size()) {
+ // unmatched length should mean zero results should be returned
+ delete [] filter_ids;
+ filter_ids = nullptr;
+ filter_ids_length = 0;
+ return;
}
- }
- std::vector<uint32_t> exclude_token_id_vec;
- if(!posting_lists.empty()) {
- posting_t::merge(posting_lists, exclude_token_id_vec);
+ std::vector<uint32_t> contains_ids;
+ posting_t::intersect(posting_lists, contains_ids);
+
+ uint32_t* phrase_ids = new uint32_t[contains_ids.size()];
+ size_t phrase_ids_size = 0;
+ posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),
+ phrase_ids, phrase_ids_size);
+
+ if(phrase_ids_size != 0) {
+ if(filter_ids_length == 0) {
+ filter_ids_length = phrase_ids_size;
+ filter_ids = phrase_ids;
+ } else {
+ uint32_t *filter_ids_merged = nullptr;
+ filter_ids_length = ArrayUtils::and_scalar(phrase_ids, phrase_ids_size, filter_ids, filter_ids_length,
+ &filter_ids_merged);
+ delete [] phrase_ids;
+ delete [] filter_ids;
+ filter_ids = filter_ids_merged;
+ }
+ } else {
+ // no results should be returned
+ delete [] phrase_ids;
+ delete [] filter_ids;
+ filter_ids = nullptr;
+ filter_ids_length = 0;
+ return;
+ }
}
- uint32_t *exclude_token_ids_merged = nullptr;
- exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
- &exclude_token_id_vec[0], exclude_token_id_vec.size(),
- &exclude_token_ids_merged);
- delete[] exclude_token_ids;
- exclude_token_ids = exclude_token_ids_merged;
+ for(const auto& q_exclude_phrase: field_query_tokens[i].q_exclude_tokens) {
+ // if phrase has multiple words, then we have to do exclusion of phrase match results
+ std::vector<void*> posting_lists;
+ for(const std::string& exclude_token: q_exclude_phrase) {
+ art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
+ (const unsigned char *) exclude_token.c_str(),
+ exclude_token.size() + 1);
+ if(leaf) {
+ posting_lists.push_back(leaf->values);
+ }
+ }
+
+ if(posting_lists.size() != q_exclude_phrase.size()) {
+ continue;
+ }
+
+ std::vector<uint32_t> contains_ids;
+ posting_t::intersect(posting_lists, contains_ids);
+
+ uint32_t* phrase_ids = new uint32_t[contains_ids.size()];
+ size_t phrase_ids_size = 0;
+
+ posting_t::get_phrase_matches(posting_lists, is_array, &contains_ids[0], contains_ids.size(),
+ phrase_ids, phrase_ids_size);
+
+ uint32_t *exclude_token_ids_merged = nullptr;
+ exclude_token_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
+ phrase_ids, phrase_ids_size,
+ &exclude_token_ids_merged);
+ delete [] phrase_ids;
+ delete [] exclude_token_ids;
+ exclude_token_ids = exclude_token_ids_merged;
+ }
}
std::vector<Topster*> ftopsters;
diff --git a/src/posting.cpp b/src/posting.cpp
index 14484af0..aa38f09c 100644
--- a/src/posting.cpp
+++ b/src/posting.cpp
@@ -493,6 +493,26 @@ void posting_t::get_matching_array_indices(const std::vector<void*>& raw_posting
}
}
+void posting_t::get_phrase_matches(const std::vector<void*>& raw_posting_lists, bool field_is_array,
+ const uint32_t* ids, uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {
+
+ std::vector<posting_list_t*> plists;
+ std::vector<posting_list_t*> expanded_plists;
+ to_expanded_plists(raw_posting_lists, plists, expanded_plists);
+
+ std::vector<posting_list_t::iterator_t> its;
+
+ for(posting_list_t* pl: plists) {
+ its.push_back(pl->new_iterator());
+ }
+
+ posting_list_t::get_phrase_matches(its, field_is_array, ids, num_ids, phrase_ids, num_phrase_ids);
+
+ for(posting_list_t* expanded_plist: expanded_plists) {
+ delete expanded_plist;
+ }
+}
+
void posting_t::block_intersector_t::split_lists(size_t concurrency,
std::vector<std::vector<posting_list_t::iterator_t>>& partial_its_vec) {
const size_t num_blocks = this->plists[0]->num_blocks();
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index a3dc949a..7ec3fe08 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -1148,6 +1148,42 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const bool
num_exact_ids = exact_id_index;
}
+void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool field_is_array, const uint32_t* ids,
+ const uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {
+
+ size_t phrase_id_index = 0;
+
+ if(its.size() == 1) {
+ for(size_t i = 0; i < num_ids; i++) {
+ phrase_ids[phrase_id_index] = ids[i];
+ phrase_id_index++;
+ }
+ } else {
+ for(size_t i = 0; i < num_ids; i++) {
+ uint32_t id = ids[i];
+
+ for (int j = its.size() - 1; j >= 0; j--) {
+ posting_list_t::iterator_t& it = its[j];
+ it.skip_to(id);
+ }
+
+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ get_offsets(its, array_token_positions);
+
+ for(auto& kv: array_token_positions) {
+ auto dist = Match(id, kv.second, false, false).distance;
+ if(dist == its.size() - 1) {
+ phrase_ids[phrase_id_index] = ids[i];
+ phrase_id_index++;
+ break;
+ }
+ }
+ }
+ }
+
+ num_phrase_ids = phrase_id_index;
+}
+
void posting_list_t::get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,
std::vector<size_t>& indices) {
std::map<size_t, std::bitset<32>> array_index_to_token_index;
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 624811a4..528485ea 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -2042,3 +2042,102 @@ TEST_F(CollectionSpecificTest, EmptyArrayShouldBeAcceptedAsFirstValue) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionSpecificTest, PhraseSearch) {
+ Collection *coll1;
+ std::vector<field> fields = {field("title", field_types::STRING, false, true),
+ field("points", field_types::INT32, false, true)};
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ auto op = collectionManager.create_collection("coll1", 1, fields, "");
+ ASSERT_TRUE(op.ok());
+ coll1 = op.get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Then and there by the down"},
+ {"Down There by the Train"},
+ {"The State Trooper"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["points"] = i;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ // without phrase search
+
+ auto results = coll1->search(R"(down there by)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ // with phrase search
+ results = coll1->search(R"("down there by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // phrase search with exclusion
+ results = coll1->search(R"("by the" -train)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // exclusion of an entire phrase
+ results = coll1->search(R"(-"by the down")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+
+ results = coll1->search(R"(-"by the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // phrase search with token with no matching doc
+ results = coll1->search(R"("by the dinosaur")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ // phrase search with no matching document
+ results = coll1->search(R"("by the state")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ // phrase search with filter condition
+ results = coll1->search(R"("there by the")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // exclude phrase with tokens that don't have any document matched
+ results = coll1->search(R"(-"by the dinosaur")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(3, results["hits"].size());
+
+ // phrase with normal non-matching token
+ results = coll1->search(R"("by the" state)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ // phrase with normal matching token
+ results = coll1->search(R"("by the" and)", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // two phrases
+ results = coll1->search(R"("by the" "then and")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ results = coll1->search(R"("by the" "there by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(2, results["hits"].size());
+
+ // two phrases with filter
+ results = coll1->search(R"("by the" "there by")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // single token phrase
+ results = coll1->search(R"("trooper")", {"title"}, "points:>=1", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 159ecec9..ab1a5e1c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4083,3 +4083,134 @@ TEST_F(CollectionTest, FieldLevelPrefixConfiguration) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionTest, QueryParsingForPhraseSearch) {
+ Collection* coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::string> q_include_tokens;
+ std::vector<std::vector<std::string>> q_exclude_tokens;
+ std::vector<std::vector<std::string>> q_phrases;
+
+ std::string q = R"(the "phrase search" query)";
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+
+ ASSERT_EQ(2, q_include_tokens.size());
+ ASSERT_EQ("the", q_include_tokens[0]);
+ ASSERT_EQ("query", q_include_tokens[1]);
+ ASSERT_EQ(1, q_phrases.size());
+ ASSERT_EQ(2, q_phrases[0].size());
+ ASSERT_EQ("phrase", q_phrases[0][0]);
+ ASSERT_EQ("search", q_phrases[0][1]);
+
+ // quoted string has trailing padded space
+
+ q = R"("space padded " query)";
+ q_include_tokens.clear();
+ q_exclude_tokens.clear();
+ q_phrases.clear();
+
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+ ASSERT_EQ(1, q_include_tokens.size());
+ ASSERT_EQ("query", q_include_tokens[0]);
+ ASSERT_EQ(1, q_phrases.size());
+ ASSERT_EQ(2, q_phrases[0].size());
+ ASSERT_EQ("space", q_phrases[0][0]);
+ ASSERT_EQ("padded", q_phrases[0][1]);
+
+ // multiple quoted strings
+
+ q = R"("first phrase" "second phrase")";
+ q_include_tokens.clear();
+ q_exclude_tokens.clear();
+ q_phrases.clear();
+
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+ ASSERT_EQ(1, q_include_tokens.size());
+ ASSERT_EQ("*", q_include_tokens[0]);
+ ASSERT_EQ(2, q_phrases.size());
+ ASSERT_EQ(2, q_phrases[0].size());
+ ASSERT_EQ("first", q_phrases[0][0]);
+ ASSERT_EQ("phrase", q_phrases[0][1]);
+ ASSERT_EQ("second", q_phrases[1][0]);
+ ASSERT_EQ("phrase", q_phrases[1][1]);
+
+ // single quoted string
+
+ q = R"("hello")";
+ q_include_tokens.clear();
+ q_exclude_tokens.clear();
+ q_phrases.clear();
+
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+ ASSERT_EQ(1, q_include_tokens.size());
+ ASSERT_EQ("*", q_include_tokens[0]);
+ ASSERT_EQ(1, q_phrases.size());
+ ASSERT_EQ(1, q_phrases[0].size());
+ ASSERT_EQ("hello", q_phrases[0][0]);
+
+ // stray trailing quote
+
+ q = R"(hello")";
+ q_include_tokens.clear();
+ q_exclude_tokens.clear();
+ q_phrases.clear();
+
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+ ASSERT_EQ(1, q_include_tokens.size());
+ ASSERT_EQ("hello", q_include_tokens[0]);
+ ASSERT_EQ(0, q_phrases.size());
+
+ // padded space one either side of quote
+ q = R"("some query " here)";
+ q_include_tokens.clear();
+ q_exclude_tokens.clear();
+ q_phrases.clear();
+
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+ ASSERT_EQ(1, q_include_tokens.size());
+ ASSERT_EQ("here", q_include_tokens[0]);
+ ASSERT_EQ(1, q_phrases.size());
+ ASSERT_EQ(2, q_phrases[0].size());
+ ASSERT_EQ("some", q_phrases[0][0]);
+ ASSERT_EQ("query", q_phrases[0][1]);
+
+ // with exclude operator
+ q = R"(-"some phrase" here)";
+ q_include_tokens.clear();
+ q_exclude_tokens.clear();
+ q_phrases.clear();
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+ ASSERT_EQ(1, q_include_tokens.size());
+ ASSERT_EQ("here", q_include_tokens[0]);
+ ASSERT_EQ(0, q_phrases.size());
+ ASSERT_EQ(1, q_exclude_tokens.size());
+ ASSERT_EQ(2, q_exclude_tokens[0].size());
+ ASSERT_EQ("some", q_exclude_tokens[0][0]);
+ ASSERT_EQ("phrase", q_exclude_tokens[0][1]);
+
+ // with multiple exclude operators
+ q = R"(-"some phrase" here -token)";
+ q_include_tokens.clear();
+ q_exclude_tokens.clear();
+ q_phrases.clear();
+ coll1->parse_search_query(q, q_include_tokens, q_exclude_tokens, q_phrases, "en", false);
+ ASSERT_EQ(1, q_include_tokens.size());
+ ASSERT_EQ("here", q_include_tokens[0]);
+ ASSERT_EQ(0, q_phrases.size());
+ ASSERT_EQ(2, q_exclude_tokens.size());
+ ASSERT_EQ(2, q_exclude_tokens[0].size());
+ ASSERT_EQ("some", q_exclude_tokens[0][0]);
+ ASSERT_EQ("phrase", q_exclude_tokens[0][1]);
+ ASSERT_EQ(1, q_exclude_tokens[1].size());
+ ASSERT_EQ("token", q_exclude_tokens[1][0]);
+
+ collectionManager.drop_collection("coll1");
+}