Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow split join tokens to be configured in always mode.

Kishore Nallanf5808532faf7
diff --git a/include/collection.h b/include/collection.h
index c7d13c12..dc34d89a 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -327,9 +327,9 @@ public:
size_t search_stop_millis = 6000*1000,
size_t min_len_1typo = 4,
size_t min_len_2typo = 7,
- bool split_join_tokens = true,
+ enable_t split_join_tokens = fallback,
size_t max_candidates = 4,
- const std::vector<infix_t>& infixes = {off},
+ const std::vector<enable_t>& infixes = {off},
const size_t max_extra_prefix = INT16_MAX,
const size_t max_extra_suffix = INT16_MAX,
const size_t facet_query_num_typos = 2,
@@ -394,7 +394,7 @@ public:
const spp::sparse_hash_set<std::string>& include_fields,
const string& highlight_fields,
const std::string& highlight_full_fields,
- const std::vector<infix_t>& infixes,
+ const std::vector<enable_t>& infixes,
std::vector<std::string>& q_tokens,
const tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<highlight_field_t>& highlight_items) const;
diff --git a/include/index.h b/include/index.h
index 0bcda7e2..7eb5f5cd 100644
--- a/include/index.h
+++ b/include/index.h
@@ -304,7 +304,7 @@ struct override_t {
}
};
-enum infix_t {
+enum enable_t {
always,
fallback,
off
@@ -339,12 +339,12 @@ struct search_args {
size_t min_len_1typo;
size_t min_len_2typo;
size_t max_candidates;
- std::vector<infix_t> infixes;
+ std::vector<enable_t> infixes;
const size_t max_extra_prefix;
const size_t max_extra_suffix;
const size_t facet_query_num_typos;
const bool filter_curated_hits;
- const bool split_join_tokens;
+ const enable_t split_join_tokens;
tsl::htrie_map<char, token_leaf> qtoken_set;
spp::sparse_hash_set<uint64_t> groups_processed;
@@ -364,9 +364,9 @@ struct search_args {
const string& default_sorting_field, bool prioritize_exact_match,
const bool prioritize_token_position, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms,
- size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<infix_t>& infixes,
+ size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<enable_t>& infixes,
const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,
- const bool filter_curated_hits, const bool split_join_tokens) :
+ const bool filter_curated_hits, const enable_t split_join_tokens) :
field_query_tokens(field_query_tokens),
search_fields(search_fields), filters(filters), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
@@ -787,9 +787,9 @@ public:
const string& default_sorting_field, bool prioritize_exact_match,
const bool prioritize_token_position, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,
- size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
+ size_t max_candidates, const std::vector<enable_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, const size_t facet_query_num_typos,
- const bool filter_curated_hits, bool split_join_tokens) const;
+ const bool filter_curated_hits, enable_t split_join_tokens) const;
void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name);
@@ -885,7 +885,7 @@ public:
size_t& exclude_token_ids_size) const;
void do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
- const std::vector<infix_t>& infixes,
+ const std::vector<enable_t>& infixes,
const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
@@ -1006,7 +1006,7 @@ public:
const size_t typo_tokens_threshold, const size_t group_limit,
const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,
const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo, size_t min_len_2typo,
- const size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
+ const size_t max_candidates, const std::vector<enable_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, uint32_t* filter_ids, uint32_t filter_ids_length,
const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,
const size_t num_search_fields, const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
diff --git a/src/collection.cpp b/src/collection.cpp
index e661cdaa..6512e27f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -751,9 +751,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
const size_t search_stop_millis,
const size_t min_len_1typo,
const size_t min_len_2typo,
- bool split_join_tokens,
+ enable_t split_join_tokens,
const size_t max_candidates,
- const std::vector<infix_t>& infixes,
+ const std::vector<enable_t>& infixes,
const size_t max_extra_prefix,
const size_t max_extra_suffix,
const size_t facet_query_num_typos,
@@ -1553,7 +1553,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
const spp::sparse_hash_set<std::string>& exclude_fields,
const string& highlight_fields,
const std::string& highlight_full_fields,
- const std::vector<infix_t>& infixes,
+ const std::vector<enable_t>& infixes,
std::vector<std::string>& q_tokens,
const tsl::htrie_map<char, token_leaf>& qtoken_set,
std::vector<highlight_field_t>& highlight_items) const {
@@ -1572,7 +1572,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
for(size_t i = 0; i < search_fields.size(); i++) {
const auto& field_name = search_fields[i];
- infix_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];
+ enable_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];
if(field_infix != off) {
fields_infixed_set.insert(field_name);
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 8f66ddaa..5f56f66c 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -713,9 +713,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::string highlight_fields;
bool exhaustive_search = false;
size_t search_cutoff_ms = 3600000;
- bool split_join_tokens = true;
+ enable_t split_join_tokens = fallback;
size_t max_candidates = 0;
- std::vector<infix_t> infixes;
+ std::vector<enable_t> infixes;
size_t max_extra_prefix = INT16_MAX;
size_t max_extra_suffix = INT16_MAX;
@@ -755,7 +755,6 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
{PRIORITIZE_TOKEN_POSITION, &prioritize_token_position},
{PRE_SEGMENTED_QUERY, &pre_segmented_query},
{EXHAUSTIVE_SEARCH, &exhaustive_search},
- {SPLIT_JOIN_TOKENS, &split_join_tokens},
{ENABLE_OVERRIDES, &enable_overrides},
};
@@ -789,6 +788,19 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
}
}
+ else if(key == SPLIT_JOIN_TOKENS) {
+ if(val == "false") {
+ split_join_tokens = off;
+ } else if(val == "true") {
+ split_join_tokens = fallback;
+ } else {
+ auto enable_op = magic_enum::enum_cast<enable_t>(val);
+ if(enable_op.has_value()) {
+ split_join_tokens = enable_op.value();
+ }
+ }
+ }
+
else {
auto find_int_it = unsigned_int_values.find(key);
if(find_int_it != unsigned_int_values.end()) {
@@ -850,7 +862,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
StringUtils::split(req_params[INFIX], infix_strs, ",");
for(auto& infix_str: infix_strs) {
- auto infix_op = magic_enum::enum_cast<infix_t>(infix_str);
+ auto infix_op = magic_enum::enum_cast<enable_t>(infix_str);
if(infix_op.has_value()) {
infixes.push_back(infix_op.value());
}
diff --git a/src/index.cpp b/src/index.cpp
index b9da716a..804147d2 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2276,9 +2276,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
const string& default_sorting_field, bool prioritize_exact_match,
const bool prioritize_token_position, bool exhaustive_search,
size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,
- size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
+ size_t max_candidates, const std::vector<enable_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, const size_t facet_query_num_typos,
- const bool filter_curated_hits, const bool split_join_tokens) const {
+ const bool filter_curated_hits, const enable_t split_join_tokens) const {
// process the filters
@@ -2397,7 +2397,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
field_values, geopoint_indices);
// try split/joining tokens if no results are found
- if(all_result_ids_len == 0 && split_join_tokens) {
+ if(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {
std::vector<std::vector<std::string>> space_resolved_queries;
for(size_t i = 0; i < num_search_fields; i++) {
@@ -3515,7 +3515,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,
}
void Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,
- const std::vector<infix_t>& infixes,
+ const std::vector<enable_t>& infixes,
const std::vector<sort_by>& sort_fields,
std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,
const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,
@@ -3531,7 +3531,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
auto& field_name = the_fields[field_id].name;
- infix_t field_infix = (field_id < infixes.size()) ? infixes[field_id] : infixes[0];
+ enable_t field_infix = (field_id < infixes.size()) ? infixes[field_id] : infixes[0];
if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {
std::vector<uint32_t> infix_ids;
@@ -5410,10 +5410,6 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const stri
leaves.push_back(leaf);
}
- if(leaves.size() == qtokens.size() && common_results_exist(leaves, false)) {
- return ;
- }
-
// When we cannot find verbatim match, we can try concatting and splitting query tokens for alternatives.
// Concatenation:
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index a8152ad0..a26a78da 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -818,7 +818,7 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringArray) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "genres: eletronic",
30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 1).get();
ASSERT_EQ(1, results["facet_counts"].size());
@@ -830,7 +830,7 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringArray) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "genres: eletronic",
30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 0).get();
ASSERT_EQ(1, results["facet_counts"].size());
@@ -863,7 +863,7 @@ TEST_F(CollectionFacetingTest, FacetQueryReturnAllCandidates) {
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "title:e", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
10, {off}, 32767, 32767, 2,
false, false);
diff --git a/test/collection_infix_search_test.cpp b/test/collection_infix_search_test.cpp
index 4f3fcdf2..6b2524f6 100644
--- a/test/collection_infix_search_test.cpp
+++ b/test/collection_infix_search_test.cpp
@@ -51,7 +51,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -69,7 +69,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}).get();
ASSERT_EQ(0, results["found"].get<size_t>());
@@ -86,7 +86,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {fallback}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -99,7 +99,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(2, results["found"].get<size_t>());
@@ -128,7 +128,7 @@ TEST_F(CollectionInfixSearchTest, InfixOnArray) {
{"model_numbers"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "model_numbers", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -166,7 +166,7 @@ TEST_F(CollectionInfixSearchTest, InfixWithFiltering) {
{"title"}, "points: 200", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, "", "", {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -183,7 +183,7 @@ TEST_F(CollectionInfixSearchTest, InfixWithFiltering) {
results = coll1->search("37IN8", {"title"}, "points:>= 200", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, "", "2", {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -216,7 +216,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}, 1).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -227,7 +227,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}, 2).get();
ASSERT_EQ(2, results["found"].get<size_t>());
@@ -240,7 +240,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}, INT16_MAX, 2).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -251,7 +251,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}, INT16_MAX, 5).get();
ASSERT_EQ(2, results["found"].get<size_t>());
@@ -286,7 +286,7 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {
{"title", "description"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always, off}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -297,7 +297,7 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {
{"title", "description"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off, always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -315,7 +315,7 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {
{"title", "description"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off, always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -346,7 +346,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -363,7 +363,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(0, results["found"].get<size_t>());
@@ -376,7 +376,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -389,7 +389,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -400,7 +400,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {
{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(0, results["found"].get<size_t>());
@@ -444,7 +444,7 @@ TEST_F(CollectionInfixSearchTest, MultiFielInfixSearch) {
{"title", "mpn"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(2, results["found"].get<size_t>());
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index 90716096..f318dde6 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -368,7 +368,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {
spp::sparse_hash_set<std::string>(), 10, "",
30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 1).get();
ASSERT_EQ(1, results["hits"].size());
@@ -385,7 +385,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {
spp::sparse_hash_set<std::string>(), 10, "",
30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 1).get();
ASSERT_EQ(1, results["hits"].size());
@@ -401,7 +401,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {
spp::sparse_hash_set<std::string>(), 10, "",
30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 0).get();
ASSERT_EQ(2, results["hits"].size());
@@ -644,7 +644,7 @@ TEST_F(CollectionOverrideTest, PinnedAndHiddenHits) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, pinned_hits, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2, 1).get();
ASSERT_EQ(4, results["found"].get<size_t>());
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index 7409ca27..f5426cbe 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -73,7 +73,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
{true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -84,7 +84,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
{true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -103,7 +103,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
{true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -143,7 +143,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
{true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -154,7 +154,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
{true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
@@ -185,7 +185,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {
{true}, 5,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {always}).get();
ASSERT_EQ(1, results["found"].get<size_t>());
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 3763b9c6..0091d165 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -240,7 +240,7 @@ TEST_F(CollectionSortingTest, FrequencyOrderedTokensWithoutDefaultSortingField)
0, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(),
10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
- false, 2).get();
+ off, 2).get();
// [11 + 10] + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2
ASSERT_EQ(21, results["found"].get<size_t>());
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index be76ec9c..ecb0d50d 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -50,7 +50,7 @@ TEST_F(CollectionSpecificMoreTest, MaxCandidatesShouldBeRespected) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,
- false, 1000).get();
+ fallback, 1000).get();
ASSERT_EQ(200, results["found"].get<size_t>());
collectionManager.drop_collection("coll1");
@@ -479,7 +479,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
false, true).get();
@@ -494,7 +494,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
false, false).get();
@@ -509,7 +509,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
false, false).get();
@@ -523,7 +523,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
false, true).get();
ASSERT_EQ(3, results["hits"].size());
@@ -558,7 +558,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRankingWithArray) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
false, false).get();
@@ -571,7 +571,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRankingWithArray) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
false, true).get();
@@ -603,7 +603,7 @@ TEST_F(CollectionSpecificMoreTest, ExactFilteringOnArray) {
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
4, {off}, 32767, 32767, 2,
false, false).get();
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index eaaebb83..ad916f83 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -2055,7 +2055,6 @@ TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeUsedForPrefixSearch) {
}
TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {
- // when the first document containing a token already cannot fit compact posting list
std::vector<field> fields = {field("title", field_types::STRING, false),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
@@ -2112,14 +2111,14 @@ TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {
results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, off).get();
ASSERT_EQ(0, results["hits"].size());
// don't split when feature is disabled
results = coll1->search("pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, off).get();
ASSERT_EQ(0, results["hits"].size());
// drop tokens should not happen on tokens split
@@ -2135,6 +2134,54 @@ TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSpecificTest, SplitJoinTokenAlways) {
+ // even if verbatim matches are found, do split+join anyway
+ std::vector<field> fields = {field("title", field_types::STRING, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Non stick cookware";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "Nonstick cookware";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "Non cookware stick";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // return only query match docs as default
+ auto results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+
+ results = coll1->search("nonstick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // in always mode, both results should be returned
+ results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, always).get();
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+
+ results = coll1->search("nonstick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, always).get();
+ ASSERT_EQ(3, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionSpecificTest, TokenCountOfWordsFarApart) {
// word proximity is calculated using a moving window of X tokens. If only 1 token is present in the best matched
// window, proximity ends up being perfect. So we've to ensure that scoring uses total tokens found and not