Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow split join tokens to be configured in always mode.
Kishore Nallanf5808532faf7
diff --git a/include/collection.h b/include/collection.hindex c7d13c12..dc34d89a 100644--- a/include/collection.h+++ b/include/collection.h@@ -327,9 +327,9 @@ public:size_t search_stop_millis = 6000*1000,size_t min_len_1typo = 4,size_t min_len_2typo = 7,- bool split_join_tokens = true,+ enable_t split_join_tokens = fallback,size_t max_candidates = 4,- const std::vector<infix_t>& infixes = {off},+ const std::vector<enable_t>& infixes = {off},const size_t max_extra_prefix = INT16_MAX,const size_t max_extra_suffix = INT16_MAX,const size_t facet_query_num_typos = 2,@@ -394,7 +394,7 @@ public:const spp::sparse_hash_set<std::string>& include_fields,const string& highlight_fields,const std::string& highlight_full_fields,- const std::vector<infix_t>& infixes,+ const std::vector<enable_t>& infixes,std::vector<std::string>& q_tokens,const tsl::htrie_map<char, token_leaf>& qtoken_set,std::vector<highlight_field_t>& highlight_items) const;diff --git a/include/index.h b/include/index.hindex 0bcda7e2..7eb5f5cd 100644--- a/include/index.h+++ b/include/index.h@@ -304,7 +304,7 @@ struct override_t {}};-enum infix_t {+enum enable_t {always,fallback,off@@ -339,12 +339,12 @@ struct search_args {size_t min_len_1typo;size_t min_len_2typo;size_t max_candidates;- std::vector<infix_t> infixes;+ std::vector<enable_t> infixes;const size_t max_extra_prefix;const size_t max_extra_suffix;const size_t facet_query_num_typos;const bool filter_curated_hits;- const bool split_join_tokens;+ const enable_t split_join_tokens;tsl::htrie_map<char, token_leaf> qtoken_set;spp::sparse_hash_set<uint64_t> groups_processed;@@ -364,9 +364,9 @@ struct search_args {const string& default_sorting_field, bool prioritize_exact_match,const bool prioritize_token_position, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms,- size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<infix_t>& infixes,+ size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<enable_t>& infixes,const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,- const bool filter_curated_hits, const bool split_join_tokens) :+ const bool filter_curated_hits, const enable_t split_join_tokens) :field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -787,9 +787,9 @@ public:const string& default_sorting_field, bool prioritize_exact_match,const bool prioritize_token_position, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,- size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,+ size_t max_candidates, const std::vector<enable_t>& infixes, const size_t max_extra_prefix,const size_t max_extra_suffix, const size_t facet_query_num_typos,- const bool filter_curated_hits, bool split_join_tokens) const;+ const bool filter_curated_hits, enable_t split_join_tokens) const;void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name);@@ -885,7 +885,7 @@ public:size_t& exclude_token_ids_size) const;void do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,- const std::vector<infix_t>& infixes,+ const std::vector<enable_t>& infixes,const std::vector<sort_by>& sort_fields,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,@@ -1006,7 +1006,7 @@ public:const size_t typo_tokens_threshold, const size_t group_limit,const std::vector<std::string>& group_by_fields, bool prioritize_exact_match,const bool exhaustive_search, const size_t concurrency, size_t min_len_1typo, size_t min_len_2typo,- const size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,+ const size_t max_candidates, const std::vector<enable_t>& infixes, const size_t max_extra_prefix,const size_t max_extra_suffix, uint32_t* filter_ids, uint32_t filter_ids_length,const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,const size_t num_search_fields, const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,diff --git a/src/collection.cpp b/src/collection.cppindex e661cdaa..6512e27f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -751,9 +751,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sconst size_t search_stop_millis,const size_t min_len_1typo,const size_t min_len_2typo,- bool split_join_tokens,+ enable_t split_join_tokens,const size_t max_candidates,- const std::vector<infix_t>& infixes,+ const std::vector<enable_t>& infixes,const size_t max_extra_prefix,const size_t max_extra_suffix,const size_t facet_query_num_typos,@@ -1553,7 +1553,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& searchconst spp::sparse_hash_set<std::string>& exclude_fields,const string& highlight_fields,const std::string& highlight_full_fields,- const std::vector<infix_t>& infixes,+ const std::vector<enable_t>& infixes,std::vector<std::string>& q_tokens,const tsl::htrie_map<char, token_leaf>& qtoken_set,std::vector<highlight_field_t>& highlight_items) const {@@ -1572,7 +1572,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& searchfor(size_t i = 0; i < search_fields.size(); i++) {const auto& field_name = search_fields[i];- infix_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];+ enable_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];if(field_infix != off) {fields_infixed_set.insert(field_name);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 8f66ddaa..5f56f66c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -713,9 +713,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::string highlight_fields;bool exhaustive_search = false;size_t search_cutoff_ms = 3600000;- bool split_join_tokens = true;+ enable_t split_join_tokens = fallback;size_t max_candidates = 0;- std::vector<infix_t> infixes;+ std::vector<enable_t> infixes;size_t max_extra_prefix = INT16_MAX;size_t max_extra_suffix = INT16_MAX;@@ -755,7 +755,6 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{PRIORITIZE_TOKEN_POSITION, &prioritize_token_position},{PRE_SEGMENTED_QUERY, &pre_segmented_query},{EXHAUSTIVE_SEARCH, &exhaustive_search},- {SPLIT_JOIN_TOKENS, &split_join_tokens},{ENABLE_OVERRIDES, &enable_overrides},};@@ -789,6 +788,19 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}}+ else if(key == SPLIT_JOIN_TOKENS) {+ if(val == "false") {+ split_join_tokens = off;+ } else if(val == "true") {+ split_join_tokens = fallback;+ } else {+ auto enable_op = magic_enum::enum_cast<enable_t>(val);+ if(enable_op.has_value()) {+ split_join_tokens = enable_op.value();+ }+ }+ }+else {auto find_int_it = unsigned_int_values.find(key);if(find_int_it != unsigned_int_values.end()) {@@ -850,7 +862,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reStringUtils::split(req_params[INFIX], infix_strs, ",");for(auto& infix_str: infix_strs) {- auto infix_op = magic_enum::enum_cast<infix_t>(infix_str);+ auto infix_op = magic_enum::enum_cast<enable_t>(infix_str);if(infix_op.has_value()) {infixes.push_back(infix_op.value());}diff --git a/src/index.cpp b/src/index.cppindex b9da716a..804147d2 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2276,9 +2276,9 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vconst string& default_sorting_field, bool prioritize_exact_match,const bool prioritize_token_position, bool exhaustive_search,size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,- size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,+ size_t max_candidates, const std::vector<enable_t>& infixes, const size_t max_extra_prefix,const size_t max_extra_suffix, const size_t facet_query_num_typos,- const bool filter_curated_hits, const bool split_join_tokens) const {+ const bool filter_curated_hits, const enable_t split_join_tokens) const {// process the filters@@ -2397,7 +2397,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vfield_values, geopoint_indices);// try split/joining tokens if no results are found- if(all_result_ids_len == 0 && split_join_tokens) {+ if(split_join_tokens == always || (all_result_ids_len == 0 && split_join_tokens == fallback)) {std::vector<std::vector<std::string>> space_resolved_queries;for(size_t i = 0; i < num_search_fields; i++) {@@ -3515,7 +3515,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,}void Index::do_infix_search(const size_t num_search_fields, const std::vector<search_field_t>& the_fields,- const std::vector<infix_t>& infixes,+ const std::vector<enable_t>& infixes,const std::vector<sort_by>& sort_fields,std::vector<std::vector<art_leaf*>>& searched_queries, const size_t group_limit,const std::vector<std::string>& group_by_fields, const size_t max_extra_prefix,@@ -3531,7 +3531,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<sefor(size_t field_id = 0; field_id < num_search_fields; field_id++) {auto& field_name = the_fields[field_id].name;- infix_t field_infix = (field_id < infixes.size()) ? infixes[field_id] : infixes[0];+ enable_t field_infix = (field_id < infixes.size()) ? infixes[field_id] : infixes[0];if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {std::vector<uint32_t> infix_ids;@@ -5410,10 +5410,6 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const strileaves.push_back(leaf);}- if(leaves.size() == qtokens.size() && common_results_exist(leaves, false)) {- return ;- }-// When we cannot find verbatim match, we can try concatting and splitting query tokens for alternatives.// Concatenation:diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex a8152ad0..a26a78da 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -818,7 +818,7 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringArray) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "genres: eletronic",30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 1).get();ASSERT_EQ(1, results["facet_counts"].size());@@ -830,7 +830,7 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringArray) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "genres: eletronic",30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 0).get();ASSERT_EQ(1, results["facet_counts"].size());@@ -863,7 +863,7 @@ TEST_F(CollectionFacetingTest, FacetQueryReturnAllCandidates) {token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "title:e", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,10, {off}, 32767, 32767, 2,false, false);diff --git a/test/collection_infix_search_test.cpp b/test/collection_infix_search_test.cppindex 4f3fcdf2..6b2524f6 100644--- a/test/collection_infix_search_test.cpp+++ b/test/collection_infix_search_test.cpp@@ -51,7 +51,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -69,7 +69,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}).get();ASSERT_EQ(0, results["found"].get<size_t>());@@ -86,7 +86,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {fallback}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -99,7 +99,7 @@ TEST_F(CollectionInfixSearchTest, InfixBasics) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(2, results["found"].get<size_t>());@@ -128,7 +128,7 @@ TEST_F(CollectionInfixSearchTest, InfixOnArray) {{"model_numbers"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "model_numbers", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -166,7 +166,7 @@ TEST_F(CollectionInfixSearchTest, InfixWithFiltering) {{"title"}, "points: 200", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, "", "", {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -183,7 +183,7 @@ TEST_F(CollectionInfixSearchTest, InfixWithFiltering) {results = coll1->search("37IN8", {"title"}, "points:>= 200", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, "", "2", {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -216,7 +216,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}, 1).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -227,7 +227,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}, 2).get();ASSERT_EQ(2, results["found"].get<size_t>());@@ -240,7 +240,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}, INT16_MAX, 2).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -251,7 +251,7 @@ TEST_F(CollectionInfixSearchTest, RespectPrefixAndSuffixLimits) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}, INT16_MAX, 5).get();ASSERT_EQ(2, results["found"].get<size_t>());@@ -286,7 +286,7 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {{"title", "description"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always, off}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -297,7 +297,7 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {{"title", "description"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off, always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -315,7 +315,7 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {{"title", "description"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off, always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -346,7 +346,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -363,7 +363,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(0, results["found"].get<size_t>());@@ -376,7 +376,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -389,7 +389,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -400,7 +400,7 @@ TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {{"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(0, results["found"].get<size_t>());@@ -444,7 +444,7 @@ TEST_F(CollectionInfixSearchTest, MultiFielInfixSearch) {{"title", "mpn"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(2, results["found"].get<size_t>());diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex 90716096..f318dde6 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -368,7 +368,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {spp::sparse_hash_set<std::string>(), 10, "",30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2, 1).get();ASSERT_EQ(1, results["hits"].size());@@ -385,7 +385,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {spp::sparse_hash_set<std::string>(), 10, "",30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2, 1).get();ASSERT_EQ(1, results["hits"].size());@@ -401,7 +401,7 @@ TEST_F(CollectionOverrideTest, IncludeHitsFilterOverrides) {spp::sparse_hash_set<std::string>(), 10, "",30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2, 0).get();ASSERT_EQ(2, results["hits"].size());@@ -644,7 +644,7 @@ TEST_F(CollectionOverrideTest, PinnedAndHiddenHits) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, pinned_hits, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2, 1).get();ASSERT_EQ(4, results["found"].get<size_t>());diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex 7409ca27..f5426cbe 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -73,7 +73,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {{true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -84,7 +84,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {{true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -103,7 +103,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {{true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -143,7 +143,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {{true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -154,7 +154,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {{true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());@@ -185,7 +185,7 @@ TEST_F(CollectionSchemaChangeTest, AddNewFieldsToCollection) {{true}, 5,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {always}).get();ASSERT_EQ(1, results["found"].get<size_t>());diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 3763b9c6..0091d165 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -240,7 +240,7 @@ TEST_F(CollectionSortingTest, FrequencyOrderedTokensWithoutDefaultSortingField)0, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(),10, "", 30, 4, "title", 20, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,- false, 2).get();+ off, 2).get();// [11 + 10] + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2ASSERT_EQ(21, results["found"].get<size_t>());diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex be76ec9c..ecb0d50d 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -50,7 +50,7 @@ TEST_F(CollectionSpecificMoreTest, MaxCandidatesShouldBeRespected) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7,- false, 1000).get();+ fallback, 1000).get();ASSERT_EQ(200, results["found"].get<size_t>());collectionManager.drop_collection("coll1");@@ -479,7 +479,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,false, true).get();@@ -494,7 +494,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,false, false).get();@@ -509,7 +509,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,false, false).get();@@ -523,7 +523,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRanking) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,false, true).get();ASSERT_EQ(3, results["hits"].size());@@ -558,7 +558,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRankingWithArray) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,false, false).get();@@ -571,7 +571,7 @@ TEST_F(CollectionSpecificMoreTest, PositionalTokenRankingWithArray) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,false, true).get();@@ -603,7 +603,7 @@ TEST_F(CollectionSpecificMoreTest, ExactFilteringOnArray) {spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,4, {off}, 32767, 32767, 2,false, false).get();diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex eaaebb83..ad916f83 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -2055,7 +2055,6 @@ TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeUsedForPrefixSearch) {}TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {- // when the first document containing a token already cannot fit compact posting liststd::vector<field> fields = {field("title", field_types::STRING, false),};Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();@@ -2112,14 +2111,14 @@ TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, off).get();ASSERT_EQ(0, results["hits"].size());// don't split when feature is disabledresults = coll1->search("pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,- "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, off).get();ASSERT_EQ(0, results["hits"].size());// drop tokens should not happen on tokens split@@ -2135,6 +2134,54 @@ TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionSpecificTest, SplitJoinTokenAlways) {+ // even if verbatim matches are found, do split+join anyway+ std::vector<field> fields = {field("title", field_types::STRING, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["title"] = "Non stick cookware";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "Nonstick cookware";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "Non cookware stick";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // return only query match docs as default+ auto results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());++ results = coll1->search("nonstick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ // in always mode, both results should be returned+ results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, always).get();+ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());++ results = coll1->search("nonstick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, always).get();+ ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionSpecificTest, TokenCountOfWordsFarApart) {// word proximity is calculated using a moving window of X tokens. If only 1 token is present in the best matched// window, proximity ends up being perfect. So we've to ensure that scoring uses total tokens found and not