Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Make splitting/joining of tokens configurable.

Kishore Nallane3d5e3df739d
diff --git a/include/collection.h b/include/collection.h
index 8fe4d616..6a273710 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -393,7 +393,8 @@ public:
const bool exhaustive_search = false,
size_t search_stop_millis = 6000*1000,
size_t min_len_1typo = 4,
- size_t min_len_2typo = 7) const;
+ size_t min_len_2typo = 7,
+ bool split_join_tokens = true) const;
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/src/collection.cpp b/src/collection.cpp
index f1c522d1..a326b15e 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -677,7 +677,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
const bool exhaustive_search,
const size_t search_stop_millis,
const size_t min_len_1typo,
- const size_t min_len_2typo) const {
+ const size_t min_len_2typo,
+ bool split_join_tokens) const {
std::shared_lock lock(mutex);
@@ -968,8 +969,11 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
synonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);
std::vector<std::vector<std::string>> space_resolved_queries;
- index->resolve_space_as_typos(field_query_tokens[i].q_include_tokens, search_field,
- space_resolved_queries);
+
+ if(split_join_tokens) {
+ index->resolve_space_as_typos(field_query_tokens[i].q_include_tokens, search_field,
+ space_resolved_queries);
+ }
// only one query is resolved for now, so just use that
if(!space_resolved_queries.empty()) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 0b3739d2..c3e0eae1 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -584,6 +584,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *SEARCH_CUTOFF_MS = "search_cutoff_ms";
const char *EXHAUSTIVE_SEARCH = "exhaustive_search";
+ const char *SPLIT_JOIN_TOKENS = "split_join_tokens";
if(req_params.count(NUM_TYPOS) == 0) {
req_params[NUM_TYPOS] = "2";
@@ -702,6 +703,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
req_params[EXHAUSTIVE_SEARCH] = "false";
}
+ if(req_params.count(SPLIT_JOIN_TOKENS) == 0) {
+ req_params[SPLIT_JOIN_TOKENS] = "true";
+ }
+
std::vector<std::string> query_by_weights_str;
std::vector<size_t> query_by_weights;
@@ -778,6 +783,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
bool prioritize_exact_match = (req_params[PRIORITIZE_EXACT_MATCH] == "true");
bool pre_segmented_query = (req_params[PRE_SEGMENTED_QUERY] == "true");
bool exhaustive_search = (req_params[EXHAUSTIVE_SEARCH] == "true");
+ bool split_join_tokens = (req_params[SPLIT_JOIN_TOKENS] == "true");
std::string filter_str = req_params.count(FILTER) != 0 ? req_params[FILTER] : "";
@@ -883,7 +889,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
enable_overrides,
req_params[HIGHLIGHT_FIELDS],
exhaustive_search,
- static_cast<size_t>(std::stol(req_params[SEARCH_CUTOFF_MS]))
+ static_cast<size_t>(std::stol(req_params[SEARCH_CUTOFF_MS])),
+ static_cast<size_t>(std::stol(req_params[MIN_LEN_1TYPO])),
+ static_cast<size_t>(std::stol(req_params[MIN_LEN_2TYPO])),
+ split_join_tokens
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 8d8026a2..6972717e 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1966,6 +1966,20 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {
{false}, 0).get();
ASSERT_EQ(0, results["hits"].size());
+ // don't join when feature is disabled
+ results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ // don't split when feature is disabled
+ results = coll1->search("pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();
+ ASSERT_EQ(0, results["hits"].size());
+
collectionManager.drop_collection("coll1");
}