Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Make splitting/joining of tokens configurable.
Kishore Nallane3d5e3df739d
diff --git a/include/collection.h b/include/collection.hindex 8fe4d616..6a273710 100644--- a/include/collection.h+++ b/include/collection.h@@ -393,7 +393,8 @@ public:const bool exhaustive_search = false,size_t search_stop_millis = 6000*1000,size_t min_len_1typo = 4,- size_t min_len_2typo = 7) const;+ size_t min_len_2typo = 7,+ bool split_join_tokens = true) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/src/collection.cpp b/src/collection.cppindex f1c522d1..a326b15e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -677,7 +677,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sconst bool exhaustive_search,const size_t search_stop_millis,const size_t min_len_1typo,- const size_t min_len_2typo) const {+ const size_t min_len_2typo,+ bool split_join_tokens) const {std::shared_lock lock(mutex);@@ -968,8 +969,11 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const ssynonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);std::vector<std::vector<std::string>> space_resolved_queries;- index->resolve_space_as_typos(field_query_tokens[i].q_include_tokens, search_field,- space_resolved_queries);++ if(split_join_tokens) {+ index->resolve_space_as_typos(field_query_tokens[i].q_include_tokens, search_field,+ space_resolved_queries);+ }// only one query is resolved for now, so just use thatif(!space_resolved_queries.empty()) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 0b3739d2..c3e0eae1 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -584,6 +584,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *SEARCH_CUTOFF_MS = "search_cutoff_ms";const char *EXHAUSTIVE_SEARCH = "exhaustive_search";+ const char *SPLIT_JOIN_TOKENS = "split_join_tokens";if(req_params.count(NUM_TYPOS) == 0) {req_params[NUM_TYPOS] = "2";@@ -702,6 +703,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rereq_params[EXHAUSTIVE_SEARCH] = "false";}+ if(req_params.count(SPLIT_JOIN_TOKENS) == 0) {+ req_params[SPLIT_JOIN_TOKENS] = "true";+ }+std::vector<std::string> query_by_weights_str;std::vector<size_t> query_by_weights;@@ -778,6 +783,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rebool prioritize_exact_match = (req_params[PRIORITIZE_EXACT_MATCH] == "true");bool pre_segmented_query = (req_params[PRE_SEGMENTED_QUERY] == "true");bool exhaustive_search = (req_params[EXHAUSTIVE_SEARCH] == "true");+ bool split_join_tokens = (req_params[SPLIT_JOIN_TOKENS] == "true");std::string filter_str = req_params.count(FILTER) != 0 ? req_params[FILTER] : "";@@ -883,7 +889,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reenable_overrides,req_params[HIGHLIGHT_FIELDS],exhaustive_search,- static_cast<size_t>(std::stol(req_params[SEARCH_CUTOFF_MS]))+ static_cast<size_t>(std::stol(req_params[SEARCH_CUTOFF_MS])),+ static_cast<size_t>(std::stol(req_params[MIN_LEN_1TYPO])),+ static_cast<size_t>(std::stol(req_params[MIN_LEN_2TYPO])),+ split_join_tokens);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 8d8026a2..6972717e 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1966,6 +1966,20 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {{false}, 0).get();ASSERT_EQ(0, results["hits"].size());+ // don't join when feature is disabled+ results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();+ ASSERT_EQ(0, results["hits"].size());++ // don't split when feature is disabled+ results = coll1->search("pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();+ ASSERT_EQ(0, results["hits"].size());+collectionManager.drop_collection("coll1");}