Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Drop tokens should not happen on split tokens.
Kishore Nallandcdb71f87b73
diff --git a/include/index.h b/include/index.hindex b8f7cd0a..6d4ab7d7 100644--- a/include/index.h+++ b/include/index.h@@ -314,6 +314,7 @@ struct search_args {const size_t max_extra_suffix;const size_t facet_query_num_typos;const bool filter_curated_hits;+ const bool split_join_tokens;spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;@@ -333,7 +334,7 @@ struct search_args {size_t concurrency, const std::vector<const override_t*>& dynamic_overrides, size_t search_cutoff_ms,size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<infix_t>& infixes,const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,- const bool filter_curated_hits) :+ const bool filter_curated_hits, const bool split_join_tokens) :field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -346,7 +347,8 @@ struct search_args {filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms),min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo), max_candidates(max_candidates),infixes(infixes), max_extra_prefix(max_extra_prefix), max_extra_suffix(max_extra_suffix),- facet_query_num_typos(facet_query_num_typos), filter_curated_hits(filter_curated_hits) {+ facet_query_num_typos(facet_query_num_typos), filter_curated_hits(filter_curated_hits),+ split_join_tokens(split_join_tokens) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -613,7 +615,7 @@ private:const std::string &field_name,nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);- bool common_results_exist(std::vector<art_leaf*>& leaves);+ bool common_results_exist(std::vector<art_leaf*>& leaves) const;public:// for limiting number of results on multiple candidates / query rewrites@@ -708,7 +710,7 @@ public:size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,const size_t max_extra_suffix, const size_t facet_query_num_typos,- const bool filter_curated_hits) const;+ const bool filter_curated_hits, bool split_join_tokens) const;Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);@@ -786,7 +788,7 @@ public:std::vector<facet_info_t>& facet_infos) const;void resolve_space_as_typos(std::vector<std::string>& qtokens, const std::string& field_name,- std::vector<std::vector<std::string>>& resolved_queries);+ std::vector<std::vector<std::string>>& resolved_queries) const;size_t num_seq_ids() const;@@ -845,7 +847,7 @@ public:const size_t max_extra_suffix, uint32_t* filter_ids, uint32_t filter_ids_length,const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,const size_t num_search_fields, const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,- std::vector<Topster*>& ftopsters, bool is_wildcard_query,+ std::vector<Topster*>& ftopsters, bool is_wildcard_query, bool split_join_tokens,std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t>& the_fields, size_t& all_result_ids_len,uint32_t*& all_result_ids,diff --git a/src/collection.cpp b/src/collection.cppindex 438a2bae..59a12633 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1005,19 +1005,6 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s// get synonymssynonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);-- std::vector<std::vector<std::string>> space_resolved_queries;-- if(split_join_tokens) {- index->resolve_space_as_typos(field_query_tokens[i].q_include_tokens, search_field,- space_resolved_queries);- }-- // only one query is resolved for now, so just use that- if(!space_resolved_queries.empty()) {- field_query_tokens[i].q_include_tokens = space_resolved_queries[0];- synonym_reduction(space_resolved_queries[0], field_query_tokens[i].q_synonyms);- }}}@@ -1035,7 +1022,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const ssearch_stop_millis,min_len_1typo, min_len_2typo, max_candidates, infixes,max_extra_prefix, max_extra_suffix, facet_query_num_typos,- filter_curated_hits);+ filter_curated_hits, split_join_tokens);index->run_search(search_params);diff --git a/src/index.cpp b/src/index.cppindex 8365dc73..d11fcba4 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1654,7 +1654,8 @@ void Index::run_search(search_args* search_params) {search_params->max_extra_prefix,search_params->max_extra_suffix,search_params->facet_query_num_typos,- search_params->filter_curated_hits);+ search_params->filter_curated_hits,+ search_params->split_join_tokens);}void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,@@ -2101,7 +2102,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vsize_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,const size_t max_extra_suffix, const size_t facet_query_num_typos,- const bool filter_curated_hits) const {+ const bool filter_curated_hits, const bool split_join_tokens) const {search_begin = std::chrono::high_resolution_clock::now();search_stop_ms = search_cutoff_ms;@@ -2221,7 +2222,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vconcurrency, min_len_1typo, min_len_2typo, max_candidates, infixes, max_extra_prefix,max_extra_suffix,filter_ids, filter_ids_length, curated_ids, curated_ids_sorted, num_search_fields,- exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, field_query_tokens,+ exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, false, field_query_tokens,the_fields, all_result_ids_len, all_result_ids, topster_ids);if(exhaustive_search || all_result_ids_len < typo_tokens_threshold || all_result_ids_len < drop_tokens_threshold) {@@ -2232,8 +2233,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vconcurrency, min_len_1typo, min_len_2typo, max_candidates, infixes, max_extra_prefix,max_extra_suffix,filter_ids, filter_ids_length, curated_ids, curated_ids_sorted, num_search_fields,- exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, field_query_tokens,- the_fields, all_result_ids_len, all_result_ids, topster_ids);+ exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, split_join_tokens,+ field_query_tokens, the_fields, all_result_ids_len, all_result_ids, topster_ids);}//auto begin0 = std::chrono::high_resolution_clock::now();@@ -2514,7 +2515,7 @@ void Index::search_fields(const std::vector<filter>& filters,const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,const size_t num_search_fields, const uint32_t* exclude_token_ids,size_t exclude_token_ids_size, std::vector<Topster*>& ftopsters, bool is_wildcard_query,- std::vector<query_tokens_t>& field_query_tokens,+ const bool split_join_tokens, std::vector<query_tokens_t>& field_query_tokens,const std::vector<search_field_t>& the_fields, size_t& all_result_ids_len,uint32_t*& all_result_ids,spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const {@@ -2552,7 +2553,6 @@ void Index::search_fields(const std::vector<filter>& filters,// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.empty() || actual_filter_ids_length > 0) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);- const std::string& field_name = the_fields[i].name;auto field_it = search_schema.find(field_name);if(field_it == search_schema.end()) {@@ -2588,6 +2588,36 @@ void Index::search_fields(const std::vector<filter>& filters,max_extra_prefix, max_extra_suffix, field_infix, field_id, field_name, query_tokens,actual_topster, filter_ids, filter_ids_length,curated_ids_sorted, field_num_results, all_result_ids, all_result_ids_len, groups_processed);++ if(field_num_results == 0 && split_join_tokens) {+ std::vector<std::vector<std::string>> space_resolved_queries;+ resolve_space_as_typos(field_query_tokens[i].q_include_tokens, field_name,+ space_resolved_queries);++ // only one query is resolved for now, so just use that+ if(!space_resolved_queries.empty()) {+ q_include_pos_tokens.clear();+ const auto& resolved_query = space_resolved_queries[0];++ for(size_t j=0; j < resolved_query.size(); j++) {+ bool is_prefix = (j == resolved_query.size()-1);+ q_include_pos_tokens.emplace_back(j, space_resolved_queries[0][j], is_prefix);+ }++ query_tokens = q_include_pos_tokens;+ search_tokens = q_include_pos_tokens;++ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size,+ num_tokens_dropped, field_it->second, field_name,+ actual_filter_ids, actual_filter_ids_length, curated_ids_sorted, sort_fields_std,+ field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,+ field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,+ query_hashes, token_order, field_prefix,+ 0, 0, exhaustive_search,+ min_len_1typo, min_len_2typo, max_candidates);+ }+ }+} else if(actual_filter_ids_length != 0) {// indicates phrase match querycurate_filtered_ids(filters, curated_ids, exclude_token_ids,@@ -4386,9 +4416,7 @@ size_t Index::num_seq_ids() const {}void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const string& field_name,- std::vector<std::vector<std::string>>& resolved_queries) {-- std::shared_lock lock(mutex);+ std::vector<std::vector<std::string>>& resolved_queries) const {auto tree_it = search_index.find(field_name);@@ -4523,7 +4551,7 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const stri}}-bool Index::common_results_exist(std::vector<art_leaf*>& leaves) {+bool Index::common_results_exist(std::vector<art_leaf*>& leaves) const {std::vector<uint32_t> result_ids;std::vector<void*> leaf_vals;diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 808e6e01..9faa484d 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1924,7 +1924,6 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {nlohmann::json doc;doc["title"] = "The nonstick pressure cooker is a great invention.";-ASSERT_TRUE(coll1->add(doc.dump()).ok());auto results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();@@ -1968,6 +1967,16 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();ASSERT_EQ(0, results["hits"].size());+ // drop tokens should not happen on tokens split+ doc["title"] = "Pressure Copper vessel";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ results = coll1->search("pressurecopper", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("Pressure Copper vessel", results["hits"][0]["document"]["title"].get<std::string>());+collectionManager.drop_collection("coll1");}