Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Drop tokens direction.
Kishore Nallan2bef32f9c6fa
diff --git a/include/collection.h b/include/collection.hindex 90586fc1..35cefee1 100644--- a/include/collection.h+++ b/include/collection.h@@ -473,7 +473,8 @@ public:const size_t remote_embedding_timeout_ms = 30000,const size_t remote_embedding_num_tries = 2,const bool prioritize_num_matching_fields = true,- const bool group_missing_values = true) const;+ const bool group_missing_values = true,+ const drop_tokens_mode_t drop_tokens_mode = right_to_left) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/index.h b/include/index.hindex a987f3f2..b3487b39 100644--- a/include/index.h+++ b/include/index.h@@ -96,6 +96,11 @@ enum text_match_type_t {max_weight};+enum drop_tokens_mode_t {+ left_to_right,+ right_to_left,+};+struct search_args {std::vector<query_tokens_t> field_query_tokens;std::vector<search_field_t> search_fields;@@ -146,6 +151,7 @@ struct search_args {vector_query_t& vector_query;size_t facet_sample_percent;size_t facet_sample_threshold;+ drop_tokens_mode_t drop_tokens_mode;search_args(std::vector<query_tokens_t> field_query_tokens, std::vector<search_field_t> search_fields,const text_match_type_t match_type,@@ -162,7 +168,7 @@ struct search_args {size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<enable_t>& infixes,const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,const bool filter_curated_hits, const enable_t split_join_tokens, vector_query_t& vector_query,- size_t facet_sample_percent, size_t facet_sample_threshold) :+ size_t facet_sample_percent, size_t facet_sample_threshold, drop_tokens_mode_t drop_tokens_mode) :field_query_tokens(field_query_tokens),search_fields(search_fields), match_type(match_type), filter_tree_root(filter_tree_root), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -180,7 +186,8 @@ struct search_args {infixes(infixes), max_extra_prefix(max_extra_prefix), max_extra_suffix(max_extra_suffix),facet_query_num_typos(facet_query_num_typos), filter_curated_hits(filter_curated_hits),split_join_tokens(split_join_tokens), vector_query(vector_query),- facet_sample_percent(facet_sample_percent), facet_sample_threshold(facet_sample_threshold) {+ facet_sample_percent(facet_sample_percent), facet_sample_threshold(facet_sample_threshold),+ drop_tokens_mode(drop_tokens_mode) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -679,7 +686,8 @@ public:const size_t max_extra_suffix, const size_t facet_query_num_typos,const bool filter_curated_hits, enable_t split_join_tokens,const vector_query_t& vector_query, size_t facet_sample_percent, size_t facet_sample_threshold,- const std::string& collection_name) const;+ const std::string& collection_name,+ const drop_tokens_mode_t drop_tokens_mode = right_to_left) const;void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,const bool is_update);diff --git a/src/collection.cpp b/src/collection.cppindex adda2fc5..eb9dee36 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1116,7 +1116,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const size_t remote_embedding_timeout_ms,const size_t remote_embedding_num_tries,const bool prioritize_num_matching_fields,- const bool group_missing_values) const {+ const bool group_missing_values,+ const drop_tokens_mode_t drop_tokens_mode) const {std::shared_lock lock(mutex);@@ -1631,7 +1632,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,min_len_1typo, min_len_2typo, max_candidates, infixes,max_extra_prefix, max_extra_suffix, facet_query_num_typos,filter_curated_hits, split_join_tokens, vector_query,- facet_sample_percent, facet_sample_threshold);+ facet_sample_percent, facet_sample_threshold, drop_tokens_mode);std::unique_ptr<search_args> search_params_guard(search_params);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 98cbc50b..5ad7d814 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -760,6 +760,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *FACET_SAMPLE_THRESHOLD = "facet_sample_threshold";const char *PRIORITIZE_NUM_MATCHING_FIELDS = "prioritize_num_matching_fields";+ const char *DROP_TOKENS_MODE = "drop_tokens_mode";// enrich params with values from embedded paramsfor(auto& item: embedded_params.items()) {@@ -872,6 +873,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& resize_t facet_sample_threshold = 0;bool prioritize_num_matching_fields = true;+ std::string drop_tokens_mode_str = "right_to_left";std::unordered_map<std::string, size_t*> unsigned_int_values = {{MIN_LEN_1TYPO, &min_len_1typo},@@ -909,6 +911,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{HIGHLIGHT_END_TAG, &highlight_end_tag},{PINNED_HITS, &pinned_hits_str},{HIDDEN_HITS, &hidden_hits_str},+ {DROP_TOKENS_MODE, &drop_tokens_mode_str},};std::unordered_map<std::string, bool*> bool_values = {@@ -1069,6 +1072,12 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reIndex::NUM_CANDIDATES_DEFAULT_MIN);}+ auto drop_tokens_mode_op = magic_enum::enum_cast<drop_tokens_mode_t>(drop_tokens_mode_str);+ drop_tokens_mode_t drop_tokens_mode;+ if(drop_tokens_mode_op.has_value()) {+ drop_tokens_mode = drop_tokens_mode_op.value();+ }+Option<nlohmann::json> result_op = collection->search(raw_query, search_fields, simple_filter_query, facet_fields,sort_fields, num_typos,per_page,@@ -1115,8 +1124,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reremote_embedding_timeout_ms,remote_embedding_num_tries,prioritize_num_matching_fields,- group_missing_values- );+ group_missing_values,+ drop_tokens_mode);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/src/index.cpp b/src/index.cppindex 5059fd8d..4b5d5cd7 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2305,7 +2305,8 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->vector_query,search_params->facet_sample_percent,search_params->facet_sample_threshold,- collection_name);+ collection_name,+ search_params->drop_tokens_mode);}void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,@@ -2757,7 +2758,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consconst bool filter_curated_hits, const enable_t split_join_tokens,const vector_query_t& vector_query,size_t facet_sample_percent, size_t facet_sample_threshold,- const std::string& collection_name) const {+ const std::string& collection_name,+ const drop_tokens_mode_t drop_tokens_mode) const {std::shared_lock lock(mutex);uint32_t filter_ids_length = 0;@@ -3100,16 +3102,24 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor (size_t qi = 0; qi < all_queries.size(); qi++) {auto& orig_tokens = all_queries[qi];size_t num_tokens_dropped = 0;+ auto curr_direction = drop_tokens_mode;+ size_t total_dirs_done = 0;while(exhaustive_search || all_result_ids_len < drop_tokens_threshold) {// When atleast two tokens from the query are available we can drop onestd::vector<token_t> truncated_tokens;std::vector<token_t> dropped_tokens;- if(orig_tokens.size() > 1 && num_tokens_dropped < 2*(orig_tokens.size()-1)) {- bool prefix_search = false;+ if(num_tokens_dropped >= orig_tokens.size() - 1) {+ // swap direction and reset counter+ curr_direction = (curr_direction == right_to_left) ? left_to_right : right_to_left;+ num_tokens_dropped = 0;+ total_dirs_done++;+ }- if (num_tokens_dropped < orig_tokens.size() - 1) {+ if(orig_tokens.size() > 1 && total_dirs_done < 2) {+ bool prefix_search = false;+ if (curr_direction == right_to_left) {// drop from rightsize_t truncated_len = orig_tokens.size() - num_tokens_dropped - 1;for (size_t i = 0; i < orig_tokens.size(); i++) {@@ -3122,7 +3132,7 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons} else {// drop from leftprefix_search = true;- size_t start_index = (num_tokens_dropped + 1) - orig_tokens.size() + 1;+ size_t start_index = (num_tokens_dropped + 1);for(size_t i = 0; i < orig_tokens.size(); i++) {if(i >= start_index) {truncated_tokens.emplace_back(orig_tokens[i]);diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 266eb0a6..fc5468eb 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2378,6 +2378,47 @@ TEST_F(CollectionSpecificMoreTest, ExhaustiveSearchWithoutExplicitDropTokens) {ASSERT_EQ(2, res["hits"].size());}+TEST_F(CollectionSpecificMoreTest, DropTokensLeftToRightFirst) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["title"] = "alpha beta";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "beta gamma";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ bool exhaustive_search = false;+ size_t drop_tokens_threshold = 1;++ auto res = coll1->search("alpha beta gamma", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {false}, drop_tokens_threshold,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,+ 0, 30000, 2, true, true, left_to_right).get();++ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("1", res["hits"][0]["document"]["id"].get<std::string>());++ res = coll1->search("alpha beta gamma", {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {false}, drop_tokens_threshold,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 10000,+ 4, 7, fallback, 4, {off}, 100, 100, 2, 2, false, "", true, 0, max_score, 100, 0,+ 0, 30000, 2, true, true, right_to_left).get();++ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());+}+TEST_F(CollectionSpecificMoreTest, DoNotHighlightFieldsForSpecialCharacterQuery) {nlohmann::json schema = R"({"name": "coll1",