Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor prefix search expansion.
Kishore Nallan85d5ce06b2d6
diff --git a/include/index.h b/include/index.hindex e0fdc8b2..09e77915 100644--- a/include/index.h+++ b/include/index.h@@ -354,13 +354,6 @@ private:uint32_t& token_bits,uint64& qhash);- static bool is_valid_token_prefix(const std::vector<search_field_t>& the_fields, size_t field_id,- const unsigned char* token_c_str, size_t token_len,- const std::vector<uint32_t>& num_typos, const std::vector<bool>& prefixes,- size_t token_num_typos, bool token_prefix,- const spp::sparse_hash_map<std::string, art_tree*>& search_index,- const std::vector<uint32_t>& prev_token_doc_ids);-void log_leaves(int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,@@ -461,6 +454,12 @@ private:std::set<uint64>& query_hashes,std::vector<uint32_t>& id_buff) const;+ static void popular_fields_of_token(const spp::sparse_hash_map<std::string, art_tree*>& search_index,+ const std::string& previous_token,+ const std::vector<search_field_t>& the_fields,+ const size_t num_search_fields,+ std::vector<size_t>& popular_field_ids);+void do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length, const std::vector<filter>& filters,const bool enable_short_circuit) const;@@ -778,6 +777,7 @@ public:void fuzzy_search_fields(const std::vector<search_field_t>& the_fields,const std::vector<token_t>& query_tokens,+ const bool dropped_tokens,const uint32_t* exclude_token_ids,size_t exclude_token_ids_size,const uint32_t* filter_ids, size_t filter_ids_length,@@ -805,6 +805,7 @@ public:const std::vector<size_t>& geopoint_indices) const;void find_across_fields(const token_t& previous_token,+ const std::string& previous_token_str,const std::vector<uint32_t>& num_typos,const std::vector<bool>& prefixes,const std::vector<search_field_t>& the_fields,diff --git a/src/collection.cpp b/src/collection.cppindex dd74a585..52e9d421 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1571,7 +1571,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,wrapper_doc["highlights"].push_back(h_json);}- //wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;+ wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;if(group_limit && group_key.empty()) {for(const auto& field_name: group_by_fields) {diff --git a/src/index.cpp b/src/index.cppindex 854d5856..8a0290d7 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1265,38 +1265,6 @@ void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {}}-bool Index::is_valid_token_prefix(const std::vector<search_field_t>& the_fields, size_t field_id,- const unsigned char* token_c_str, size_t token_len,- const std::vector<uint32_t>& num_typos, const std::vector<bool>& prefixes,- size_t token_num_typos, bool token_prefix,- const spp::sparse_hash_map<std::string, art_tree*>& search_index,- const std::vector<uint32_t>& prev_token_doc_ids) {-- const std::string& field_name = the_fields[field_id].name;- const uint32_t field_num_typos = (field_id < num_typos.size()) ? num_typos[field_id] : num_typos[0];- const bool field_prefix = (field_id < prefixes.size()) ? prefixes[field_id] : prefixes[0];-- if (token_num_typos > field_num_typos) {- // since the token can come from any field, we still have to respect per-field num_typos- return false;- }-- if (token_prefix && !field_prefix) {- // even though this token is an outcome of prefix search, we can't use it for this field, since- // this field has prefix search disabled.- return false;- }-- art_tree* tree = search_index.at(field_name);- art_leaf* leaf = static_cast<art_leaf*>(art_search(tree, token_c_str, token_len));-- if (!leaf) {- return false;- }-- return posting_t::contains_atleast_one(leaf->values, &prev_token_doc_ids[0], prev_token_doc_ids.size());-}-void Index::search_all_candidates(const size_t num_search_fields,const std::vector<search_field_t>& the_fields,const uint32_t* filter_ids, size_t filter_ids_length,@@ -1331,89 +1299,6 @@ void Index::search_all_candidates(const size_t num_search_fields,LOG(INFO) << "token_candidates_vec.size(): " << token_candidates_vec.size();}*/- std::set<std::string> trimmed_candidates;-- if(token_candidates_vec.size() >= 2 && token_candidates_vec.back().candidates.size() > max_candidates) {- std::vector<uint32_t> prev_token_doc_ids; // documents that contain the previous token across fields- std::vector<size_t> top_prefix_field_ids; // fields which contained the token the most across documents-- find_across_fields(query_tokens[query_tokens.size()-2], num_typos, prefixes, the_fields, num_search_fields,- filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,- prev_token_doc_ids, top_prefix_field_ids);-- //LOG(INFO) << "prev_token_doc_ids found: " << prev_token_doc_ids.size();-- std::unordered_set<size_t> processed_field_ids;-- for(auto& token_str: token_candidates_vec.back().candidates) {- //LOG(INFO) << "Prefix token: " << token_str;- const bool prefix_search = query_tokens.back().is_prefix_searched;- const uint32_t token_num_typos = query_tokens.back().num_typos;- const bool token_prefix = (token_str.size() > token_candidates_vec.back().token.value.size());-- auto token_c_str = (const unsigned char*) token_str.c_str();- const size_t token_len = token_str.size() + 1;-- for(size_t i = 0; i < num_search_fields; i++) {- if(!is_valid_token_prefix(the_fields, i, token_c_str, token_len, num_typos, prefixes,- token_num_typos, token_prefix, search_index, prev_token_doc_ids)) {- continue;- }-- trimmed_candidates.insert(token_str);- processed_field_ids.insert(i);- if(trimmed_candidates.size() == max_candidates) {- goto outer1;- }- }- }-- outer1:-- for(auto& token_str: token_candidates_vec.back().candidates) {- //LOG(INFO) << "Prefix token: " << token_str;- const bool prefix_search = query_tokens.back().is_prefix_searched;- const uint32_t token_num_typos = query_tokens.back().num_typos;- const bool token_prefix = (token_str.size() > token_candidates_vec.back().token.value.size());- auto token_c_str = (const unsigned char*) token_str.c_str();- const size_t token_len = token_str.size() + 1;-- size_t top_fields_processed = 0;- for(auto field_id: top_prefix_field_ids) {- if(processed_field_ids.count(field_id) != 0) {- continue;- }-- if(!is_valid_token_prefix(the_fields, field_id, token_c_str, token_len, num_typos, prefixes,- token_num_typos, token_prefix, search_index, prev_token_doc_ids)) {- continue;- }-- top_fields_processed++;- trimmed_candidates.insert(token_str);-- if(top_fields_processed == 3) {- // limit to only 3 unprocessed top fields- goto outer2;- }- }- }-- outer2:-- if(trimmed_candidates.empty()) {- return ;- }-- /*LOG(INFO) << "Final trimmed_candidates.size: " << trimmed_candidates.size();- for(const auto& trimmed_candidate: trimmed_candidates) {- LOG(INFO) << "trimmed_candidate: " << trimmed_candidate;- }*/-- token_candidates_vec.back().candidates.clear();- token_candidates_vec.back().candidates.assign(trimmed_candidates.begin(), trimmed_candidates.end());- }-auto product = []( long long a, tok_candidates & b ) { return a*b.candidates.size(); };long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);@@ -2679,7 +2564,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v}}- fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, excluded_result_ids,+ fuzzy_search_fields(the_fields, field_query_tokens[0].q_include_tokens, false, excluded_result_ids,excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,@@ -2716,7 +2601,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vspace_resolved_queries[0][j].size(), 0);}- fuzzy_search_fields(the_fields, resolved_tokens, excluded_result_ids,+ fuzzy_search_fields(the_fields, resolved_tokens, false, excluded_result_ids,excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,@@ -2772,7 +2657,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vdrop_token_prefixes.push_back(p && prefix_search);}- fuzzy_search_fields(the_fields, truncated_tokens, excluded_result_ids,+ fuzzy_search_fields(the_fields, truncated_tokens, true, excluded_result_ids,excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, num_typos, searched_queries, qtoken_set, topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,@@ -2997,6 +2882,7 @@ void Index::process_curated_ids(const std::vector<std::pair<uint32_t, uint32_t>>void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,const std::vector<token_t>& query_tokens,+ const bool dropped_tokens,const uint32_t* exclude_token_ids,size_t exclude_token_ids_size,const uint32_t* filter_ids, size_t filter_ids_length,@@ -3087,7 +2973,34 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,} else {//auto begin = std::chrono::high_resolution_clock::now();+ // Prefix query with a preceding token should be handled in such a way that we give preference to+ // possible phrase continuation. Example: "steve j" for "steve jobs" name field query. To do this,+ // we will first attempt to match the prefix with the most "popular" fields of the preceding token.+ // Tokens matched from popular fields will also be searched across other query fields.+ // Only when we find *no results* for such an expansion, we will attempt cross field matching.+ bool last_token = query_tokens.size() > 1 && !dropped_tokens &&+ (token_index == (query_tokens.size() - 1));++ std::vector<size_t> query_field_ids(num_search_fields);for(size_t field_id = 0; field_id < num_search_fields; field_id++) {+ query_field_ids[field_id] = the_fields[field_id].orig_index;+ }++ std::vector<size_t> popular_field_ids; // fields containing the token most across documents++ if(last_token) {+ popular_fields_of_token(search_index,+ token_candidates_vec.back().candidates[0],+ the_fields, num_search_fields, popular_field_ids);++ if(popular_field_ids.empty()) {+ break;+ }+ }++ const std::vector<size_t>& field_ids = last_token ? popular_field_ids : query_field_ids;++ for(size_t field_id: field_ids) {// NOTE: when accessing other field ordered properties like prefixes or num_typos we have to index// them by `the_field.orig_index` since the original fields could be reordered on their weights.auto& the_field = the_fields[field_id];@@ -3127,19 +3040,109 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,continue;}+ std::vector<uint32_t> prev_token_doc_ids; // documents that contain the previous token++ if(last_token) {+ auto& prev_token = token_candidates_vec.back().candidates[0];+ art_leaf* prev_leaf = static_cast<art_leaf*>(+ art_search(search_index.at(the_field.name),+ reinterpret_cast<const unsigned char*>(prev_token.c_str()),+ prev_token.size() + 1));++ if(!prev_leaf) {+ continue;+ }++ posting_t::merge({prev_leaf->values}, prev_token_doc_ids);+ }+for(size_t i = 0; i < field_leaves.size(); i++) {auto leaf = field_leaves[i];std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);if(unique_tokens.count(tok) == 0) {+ if(last_token) {+ if(!posting_t::contains_atleast_one(leaf->values, &prev_token_doc_ids[0],+ prev_token_doc_ids.size())) {+ continue;+ }+ }+unique_tokens.emplace(tok);leaf_tokens.push_back(tok);}++ if(leaf_tokens.size() >= max_candidates) {+ token_cost_cache.emplace(token_cost_hash, leaf_tokens);+ goto token_done;+ }}token_cost_cache.emplace(token_cost_hash, leaf_tokens);}++ if(last_token && leaf_tokens.size() < max_candidates) {+ // field-wise matching with previous token has failed, have to look at cross fields matching docs+ std::vector<uint32_t> prev_token_doc_ids;+ find_across_fields(token_candidates_vec.back().token,+ token_candidates_vec.back().candidates[0],+ num_typos, prefixes,+ the_fields, num_search_fields, filter_ids, filter_ids_length, exclude_token_ids,+ exclude_token_ids_size, prev_token_doc_ids, popular_field_ids);++ for(size_t field_id: query_field_ids) {+ auto& the_field = the_fields[field_id];+ const bool field_prefix = (the_field.orig_index < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;+ const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;+ const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;+ int64_t field_num_typos = (the_field.orig_index < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];++ auto& locale = search_schema.at(the_field.name).locale;+ if(locale != "" && locale != "en" && locale != "th" && !Tokenizer::is_cyrillic(locale)) {+ // disable fuzzy trie traversal for non-english locales+ field_num_typos = 0;+ }++ if(costs[token_index] > field_num_typos) {+ continue;+ }++ std::vector<art_leaf*> field_leaves;+ int max_words = 100000;+ art_fuzzy_search(search_index.at(the_field.name), (const unsigned char *) token.c_str(), token_len,+ costs[token_index], costs[token_index], max_words, token_order, prefix_search,+ filter_ids, filter_ids_length, field_leaves, unique_tokens);++ if(field_leaves.empty()) {+ // look at the next field+ continue;+ }++ for(size_t i = 0; i < field_leaves.size(); i++) {+ auto leaf = field_leaves[i];+ std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);+ if(unique_tokens.count(tok) == 0) {+ if(!posting_t::contains_atleast_one(leaf->values, &prev_token_doc_ids[0],+ prev_token_doc_ids.size())) {+ continue;+ }++ unique_tokens.emplace(tok);+ leaf_tokens.push_back(tok);+ }++ if(leaf_tokens.size() >= max_candidates) {+ token_cost_cache.emplace(token_cost_hash, leaf_tokens);+ goto token_done;+ }+ }++ token_cost_cache.emplace(token_cost_hash, leaf_tokens);+ }+ }}+ token_done:+if(!leaf_tokens.empty()) {//log_leaves(costs[token_index], token, leaves);token_candidates_vec.push_back(tok_candidates{query_tokens[token_index], costs[token_index],@@ -3169,7 +3172,6 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,if(token_candidates_vec.size() == query_tokens.size()) {std::vector<uint32_t> id_buff;-search_all_candidates(num_search_fields, the_fields, filter_ids, filter_ids_length,exclude_token_ids, exclude_token_ids_size,sort_fields, token_candidates_vec, searched_queries, qtoken_set, topster,@@ -3203,7 +3205,40 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,}}+void Index::popular_fields_of_token(const spp::sparse_hash_map<std::string, art_tree*>& search_index,+ const std::string& previous_token,+ const std::vector<search_field_t>& the_fields,+ const size_t num_search_fields,+ std::vector<size_t>& popular_field_ids) {++ const auto token_c_str = (const unsigned char*) previous_token.c_str();+ const int token_len = (int) previous_token.size() + 1;++ std::vector<std::pair<size_t, size_t>> field_id_doc_counts;++ for(size_t i = 0; i < num_search_fields; i++) {+ const std::string& field_name = the_fields[i].name;+ auto leaf = static_cast<art_leaf*>(art_search(search_index.at(field_name), token_c_str, token_len));++ if(!leaf) {+ continue;+ }++ auto num_docs = posting_t::num_ids(leaf->values);+ field_id_doc_counts.emplace_back(i, num_docs);+ }++ std::sort(field_id_doc_counts.begin(), field_id_doc_counts.end(), [](const auto& p1, const auto& p2) {+ return p1.second > p2.second;+ });++ for(const auto& field_id_doc_count: field_id_doc_counts) {+ popular_field_ids.push_back(field_id_doc_count.first);+ }+}+void Index::find_across_fields(const token_t& previous_token,+ const std::string& previous_token_str,const std::vector<uint32_t>& num_typos,const std::vector<bool>& prefixes,const std::vector<search_field_t>& the_fields,@@ -3225,7 +3260,7 @@ void Index::find_across_fields(const token_t& previous_token,const uint32_t token_num_typos = previous_token.num_typos;const bool token_prefix = previous_token.is_prefix_searched;- auto& token_str = previous_token.value;+ auto& token_str = previous_token_str;auto token_c_str = (const unsigned char*) token_str.c_str();const size_t token_len = token_str.size() + 1;std::vector<posting_list_t::iterator_t> its;@@ -3869,7 +3904,7 @@ void Index::do_synonym_search(const std::vector<search_field_t>& the_fields,for(const auto& syn_tokens: q_pos_synonyms) {query_hashes.clear();- fuzzy_search_fields(the_fields, syn_tokens, exclude_token_ids,+ fuzzy_search_fields(the_fields, syn_tokens, false, exclude_token_ids,exclude_token_ids_size, filter_ids, filter_ids_length, curated_ids_sorted,sort_fields_std, {0}, searched_queries, qtoken_set, actual_topster, groups_processed,all_result_ids, all_result_ids_len, group_limit, group_by_fields, prioritize_exact_match,diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 4ed3caf6..ba14f33a 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -141,9 +141,24 @@ TEST_F(CollectionSpecificMoreTest, PrefixExpansionOnMultiField) {coll1->add(doc.dump());}- auto results = coll1->search("john s", {"location", "name"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true}).get();+ auto results = coll1->search("john s", {"location", "name"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ 0, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "title", 20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false,+ true, "", false, 6000*1000, 4, 7, off, 4).get();++ // tokens are ordered by max_score and prefix continuation on the same field is prioritized+ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][2]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][3]["document"]["id"].get<std::string>());++ // when more than 4 candidates are requested, "s" matches with other fields are returned+ results = coll1->search("john s", {"location", "name"}, "", {}, {}, {0}, 100, 1, MAX_SCORE, {true},+ 0, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "title", 20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false,+ true, "", false, 6000*1000, 4, 7, off, 10).get();- // tokens are ordered by max_score, but prefix continuation on the same field should be prioritizedASSERT_EQ(7, results["hits"].size());ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex a7e8a2a2..93bf05a4 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -473,7 +473,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {ASSERT_EQ(4, results["hits"].size());ASSERT_EQ(11, results["found"].get<uint32_t>());- std::vector<std::string> ids = {"19", "6", "21", "22"};+ std::vector<std::string> ids = {"19", "22", "6", "13"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -1040,7 +1040,7 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {FREQUENCY, {true}, 1000, empty, empty, 10).get();ASSERT_EQ(3, results["hits"].size());- ASSERT_EQ(7, results["found"].get<int>());+ ASSERT_EQ(6, results["found"].get<int>());// cannot fetch more than in-built limit of 250auto res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 251, 1,@@ -1062,13 +1062,13 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {FREQUENCY, {true}, 1000, empty, empty, 10).get();ASSERT_EQ(3, results["hits"].size());- ASSERT_EQ(7, results["found"].get<int>());+ ASSERT_EQ(6, results["found"].get<int>());results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 3, 2,FREQUENCY, {true}, 1000, empty, empty, 10).get();ASSERT_EQ(3, results["hits"].size());- ASSERT_EQ(7, results["found"].get<int>());+ ASSERT_EQ(6, results["found"].get<int>());collectionManager.drop_collection("coll_mul_fields");}