Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Speed up exact matching on single tokens.
Kishore Nallan76915eed2016
diff --git a/include/index.h b/include/index.hindex fc9367e5..00df62d2 100644--- a/include/index.h+++ b/include/index.h@@ -232,6 +232,7 @@ private:const size_t combination_limit = Index::COMBINATION_LIMIT) const;void search_candidates(const uint8_t & field_id,+ bool field_is_array,uint32_t* filter_ids, size_t filter_ids_length,const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,const std::vector<uint32_t>& curated_ids,@@ -324,17 +325,17 @@ public:static void concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids);void score_results(const std::vector<sort_by> &sort_fields, const uint16_t &query_index, const uint8_t &field_id,- const uint32_t total_cost, Topster *topster, const std::vector<art_leaf *> &query_suggestion,+ bool field_is_array, const uint32_t total_cost,+ Topster *topster, const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t> &groups_processed,- const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions,const uint32_t seq_id, const int sort_order[3],std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const size_t group_limit,const std::vector<std::string> &group_by_fields, uint32_t token_bits,const std::vector<token_t> &query_tokens,- bool use_single_token_score,- bool prioritize_exact_match) const;+ bool prioritize_exact_match,+ std::vector<posting_list_t::iterator_t>& posting_lists) const;static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);diff --git a/include/match_score.h b/include/match_score.hindex a42da5c0..c76ecb97 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -44,7 +44,8 @@ struct Match {}- Match(uint8_t words_present, uint8_t distance) : words_present(words_present), distance(distance), exact_match(0) {+ Match(uint8_t words_present, uint8_t distance, uint8_t exact_match = 0) :+ words_present(words_present), distance(distance), exact_match(exact_match) {}diff --git a/include/posting_list.h b/include/posting_list.hindex 39c20354..ee09777c 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -162,6 +162,11 @@ public:std::vector<iterator_t>& its,std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos);++ static bool is_single_token_exact_match(+ posting_list_t::iterator_t& it,+ bool field_is_array+ );};template<class T>diff --git a/src/index.cpp b/src/index.cppindex 266b7072..1a2b1001 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -881,7 +881,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}}-void Index::search_candidates(const uint8_t & field_id,+void Index::search_candidates(const uint8_t & field_id, bool field_is_array,uint32_t* filter_ids, size_t filter_ids_length,const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,const std::vector<uint32_t>& curated_ids,@@ -960,26 +960,18 @@ void Index::search_candidates(const uint8_t & field_id,excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length);- // We will have to be judicious about computing full match score: only when token does not match exact query- bool use_single_token_score = (query_suggestion.size() == 1) &&- (query_suggestion.size() == query_tokens.size()) &&- ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));+ // We fetch offset positions only for multi token query+ bool fetch_offsets = (query_suggestion.size() > 1);std::vector<uint32_t> result_id_vec;posting_t::block_intersector_t intersector(posting_lists, iter_state);intersector.intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;-- if(!use_single_token_score) {- posting_list_t::get_offsets(its, array_token_positions);- }-- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion, groups_processed, array_token_positions,+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, field_is_array,+ total_cost, topster, query_suggestion, groups_processed,seq_id, sort_order, field_values, geopoint_indices,group_limit, group_by_fields, token_bits, query_tokens,- use_single_token_score, prioritize_exact_match);+ prioritize_exact_match, its);result_id_vec.push_back(seq_id);});@@ -1567,12 +1559,14 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint32_t token_bits = 255;+ std::vector<posting_list_t::iterator_t> plists;+for(size_t i = 0; i < filter_ids_length; i++) {const uint32_t seq_id = filter_ids[i];- score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},- groups_processed, {}, seq_id, sort_order, field_values, geopoint_indices,+ score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0, topster, {},+ groups_processed, seq_id, sort_order, field_values, geopoint_indices,group_limit, group_by_fields, token_bits, {},- true, prioritize_exact_match);+ prioritize_exact_match, plists);}collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);@@ -1879,7 +1873,9 @@ void Index::search_field(const uint8_t & field_id,const size_t combination_limit) const {size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;- if(search_schema.at(field).locale != "" && search_schema.at(field).locale != "en") {+ auto& the_field = search_schema.at(field);++ if(the_field.locale != "" && the_field.locale != "en") {// disable fuzzy trie traversal for non-english localesmax_cost = 0;}@@ -1990,7 +1986,8 @@ void Index::search_field(const uint8_t & field_id,if(!token_candidates_vec.empty()) {// If atleast one token is found, go ahead and search for candidates- search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,+ search_candidates(field_id, the_field.is_array(), filter_ids, filter_ids_length,+ exclude_token_ids, exclude_token_ids_size,curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,groups_processed, all_result_ids, all_result_ids_len, field_num_results,typo_tokens_threshold, group_limit, group_by_fields, query_tokens,@@ -2064,18 +2061,17 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect}void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,- const uint8_t & field_id, const uint32_t total_cost, Topster* topster,+ const uint8_t & field_id, bool field_is_array, const uint32_t total_cost, Topster* topster,const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions,const uint32_t seq_id, const int sort_order[3],std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const size_t group_limit, const std::vector<std::string>& group_by_fields,uint32_t token_bits,const std::vector<token_t>& query_tokens,- bool use_single_token_score,- bool prioritize_exact_match) const {+ bool prioritize_exact_match,+ std::vector<posting_list_t::iterator_t>& posting_lists) const {spp::sparse_hash_map<uint32_t, int64_t>* TEXT_MATCH_SENTINEL = &text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t>* SEQ_ID_SENTINEL = &seq_id_sentinel_value;@@ -2117,13 +2113,19 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16uint64_t match_score = 0;- if (use_single_token_score || array_token_positions.empty()) {- Match single_token_match = Match(1, 0);- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);- match_score = single_token_match_score;+ if (posting_lists.size() == 1) {+ const uint8_t is_exact_match = uint8_t(+ total_cost == 0 && prioritize_exact_match &&+ posting_list_t::is_single_token_exact_match(posting_lists[0], field_is_array)+ );+ Match single_token_match = Match(1, 0, is_exact_match);+ match_score = single_token_match.get_match_score(total_cost);} else {uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;+ posting_list_t::get_offsets(posting_lists, array_token_positions);+for (const auto& kv: array_token_positions) {const std::vector<token_positions_t>& token_positions = kv.second;if (token_positions.empty()) {diff --git a/src/posting_list.cpp b/src/posting_list.cppindex f4501f67..944ac8d9 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -791,6 +791,49 @@ bool posting_list_t::get_offsets(std::vector<iterator_t>& its,return true;}+bool posting_list_t::is_single_token_exact_match(posting_list_t::iterator_t& it, bool field_is_array) {+ block_t* curr_block = it.block();+ uint32_t curr_index = it.index();++ if(curr_block == nullptr || curr_index == UINT32_MAX) {+ return false;+ }++ uint32_t* offsets = it.offsets;+ uint32_t start_offset = it.offset_index[curr_index];++ if(!field_is_array && offsets[start_offset] != 1) {+ // allows us to skip other computes fast+ return false;+ }++ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?+ curr_block->offsets.getLength() :+ it.offset_index[curr_index + 1];++ if(field_is_array) {+ int prev_pos = -1;++ while(start_offset < end_offset) {+ int pos = offsets[start_offset];+ start_offset++;++ if(pos == prev_pos && pos == 1 && start_offset+1 < end_offset && offsets[start_offset+1] == 0) {+ return true;+ }++ prev_pos = pos;+ }++ return false;+ } else if((end_offset - start_offset) == 2 && offsets[end_offset-1] == 0) {+ // already checked for `offsets[start_offset] == 1` first+ return true;+ }++ return false;+}+bool posting_list_t::at_end(const std::vector<posting_list_t::iterator_t>& its) {// if any one iterator is at end, we can stopfor(const auto& it : its) {