Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Speed up exact matching on single tokens.

Kishore Nallan76915eed2016
diff --git a/include/index.h b/include/index.h
index fc9367e5..00df62d2 100644
--- a/include/index.h
+++ b/include/index.h
@@ -232,6 +232,7 @@ private:
const size_t combination_limit = Index::COMBINATION_LIMIT) const;
void search_candidates(const uint8_t & field_id,
+ bool field_is_array,
uint32_t* filter_ids, size_t filter_ids_length,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
const std::vector<uint32_t>& curated_ids,
@@ -324,17 +325,17 @@ public:
static void concat_topster_ids(Topster* topster, spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids);
void score_results(const std::vector<sort_by> &sort_fields, const uint16_t &query_index, const uint8_t &field_id,
- const uint32_t total_cost, Topster *topster, const std::vector<art_leaf *> &query_suggestion,
+ bool field_is_array, const uint32_t total_cost,
+ Topster *topster, const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_set<uint64_t> &groups_processed,
- const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions,
const uint32_t seq_id, const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
const size_t group_limit,
const std::vector<std::string> &group_by_fields, uint32_t token_bits,
const std::vector<token_t> &query_tokens,
- bool use_single_token_score,
- bool prioritize_exact_match) const;
+ bool prioritize_exact_match,
+ std::vector<posting_list_t::iterator_t>& posting_lists) const;
static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);
diff --git a/include/match_score.h b/include/match_score.h
index a42da5c0..c76ecb97 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -44,7 +44,8 @@ struct Match {
}
- Match(uint8_t words_present, uint8_t distance) : words_present(words_present), distance(distance), exact_match(0) {
+ Match(uint8_t words_present, uint8_t distance, uint8_t exact_match = 0) :
+ words_present(words_present), distance(distance), exact_match(exact_match) {
}
diff --git a/include/posting_list.h b/include/posting_list.h
index 39c20354..ee09777c 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -162,6 +162,11 @@ public:
std::vector<iterator_t>& its,
std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos
);
+
+ static bool is_single_token_exact_match(
+ posting_list_t::iterator_t& it,
+ bool field_is_array
+ );
};
template<class T>
diff --git a/src/index.cpp b/src/index.cpp
index 266b7072..1a2b1001 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -881,7 +881,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
}
}
-void Index::search_candidates(const uint8_t & field_id,
+void Index::search_candidates(const uint8_t & field_id, bool field_is_array,
uint32_t* filter_ids, size_t filter_ids_length,
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
const std::vector<uint32_t>& curated_ids,
@@ -960,26 +960,18 @@ void Index::search_candidates(const uint8_t & field_id,
excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length
);
- // We will have to be judicious about computing full match score: only when token does not match exact query
- bool use_single_token_score = (query_suggestion.size() == 1) &&
- (query_suggestion.size() == query_tokens.size()) &&
- ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));
+ // We fetch offset positions only for multi token query
+ bool fetch_offsets = (query_suggestion.size() > 1);
std::vector<uint32_t> result_id_vec;
posting_t::block_intersector_t intersector(posting_lists, iter_state);
intersector.intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
-
- if(!use_single_token_score) {
- posting_list_t::get_offsets(its, array_token_positions);
- }
-
- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
- query_suggestion, groups_processed, array_token_positions,
+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, field_is_array,
+ total_cost, topster, query_suggestion, groups_processed,
seq_id, sort_order, field_values, geopoint_indices,
group_limit, group_by_fields, token_bits, query_tokens,
- use_single_token_score, prioritize_exact_match);
+ prioritize_exact_match, its);
result_id_vec.push_back(seq_id);
});
@@ -1567,12 +1559,14 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
uint32_t token_bits = 255;
+ std::vector<posting_list_t::iterator_t> plists;
+
for(size_t i = 0; i < filter_ids_length; i++) {
const uint32_t seq_id = filter_ids[i];
- score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},
- groups_processed, {}, seq_id, sort_order, field_values, geopoint_indices,
+ score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0, topster, {},
+ groups_processed, seq_id, sort_order, field_values, geopoint_indices,
group_limit, group_by_fields, token_bits, {},
- true, prioritize_exact_match);
+ prioritize_exact_match, plists);
}
collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);
@@ -1879,7 +1873,9 @@ void Index::search_field(const uint8_t & field_id,
const size_t combination_limit) const {
size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
- if(search_schema.at(field).locale != "" && search_schema.at(field).locale != "en") {
+ auto& the_field = search_schema.at(field);
+
+ if(the_field.locale != "" && the_field.locale != "en") {
// disable fuzzy trie traversal for non-english locales
max_cost = 0;
}
@@ -1990,7 +1986,8 @@ void Index::search_field(const uint8_t & field_id,
if(!token_candidates_vec.empty()) {
// If atleast one token is found, go ahead and search for candidates
- search_candidates(field_id, filter_ids, filter_ids_length, exclude_token_ids, exclude_token_ids_size,
+ search_candidates(field_id, the_field.is_array(), filter_ids, filter_ids_length,
+ exclude_token_ids, exclude_token_ids_size,
curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,
groups_processed, all_result_ids, all_result_ids_len, field_num_results,
typo_tokens_threshold, group_limit, group_by_fields, query_tokens,
@@ -2064,18 +2061,17 @@ void Index::log_leaves(const int cost, const std::string &token, const std::vect
}
void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index,
- const uint8_t & field_id, const uint32_t total_cost, Topster* topster,
+ const uint8_t & field_id, bool field_is_array, const uint32_t total_cost, Topster* topster,
const std::vector<art_leaf *> &query_suggestion,
spp::sparse_hash_set<uint64_t>& groups_processed,
- const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions,
const uint32_t seq_id, const int sort_order[3],
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,
const std::vector<size_t>& geopoint_indices,
const size_t group_limit, const std::vector<std::string>& group_by_fields,
uint32_t token_bits,
const std::vector<token_t>& query_tokens,
- bool use_single_token_score,
- bool prioritize_exact_match) const {
+ bool prioritize_exact_match,
+ std::vector<posting_list_t::iterator_t>& posting_lists) const {
spp::sparse_hash_map<uint32_t, int64_t>* TEXT_MATCH_SENTINEL = &text_match_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t>* SEQ_ID_SENTINEL = &seq_id_sentinel_value;
@@ -2117,13 +2113,19 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
uint64_t match_score = 0;
- if (use_single_token_score || array_token_positions.empty()) {
- Match single_token_match = Match(1, 0);
- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);
- match_score = single_token_match_score;
+ if (posting_lists.size() == 1) {
+ const uint8_t is_exact_match = uint8_t(
+ total_cost == 0 && prioritize_exact_match &&
+ posting_list_t::is_single_token_exact_match(posting_lists[0], field_is_array)
+ );
+ Match single_token_match = Match(1, 0, is_exact_match);
+ match_score = single_token_match.get_match_score(total_cost);
} else {
uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;
+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
+ posting_list_t::get_offsets(posting_lists, array_token_positions);
+
for (const auto& kv: array_token_positions) {
const std::vector<token_positions_t>& token_positions = kv.second;
if (token_positions.empty()) {
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index f4501f67..944ac8d9 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -791,6 +791,49 @@ bool posting_list_t::get_offsets(std::vector<iterator_t>& its,
return true;
}
+bool posting_list_t::is_single_token_exact_match(posting_list_t::iterator_t& it, bool field_is_array) {
+ block_t* curr_block = it.block();
+ uint32_t curr_index = it.index();
+
+ if(curr_block == nullptr || curr_index == UINT32_MAX) {
+ return false;
+ }
+
+ uint32_t* offsets = it.offsets;
+ uint32_t start_offset = it.offset_index[curr_index];
+
+ if(!field_is_array && offsets[start_offset] != 1) {
+ // allows us to skip other computes fast
+ return false;
+ }
+
+ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?
+ curr_block->offsets.getLength() :
+ it.offset_index[curr_index + 1];
+
+ if(field_is_array) {
+ int prev_pos = -1;
+
+ while(start_offset < end_offset) {
+ int pos = offsets[start_offset];
+ start_offset++;
+
+ if(pos == prev_pos && pos == 1 && start_offset+1 < end_offset && offsets[start_offset+1] == 0) {
+ return true;
+ }
+
+ prev_pos = pos;
+ }
+
+ return false;
+ } else if((end_offset - start_offset) == 2 && offsets[end_offset-1] == 0) {
+ // already checked for `offsets[start_offset] == 1` first
+ return true;
+ }
+
+ return false;
+}
+
bool posting_list_t::at_end(const std::vector<posting_list_t::iterator_t>& its) {
// if any one iterator is at end, we can stop
for(const auto& it : its) {