Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Skip offset compute for non-exact single token.
Kishore Nallan22670b13420d
diff --git a/include/index.h b/include/index.hindex c08479a4..0bf56522 100644--- a/include/index.h+++ b/include/index.h@@ -325,7 +325,9 @@ public:const uint32_t* result_ids, size_t result_ids_size,const size_t group_limit,const std::vector<std::string> &group_by_fields, uint32_t token_bits,- const std::vector<token_t> &query_tokens, bool prioritize_exact_match) const;+ const std::vector<token_t> &query_tokens,+ bool use_single_token_score,+ bool prioritize_exact_match) const;static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);diff --git a/include/posting_list.h b/include/posting_list.hindex 657d3a39..1ca4ba9d 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -55,8 +55,8 @@ public:explicit iterator_t(block_t* root);iterator_t(iterator_t&& rhs) noexcept;~iterator_t();- [[nodiscard]] bool valid() const;- void next();+ [[nodiscard]] inline bool valid() const;+ void inline next();void skip_to(uint32_t id);[[nodiscard]] inline uint32_t id();[[nodiscard]] inline uint32_t index() const;diff --git a/src/index.cpp b/src/index.cppindex ed9a4327..e426c62f 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -997,13 +997,22 @@ void Index::search_candidates(const uint8_t & field_id,}}+ // We will have to be judicious about computing full match score: only when token does not match exact query+ bool use_single_token_score = (query_suggestion.size() == 1) &&+ (query_suggestion.size() == query_tokens.size()) &&+ ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));+std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;- posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);++ if(!use_single_token_score) {+ posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);+ }score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,query_suggestion, groups_processed, array_token_positions_vec,&updated_iter_state.ids[0], updated_iter_state.ids.size(),- group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);+ group_limit, group_by_fields, token_bits, query_tokens,+ use_single_token_score, prioritize_exact_match);}if(result_id_vec.empty()) {@@ -1565,7 +1574,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,uint32_t token_bits = 255;score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},groups_processed, {}, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {},- prioritize_exact_match);+ true, prioritize_exact_match);collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);all_result_ids_len = filter_ids_length;@@ -2059,6 +2068,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const size_t group_limit, const std::vector<std::string>& group_by_fields,uint32_t token_bits,const std::vector<token_t>& query_tokens,+ bool use_single_token_score,bool prioritize_exact_match) const {int sort_order[3]; // 1 or -1 based on DESC or ASC respectively@@ -2123,11 +2133,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16//auto begin = std::chrono::high_resolution_clock::now();//const std::string first_token((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1);- // We will have to be judicious about computing full match score: only when token does not match exact query- bool use_single_token_score = (query_suggestion.size() == 1) &&- (query_suggestion.size() == query_tokens.size()) &&- ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));-for (size_t i = 0; i < result_ids_size; i++) {const uint32_t seq_id = result_ids[i];