Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Optimize `filter_result_iterator_t::compute_result`, skip tokenization.
Harpreet Sangar93915a77f4cd
diff --git a/src/filter_result_iterator.cpp b/src/filter_result_iterator.cppindex 90e79fb7..ce2721e2 100644--- a/src/filter_result_iterator.cpp+++ b/src/filter_result_iterator.cpp@@ -1599,9 +1599,18 @@ void filter_result_iterator_t::compute_result() {return;}+ // Resetting posting_list_iterators.+ for (uint32_t i = 0; i < posting_lists.size(); i++) {+ auto const& plists = posting_lists[i];++ posting_list_iterators[i].clear();+ for (auto const& plist: plists) {+ posting_list_iterators[i].push_back(plist->new_iterator());+ }+ }+auto const& a_filter = filter_node->filter_exp;auto const& f = index->search_schema.at(a_filter.field_name);- art_tree* t = index->search_index.at(a_filter.field_name);uint32_t* or_ids = nullptr;size_t or_ids_size = 0;@@ -1609,37 +1618,12 @@ void filter_result_iterator_t::compute_result() {// aggregates IDs across array of filter values and reduces excessive ORingstd::vector<uint32_t> f_id_buff;- for (const std::string& filter_value : a_filter.values) {- std::vector<void*> posting_lists;-- // there could be multiple tokens in a filter value, which we have to treat as ANDs- // e.g. country: South Africa- Tokenizer tokenizer(filter_value, true, false, f.locale, index->symbols_to_index, index->token_separators);-- std::string str_token;- size_t token_index = 0;- std::vector<std::string> str_tokens;-- while (tokenizer.next(str_token, token_index)) {- str_tokens.push_back(str_token);-- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),- str_token.length()+1);- if (leaf == nullptr) {- continue;- }-- posting_lists.push_back(leaf->values);- }-- if (posting_lists.size() != str_tokens.size()) {- continue;- }-- if(a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {+ for (uint32_t i = 0; i < posting_lists.size(); i++) {+ auto& p_list = posting_lists[i];+ if (a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {// needs intersection + exact matching (unlike CONTAINS)std::vector<uint32_t> result_id_vec;- posting_t::intersect(posting_lists, result_id_vec);+ posting_list_t::intersect(p_list, result_id_vec);if (result_id_vec.empty()) {continue;@@ -1650,8 +1634,9 @@ void filter_result_iterator_t::compute_result() {size_t exact_str_ids_size = 0;std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);- posting_t::get_exact_matches(posting_lists, f.is_array(), result_id_vec.data(), result_id_vec.size(),- exact_str_ids, exact_str_ids_size);+ posting_list_t::get_exact_matches(posting_list_iterators[i], f.is_array(),+ result_id_vec.data(), result_id_vec.size(),+ exact_str_ids, exact_str_ids_size);if (exact_str_ids_size == 0) {continue;@@ -1663,7 +1648,7 @@ void filter_result_iterator_t::compute_result() {} else {// CONTAINSsize_t before_size = f_id_buff.size();- posting_t::intersect(posting_lists, f_id_buff);+ posting_list_t::intersect(p_list, f_id_buff);if (f_id_buff.size() == before_size) {continue;}