Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'postings-refactor' into postings-refactor-integration
Kishore Nallan88f0d8b6a403
diff --cc src/index.cppindex 9354d1f0,c1c65092..46376937--- a/src/index.cpp+++ b/src/index.cpp@@@ -904,106 -933,91 +920,111 @@@ void Index::search_candidates(const uincontinue;}- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();+ // Prepare excluded document IDs that we can later remove from the result set+ uint32_t* excluded_result_ids = nullptr;+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);- // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)- for(size_t i=1; i < query_suggestion.size(); i++) {- uint32_t* out = nullptr;- uint32_t* ids = query_suggestion[i]->values->ids.uncompress();- result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);- delete[] ids;- delete[] result_ids;- result_ids = out;- }+ std::vector<void*> posting_lists;- if(result_size == 0) {- delete[] result_ids;- continue;+ for(auto& query_leaf : query_suggestion) {+ posting_lists.push_back(query_leaf->values);}- // Exclude document IDs associated with excluded tokens from the result set- if(exclude_token_ids_size != 0) {- uint32_t *excluded_result_ids = nullptr;- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,- &excluded_result_ids);- delete[] result_ids;- result_ids = excluded_result_ids;- }+ std::vector<posting_list_t::iterator_t> its;+ posting_list_t::result_iter_state_t iter_state;+ std::vector<uint32_t> result_id_vec;- if(!curated_ids.empty()) {- uint32_t *excluded_result_ids = nullptr;- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],- curated_ids.size(), &excluded_result_ids);+ size_t excluded_result_ids_index = 0;+ size_t filter_ids_index = 0;- delete [] result_ids;- result_ids = excluded_result_ids;- }+ posting_t::block_intersector_t intersector(posting_lists, 1000, iter_state);+ bool has_more = true;- //LOG(INFO) << "n: " << n;- /*std::stringstream log_query;- for(size_t i=0; i < query_suggestion.size(); i++) {- log_query << query_suggestion[i]->key << " ";- }*/+ while(has_more) {+ has_more = intersector.intersect();+ posting_list_t::result_iter_state_t updated_iter_state;+ size_t id_block_index = 0;- if(filter_ids != nullptr) {- // intersect once again with filter ids- uint32_t* filtered_result_ids = nullptr;- size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,- result_size, &filtered_result_ids);+ for(size_t i = 0; i < iter_state.ids.size(); i++) {+ uint32_t id = iter_state.ids[i];- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,- filtered_results_size, &new_all_result_ids);- delete [] *all_result_ids;- *all_result_ids = new_all_result_ids;+ // decide if this result id should be excluded+ if(excluded_result_ids_size != 0) {+ while(excluded_result_ids_index < excluded_result_ids_size &&+ excluded_result_ids[excluded_result_ids_index] < id) {+ excluded_result_ids_index++;+ }- // go through each matching document id and calculate match score- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion,- groups_processed, filtered_result_ids, filtered_results_size,- group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);+ if(excluded_result_ids_index < excluded_result_ids_size &&+ id == excluded_result_ids[excluded_result_ids_index]) {+ excluded_result_ids_index++;+ continue;+ }+ }- field_num_results += filtered_results_size;+ bool id_found_in_filter = true;- delete[] filtered_result_ids;- delete[] result_ids;- } else {- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,- result_size, &new_all_result_ids);- delete [] *all_result_ids;- *all_result_ids = new_all_result_ids;+ // decide if this result be matched with filter results+ if(filter_ids_length != 0) {+ id_found_in_filter = false;- /*if(result_size != 0) {- LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size- << ", popcount: " << (__builtin_popcount(token_bits) - 1);- }*/++ /*if(result_size != 0) {++ LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size++ << ", popcount: " << (__builtin_popcount(token_bits) - 1);++ }*/+- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion,- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits,- query_tokens, prioritize_exact_match);+ // e.g. [1, 3] vs [2, 3]++ while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {+ filter_ids_index++;+ }++ if(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] == id) {+ filter_ids_index++;+ id_found_in_filter = true;+ }+ }- field_num_results += result_size;+ if(id_found_in_filter) {+ result_id_vec.push_back(id);- delete[] result_ids;+ updated_iter_state.num_lists = iter_state.num_lists;+ updated_iter_state.ids.push_back(id);++ for(size_t k = 0; k < iter_state.num_lists; k++) {+ updated_iter_state.blocks.push_back(iter_state.blocks[id_block_index]);+ updated_iter_state.indices.push_back(iter_state.indices[id_block_index++]);+ }+ }+ }++ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;+ posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);++ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,+ query_suggestion, groups_processed, array_token_positions_vec,+ &updated_iter_state.ids[0], updated_iter_state.ids.size(),+ group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);}+ if(result_id_vec.empty()) {+ continue;+ }++ uint32_t* result_ids = &result_id_vec[0];+ result_size = result_id_vec.size();++ field_num_results += result_id_vec.size();++ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,+ result_size, &new_all_result_ids);+ delete [] *all_result_ids;+ *all_result_ids = new_all_result_ids;++ delete [] excluded_result_ids;+searched_queries.push_back(actual_query_suggestion);//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;@@@ -2050,12 -2146,14 +2093,14 @@@ void Index::score_results(const std::veuint64_t match_score = 0;- if (use_single_token_score) {+ if (use_single_token_score || array_token_positions_vec.empty()) {match_score = single_token_match_score;} else {- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;- populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);+ const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions =+ array_token_positions_vec[i];+ uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;+for (const auto& kv: array_token_positions) {const std::vector<token_positions_t>& token_positions = kv.second;if (token_positions.empty()) {