Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'postings-refactor' into postings-refactor-integration

Kishore Nallan88f0d8b6a403
diff --cc src/index.cpp
index 9354d1f0,c1c65092..46376937
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -904,106 -933,91 +920,111 @@@ void Index::search_candidates(const uin
continue;
}
- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
+ // Prepare excluded document IDs that we can later remove from the result set
+ uint32_t* excluded_result_ids = nullptr;
+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);
- // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
- for(size_t i=1; i < query_suggestion.size(); i++) {
- uint32_t* out = nullptr;
- uint32_t* ids = query_suggestion[i]->values->ids.uncompress();
- result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);
- delete[] ids;
- delete[] result_ids;
- result_ids = out;
- }
+ std::vector<void*> posting_lists;
- if(result_size == 0) {
- delete[] result_ids;
- continue;
+ for(auto& query_leaf : query_suggestion) {
+ posting_lists.push_back(query_leaf->values);
}
- // Exclude document IDs associated with excluded tokens from the result set
- if(exclude_token_ids_size != 0) {
- uint32_t *excluded_result_ids = nullptr;
- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,
- &excluded_result_ids);
- delete[] result_ids;
- result_ids = excluded_result_ids;
- }
+ std::vector<posting_list_t::iterator_t> its;
+ posting_list_t::result_iter_state_t iter_state;
+ std::vector<uint32_t> result_id_vec;
- if(!curated_ids.empty()) {
- uint32_t *excluded_result_ids = nullptr;
- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],
- curated_ids.size(), &excluded_result_ids);
+ size_t excluded_result_ids_index = 0;
+ size_t filter_ids_index = 0;
- delete [] result_ids;
- result_ids = excluded_result_ids;
- }
+ posting_t::block_intersector_t intersector(posting_lists, 1000, iter_state);
+ bool has_more = true;
- //LOG(INFO) << "n: " << n;
- /*std::stringstream log_query;
- for(size_t i=0; i < query_suggestion.size(); i++) {
- log_query << query_suggestion[i]->key << " ";
- }*/
+ while(has_more) {
+ has_more = intersector.intersect();
+ posting_list_t::result_iter_state_t updated_iter_state;
+ size_t id_block_index = 0;
- if(filter_ids != nullptr) {
- // intersect once again with filter ids
- uint32_t* filtered_result_ids = nullptr;
- size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
- result_size, &filtered_result_ids);
+ for(size_t i = 0; i < iter_state.ids.size(); i++) {
+ uint32_t id = iter_state.ids[i];
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
- filtered_results_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
+ // decide if this result id should be excluded
+ if(excluded_result_ids_size != 0) {
+ while(excluded_result_ids_index < excluded_result_ids_size &&
+ excluded_result_ids[excluded_result_ids_index] < id) {
+ excluded_result_ids_index++;
+ }
- // go through each matching document id and calculate match score
- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
- query_suggestion,
- groups_processed, filtered_result_ids, filtered_results_size,
- group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);
+ if(excluded_result_ids_index < excluded_result_ids_size &&
+ id == excluded_result_ids[excluded_result_ids_index]) {
+ excluded_result_ids_index++;
+ continue;
+ }
+ }
- field_num_results += filtered_results_size;
+ bool id_found_in_filter = true;
- delete[] filtered_result_ids;
- delete[] result_ids;
- } else {
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
- result_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
+ // decide if this result be matched with filter results
+ if(filter_ids_length != 0) {
+ id_found_in_filter = false;
- /*if(result_size != 0) {
- LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size
- << ", popcount: " << (__builtin_popcount(token_bits) - 1);
- }*/
++ /*if(result_size != 0) {
++ LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size
++ << ", popcount: " << (__builtin_popcount(token_bits) - 1);
++ }*/
+
- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
- query_suggestion,
- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits,
- query_tokens, prioritize_exact_match);
+ // e.g. [1, 3] vs [2, 3]
+
+ while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {
+ filter_ids_index++;
+ }
+
+ if(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] == id) {
+ filter_ids_index++;
+ id_found_in_filter = true;
+ }
+ }
- field_num_results += result_size;
+ if(id_found_in_filter) {
+ result_id_vec.push_back(id);
- delete[] result_ids;
+ updated_iter_state.num_lists = iter_state.num_lists;
+ updated_iter_state.ids.push_back(id);
+
+ for(size_t k = 0; k < iter_state.num_lists; k++) {
+ updated_iter_state.blocks.push_back(iter_state.blocks[id_block_index]);
+ updated_iter_state.indices.push_back(iter_state.indices[id_block_index++]);
+ }
+ }
+ }
+
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;
+ posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);
+
+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
+ query_suggestion, groups_processed, array_token_positions_vec,
+ &updated_iter_state.ids[0], updated_iter_state.ids.size(),
+ group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);
}
+ if(result_id_vec.empty()) {
+ continue;
+ }
+
+ uint32_t* result_ids = &result_id_vec[0];
+ result_size = result_id_vec.size();
+
+ field_num_results += result_id_vec.size();
+
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
+ result_size, &new_all_result_ids);
+ delete [] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+
+ delete [] excluded_result_ids;
+
searched_queries.push_back(actual_query_suggestion);
//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
@@@ -2050,12 -2146,14 +2093,14 @@@ void Index::score_results(const std::ve
uint64_t match_score = 0;
- if (use_single_token_score) {
+ if (use_single_token_score || array_token_positions_vec.empty()) {
match_score = single_token_match_score;
} else {
- std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
- populate_token_positions(query_suggestion, leaf_to_indices, i, array_token_positions);
+ const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions =
+ array_token_positions_vec[i];
+ uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;
+
for (const auto& kv: array_token_positions) {
const std::vector<token_positions_t>& token_positions = kv.second;
if (token_positions.empty()) {