Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Copy-free intersect + score.
Kishore Nallan0e2adb4242f1
diff --git a/include/collection.h b/include/collection.hindex 4d499e84..4d3d5c74 100644--- a/include/collection.h+++ b/include/collection.h@@ -371,8 +371,6 @@ private:return std::tie(a_count, a_value_size) > std::tie(b_count, b_value_size);}- void free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const;-Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters) const;static Option<bool> parse_geopoint_filter_value(std::string& raw_value,diff --git a/include/index.h b/include/index.hindex 0bf56522..fc9367e5 100644--- a/include/index.h+++ b/include/index.h@@ -191,6 +191,11 @@ private:StringUtils string_utils;+ // used as sentinels++ static spp::sparse_hash_map<uint32_t, int64_t> text_match_sentinel_value;+ static spp::sparse_hash_map<uint32_t, int64_t> seq_id_sentinel_value;+// Internal utility functionsstatic inline uint32_t next_suggestion(const std::vector<token_candidates> &token_candidates_vec,@@ -321,8 +326,10 @@ public:void score_results(const std::vector<sort_by> &sort_fields, const uint16_t &query_index, const uint8_t &field_id,const uint32_t total_cost, Topster *topster, const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t> &groups_processed,- const std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec,- const uint32_t* result_ids, size_t result_ids_size,+ const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions,+ const uint32_t seq_id, const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,const size_t group_limit,const std::vector<std::string> &group_by_fields, uint32_t token_bits,const std::vector<token_t> &query_tokens,diff --git a/include/posting.h b/include/posting.hindex 767fab1f..cd9d48aa 100644--- a/include/posting.h+++ b/include/posting.h@@ -48,25 +48,21 @@ private:public:struct block_intersector_t {- size_t batch_size;std::vector<posting_list_t::iterator_t> its;std::vector<posting_list_t*> plists;std::vector<uint32_t> expanded_plist_indices;-posting_list_t::result_iter_state_t& iter_state;block_intersector_t(const std::vector<void*>& raw_posting_lists,- size_t batch_size,posting_list_t::result_iter_state_t& iter_state):- batch_size(batch_size), iter_state(iter_state) {+ iter_state(iter_state) {+to_expanded_plists(raw_posting_lists, plists, expanded_plist_indices);its.reserve(plists.size());for(const auto& posting_list: plists) {its.push_back(posting_list->new_iterator());}-- iter_state.num_lists = plists.size();}~block_intersector_t() {@@ -75,9 +71,8 @@ public:}}- bool intersect() {- return posting_list_t::block_intersect(plists, batch_size, its, iter_state);;- }+ template<class T>+ bool intersect(T func);};static void upsert(void*& obj, uint32_t id, const std::vector<uint32_t>& offsets);@@ -101,6 +96,12 @@ public:static void get_array_token_positions(uint32_t id,const std::vector<void*>& posting_lists,- std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions);-};\ No newline at end of file+};++template<class T>+bool posting_t::block_intersector_t::intersect(T func) {+ posting_list_t::block_intersect<T>(plists, its, iter_state, func);+ return true;+}diff --git a/include/posting_list.h b/include/posting_list.hindex 1ca4ba9d..39c20354 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -47,27 +47,41 @@ public:block_t* curr_block;uint32_t curr_index;- // uncompressed data structures for performance- block_t* uncompressed_block;- uint32_t* ids;public:+ // uncompressed data structures for performance+ uint32_t* ids = nullptr;+ uint32_t* offset_index = nullptr;+ uint32_t* offsets = nullptr;+explicit iterator_t(block_t* root);iterator_t(iterator_t&& rhs) noexcept;~iterator_t();- [[nodiscard]] inline bool valid() const;- void inline next();+ [[nodiscard]] bool valid() const;+ void next();void skip_to(uint32_t id);- [[nodiscard]] inline uint32_t id();+ [[nodiscard]] uint32_t id() const;[[nodiscard]] inline uint32_t index() const;[[nodiscard]] inline block_t* block() const;};struct result_iter_state_t {- size_t num_lists;- std::vector<block_t*> blocks;- std::vector<uint32_t> indices;- std::vector<uint32_t> ids;+ uint32_t* excluded_result_ids = nullptr;+ size_t excluded_result_ids_size = 0;+ uint32_t* filter_ids = nullptr;+ size_t filter_ids_length = 0;++ size_t excluded_result_ids_index = 0;+ size_t filter_ids_index = 0;++ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;++ result_iter_state_t() = default;++ result_iter_state_t(uint32_t* excluded_result_ids, size_t excluded_result_ids_size, uint32_t* filter_ids,+ size_t filter_ids_length) : excluded_result_ids(excluded_result_ids),+ excluded_result_ids_size(excluded_result_ids_size),+ filter_ids(filter_ids), filter_ids_length(filter_ids_length) {}};private:@@ -134,15 +148,83 @@ public:static void intersect(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids);+ template<class T>static bool block_intersect(const std::vector<posting_list_t*>& posting_lists,- size_t batch_size,std::vector<posting_list_t::iterator_t>& its,- result_iter_state_t& iter_state+ result_iter_state_t& istate,+ T func);+ static bool take_id(result_iter_state_t& istate, uint32_t id);+static bool get_offsets(- result_iter_state_t& iter_state,- std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions+ std::vector<iterator_t>& its,+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos);};++template<class T>+bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting_lists,+ std::vector<posting_list_t::iterator_t>& its,+ result_iter_state_t& istate,+ T func) {++ if(posting_lists.empty()) {+ return false;+ }++ if(its.empty()) {+ its.reserve(posting_lists.size());++ for(const auto& posting_list: posting_lists) {+ its.push_back(posting_list->new_iterator());+ }++ } else {+ // already in the middle of iteration: prepare for next batch++ }++ size_t num_lists = its.size();++ switch (num_lists) {+ case 1:+ while(its[0].valid()) {+ if(posting_list_t::take_id(istate, its[0].id())) {+ func(its[0].id(), its);+ }++ its[0].next();+ }+ break;+ case 2:+ while(!at_end2(its)) {+ if(equals2(its)) {+ if(posting_list_t::take_id(istate, its[0].id())) {+ func(its[0].id(), its);+ }++ advance_all2(its);+ } else {+ advance_non_largest2(its);+ }+ }+ break;+ default:+ while(!at_end(its)) {+ if(equals(its)) {+ //LOG(INFO) << its[0].id();+ if(posting_list_t::take_id(istate, its[0].id())) {+ func(its[0].id(), its);+ }++ advance_all(its);+ } else {+ advance_non_largest(its);+ }+ }+ }++ return false;+}diff --git a/src/collection.cpp b/src/collection.cppindex 9f206af2..683ad911 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1150,9 +1150,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::bool highlighted_fully = (fields_highlighted_fully.find(field_name) != fields_highlighted_fully.end());highlight_t highlight;+ //LOG(INFO) << "Highlighting: " << document;+ /*if(document["title"] == "Quantum Quest: A Cassini Space Odyssey") {+ LOG(INFO) << "here!";+ }*/highlight_result(search_field, searched_queries, q_tokens, field_order_kv, document,string_utils, snippet_threshold, highlight_affix_num_tokens,highlighted_fully, highlight_start_tag, highlight_end_tag, highlight);+ //LOG(INFO) << "End";if(!highlight.snippets.empty()) {highlights.push_back(highlight);@@ -1547,7 +1552,7 @@ void Collection::highlight_result(const field &search_field,Index* index = indices[field_order_kv->key % num_memory_shards];art_leaf* actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);- if(actual_leaf != nullptr) {+ if(actual_leaf != nullptr && posting_t::contains(actual_leaf->values, field_order_kv->key)) {query_suggestion.push_back(actual_leaf);query_suggestion_tokens.insert(token);//LOG(INFO) << "field: " << search_field.name << ", key: " << token;@@ -1557,7 +1562,9 @@ void Collection::highlight_result(const field &search_field,qindex++;} while(field_order_kv->query_indices != nullptr && qindex < field_order_kv->query_indices[0]);- for(const std::string& q_token: q_tokens) {+ for(size_t i = 0; i < q_tokens.size(); i++) {+ const std::string& q_token = q_tokens[i];+if(query_suggestion_tokens.count(q_token) != 0) {continue;}@@ -1566,32 +1573,29 @@ void Collection::highlight_result(const field &search_field,art_leaf *actual_leaf = index->get_token_leaf(search_field.name,reinterpret_cast<const unsigned char *>(q_token.c_str()),q_token.size() + 1);- if(actual_leaf != nullptr) {++ if(actual_leaf != nullptr && posting_t::contains(actual_leaf->values, field_order_kv->key)) {query_suggestion.push_back(actual_leaf);query_suggestion_tokens.insert(q_token);+ } else if(i == q_tokens.size()-1) {+ // we will copy the last token for highlighting prefix matches+ query_suggestion_tokens.insert(q_token);}}- if(query_suggestion.empty()) {+ if(query_suggestion_tokens.empty()) {// none of the tokens from the query were found on this fieldreturn ;}- //LOG(INFO) << "Document ID: " << document["id"];-std::vector<void*> posting_lists;for(art_leaf* leaf: query_suggestion) {posting_lists.push_back(leaf->values);}- std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;- posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions_vec);+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;+ posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);- if(array_token_positions_vec.empty()) {- return;- }-- std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions = array_token_positions_vec[0];std::vector<match_index_t> match_indices;for(const auto& kv: array_token_positions) {@@ -1639,6 +1643,14 @@ void Collection::highlight_result(const field &search_field,}}+ if(!document.contains(search_field.name)) {+ // could be an optional field+ continue;+ }++ /*LOG(INFO) << "field: " << document[search_field.name] << ", id: " << field_order_kv->key+ << ", index: " << match_index.index;*/+std::string text = (search_field.type == field_types::STRING) ? document[search_field.name] :document[search_field.name][match_index.index];Tokenizer tokenizer(text, true, false, search_field.locale);@@ -1793,12 +1805,6 @@ void Collection::highlight_result(const field &search_field,highlight.match_score = match_indices[0].match_score;}-void Collection::free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const {- for(uint32_t* leaf_indices: leaf_to_indices) {- delete [] leaf_indices;- }-}-Option<nlohmann::json> Collection::get(const std::string & id) const {std::string seq_id_str;StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);diff --git a/src/index.cpp b/src/index.cppindex e426c62f..3ff5aad3 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -20,6 +20,9 @@#include <posting.h>#include "logger.h"+spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;+spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;+Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema) {@@ -615,6 +618,8 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, clast_token = token;}+ //LOG(INFO) << "Str: " << str << ", last_token: " << last_token;+if(token_set.empty()) {continue;}@@ -896,6 +901,29 @@ void Index::search_candidates(const uint8_t & field_id,auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;+ std::vector<size_t> geopoint_indices;++ for (size_t i = 0; i < sort_fields.size(); i++) {+ sort_order[i] = 1;+ if (sort_fields[i].order == sort_field_const::asc) {+ sort_order[i] = -1;+ }++ if (sort_fields[i].name == sort_field_const::text_match) {+ field_values[i] = &text_match_sentinel_value;+ } else if (sort_fields[i].name == sort_field_const::seq_id) {+ field_values[i] = &seq_id_sentinel_value;+ } else if (sort_index.count(sort_fields[i].name) != 0) {+ field_values[i] = sort_index.at(sort_fields[i].name);++ if (sort_schema.at(sort_fields[i].name).is_geopoint()) {+ geopoint_indices.push_back(i);+ }+ }+ }+for(long long n=0; n<N && n<combination_limit; ++n) {// every element in `query_suggestion` contains a token and its associated hitsstd::vector<art_leaf*> query_suggestion(token_candidates_vec.size());@@ -917,12 +945,6 @@ void Index::search_candidates(const uint8_t & field_id,}LOG(INFO) << fullq.str();*/- // initialize results with the starting element (for further intersection)- size_t result_size = posting_t::num_ids(query_suggestion[0]->values);- if(result_size == 0) {- continue;- }-// Prepare excluded document IDs that we can later remove from the result setuint32_t* excluded_result_ids = nullptr;size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,@@ -934,95 +956,42 @@ void Index::search_candidates(const uint8_t & field_id,posting_lists.push_back(query_leaf->values);}- std::vector<posting_list_t::iterator_t> its;- posting_list_t::result_iter_state_t iter_state;- std::vector<uint32_t> result_id_vec;-- size_t excluded_result_ids_index = 0;- size_t filter_ids_index = 0;-- posting_t::block_intersector_t intersector(posting_lists, 1000, iter_state);- bool has_more = true;+ posting_list_t::result_iter_state_t iter_state(+ excluded_result_ids, excluded_result_ids_size, filter_ids, filter_ids_length+ );- while(has_more) {- has_more = intersector.intersect();- posting_list_t::result_iter_state_t updated_iter_state;- size_t id_block_index = 0;+ // We will have to be judicious about computing full match score: only when token does not match exact query+ bool use_single_token_score = (query_suggestion.size() == 1) &&+ (query_suggestion.size() == query_tokens.size()) &&+ ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));- for(size_t i = 0; i < iter_state.ids.size(); i++) {- uint32_t id = iter_state.ids[i];-- // decide if this result id should be excluded- if(excluded_result_ids_size != 0) {- while(excluded_result_ids_index < excluded_result_ids_size &&- excluded_result_ids[excluded_result_ids_index] < id) {- excluded_result_ids_index++;- }-- if(excluded_result_ids_index < excluded_result_ids_size &&- id == excluded_result_ids[excluded_result_ids_index]) {- excluded_result_ids_index++;- continue;- }- }-- bool id_found_in_filter = true;-- // decide if this result be matched with filter results- if(filter_ids_length != 0) {- id_found_in_filter = false;-- // e.g. [1, 3] vs [2, 3]-- while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {- filter_ids_index++;- }-- if(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] == id) {- filter_ids_index++;- id_found_in_filter = true;- }- }-- if(id_found_in_filter) {- result_id_vec.push_back(id);-- updated_iter_state.num_lists = iter_state.num_lists;- updated_iter_state.ids.push_back(id);-- for(size_t k = 0; k < iter_state.num_lists; k++) {- updated_iter_state.blocks.push_back(iter_state.blocks[id_block_index]);- updated_iter_state.indices.push_back(iter_state.indices[id_block_index++]);- }- }- }-- // We will have to be judicious about computing full match score: only when token does not match exact query- bool use_single_token_score = (query_suggestion.size() == 1) &&- (query_suggestion.size() == query_tokens.size()) &&- ((std::string((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1) != query_tokens[0].value));+ std::vector<uint32_t> result_id_vec;+ posting_t::block_intersector_t intersector(posting_lists, iter_state);- std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;+ intersector.intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {+ std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;if(!use_single_token_score) {- posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);+ posting_list_t::get_offsets(its, array_token_positions);}score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion, groups_processed, array_token_positions_vec,- &updated_iter_state.ids[0], updated_iter_state.ids.size(),+ query_suggestion, groups_processed, array_token_positions,+ seq_id, sort_order, field_values, geopoint_indices,group_limit, group_by_fields, token_bits, query_tokens,use_single_token_score, prioritize_exact_match);- }++ result_id_vec.push_back(seq_id);+ });if(result_id_vec.empty()) {continue;}uint32_t* result_ids = &result_id_vec[0];- result_size = result_id_vec.size();+ size_t result_size = result_id_vec.size();- field_num_results += result_id_vec.size();+ field_num_results += result_size;uint32_t* new_all_result_ids = nullptr;all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,@@ -1571,10 +1540,41 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,filter_ids = excluded_result_ids;}++ // FIXME: duplicated+ int sort_order[3]; // 1 or -1 based on DESC or ASC respectively+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values;+ std::vector<size_t> geopoint_indices;++ for (size_t i = 0; i < sort_fields_std.size(); i++) {+ sort_order[i] = 1;+ if (sort_fields_std[i].order == sort_field_const::asc) {+ sort_order[i] = -1;+ }++ if (sort_fields_std[i].name == sort_field_const::text_match) {+ field_values[i] = &text_match_sentinel_value;+ } else if (sort_fields_std[i].name == sort_field_const::seq_id) {+ field_values[i] = &seq_id_sentinel_value;+ } else if (sort_index.count(sort_fields_std[i].name) != 0) {+ field_values[i] = sort_index.at(sort_fields_std[i].name);++ if (sort_schema.at(sort_fields_std[i].name).is_geopoint()) {+ geopoint_indices.push_back(i);+ }+ }+ }+uint32_t token_bits = 255;- score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},- groups_processed, {}, filter_ids, filter_ids_length, group_limit, group_by_fields, token_bits, {},- true, prioritize_exact_match);++ for(size_t i = 0; i < filter_ids_length; i++) {+ const uint32_t seq_id = filter_ids[i];+ score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, 0, topster, {},+ groups_processed, {}, seq_id, sort_order, field_values, geopoint_indices,+ group_limit, group_by_fields, token_bits, {},+ true, prioritize_exact_match);+ }+collate_included_ids(field_query_tokens[0].q_include_tokens, field, field_id, included_ids_map, curated_topster, searched_queries);all_result_ids_len = filter_ids_length;@@ -2063,191 +2063,163 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const uint8_t & field_id, const uint32_t total_cost, Topster* topster,const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec,- const uint32_t* result_ids, size_t result_ids_size,+ const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions,+ const uint32_t seq_id, const int sort_order[3],+ std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,+ const std::vector<size_t>& geopoint_indices,const size_t group_limit, const std::vector<std::string>& group_by_fields,uint32_t token_bits,const std::vector<token_t>& query_tokens,bool use_single_token_score,bool prioritize_exact_match) const {- int sort_order[3]; // 1 or -1 based on DESC or ASC respectively- spp::sparse_hash_map<uint32_t, int64_t>* field_values[3];-+ spp::sparse_hash_map<uint32_t, int64_t>* TEXT_MATCH_SENTINEL = &text_match_sentinel_value;+ spp::sparse_hash_map<uint32_t, int64_t>* SEQ_ID_SENTINEL = &seq_id_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> geopoint_distances[3];- spp::sparse_hash_map<uint32_t, int64_t> text_match_sentinel_value, seq_id_sentinel_value;- spp::sparse_hash_map<uint32_t, int64_t> *TEXT_MATCH_SENTINEL = &text_match_sentinel_value;- spp::sparse_hash_map<uint32_t, int64_t> *SEQ_ID_SENTINEL = &seq_id_sentinel_value;-- for (size_t i = 0; i < sort_fields.size(); i++) {- sort_order[i] = 1;- if (sort_fields[i].order == sort_field_const::asc) {- sort_order[i] = -1;- }-- if (sort_fields[i].name == sort_field_const::text_match) {- field_values[i] = TEXT_MATCH_SENTINEL;- } else if (sort_fields[i].name == sort_field_const::seq_id) {- field_values[i] = SEQ_ID_SENTINEL;- } else if (sort_schema.at(sort_fields[i].name).is_geopoint()) {- // we have to populate distances that will be used for match scoring- spp::sparse_hash_map<uint32_t, int64_t> *geopoints = sort_index.at(sort_fields[i].name);+ for(auto& i: geopoint_indices) {+ spp::sparse_hash_map<uint32_t, int64_t>* geopoints = field_values[i];- S2LatLng reference_lat_lng;- GeoPoint::unpack_lat_lng(sort_fields[i].geopoint, reference_lat_lng);+ S2LatLng reference_lat_lng;+ GeoPoint::unpack_lat_lng(sort_fields[i].geopoint, reference_lat_lng);- for (size_t rindex = 0; rindex < result_ids_size; rindex++) {- const uint32_t seq_id = result_ids[rindex];- auto it = geopoints->find(seq_id);- int64_t dist = INT32_MAX;+ auto it = geopoints->find(seq_id);+ int64_t dist = INT32_MAX;- if(it != geopoints->end()) {- int64_t packed_latlng = it->second;- S2LatLng s2_lat_lng;- GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);- dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);- }+ if(it != geopoints->end()) {+ int64_t packed_latlng = it->second;+ S2LatLng s2_lat_lng;+ GeoPoint::unpack_lat_lng(packed_latlng, s2_lat_lng);+ dist = GeoPoint::distance(s2_lat_lng, reference_lat_lng);+ }- if(dist < sort_fields[i].exclude_radius) {- dist = 0;- }+ if(dist < sort_fields[i].exclude_radius) {+ dist = 0;+ }- if(sort_fields[i].geo_precision > 0) {- dist = dist + sort_fields[i].geo_precision - 1 -- (dist + sort_fields[i].geo_precision - 1) % sort_fields[i].geo_precision;- }+ if(sort_fields[i].geo_precision > 0) {+ dist = dist + sort_fields[i].geo_precision - 1 -+ (dist + sort_fields[i].geo_precision - 1) % sort_fields[i].geo_precision;+ }- geopoint_distances[i].emplace(seq_id, dist);- }+ geopoint_distances[i].emplace(seq_id, dist);- field_values[i] = &geopoint_distances[i];- } else {- field_values[i] = sort_index.at(sort_fields[i].name);- }+ // Swap (id -> latlong) index to (id -> distance) index+ field_values[i] = &geopoint_distances[i];}- Match single_token_match = Match(1, 0);- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);-//auto begin = std::chrono::high_resolution_clock::now();//const std::string first_token((const char*)query_suggestion[0]->key, query_suggestion[0]->key_len-1);- for (size_t i = 0; i < result_ids_size; i++) {- const uint32_t seq_id = result_ids[i];-- uint64_t match_score = 0;-- if (use_single_token_score || array_token_positions_vec.empty()) {- match_score = single_token_match_score;- } else {- const std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions =- array_token_positions_vec[i];-- uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;+ uint64_t match_score = 0;- for (const auto& kv: array_token_positions) {- const std::vector<token_positions_t>& token_positions = kv.second;- if (token_positions.empty()) {- continue;- }- const Match &match = Match(seq_id, token_positions, false, prioritize_exact_match);- uint64_t this_match_score = match.get_match_score(total_cost);-- total_tokens_found += ((this_match_score >> 24) & 0xFF);- total_num_typos += 255 - ((this_match_score >> 16) & 0xFF);- total_distance += 100 - ((this_match_score >> 8) & 0xFF);- total_verbatim += (this_match_score & 0xFF);-- /*std::ostringstream os;- os << name << ", total_cost: " << (255 - total_cost)- << ", words_present: " << match.words_present- << ", match_score: " << match_score- << ", match.distance: " << match.distance- << ", seq_id: " << seq_id << std::endl;- LOG(INFO) << os.str();*/- }-- match_score = (- (uint64_t(total_tokens_found) << 24) |- (uint64_t(255 - total_num_typos) << 16) |- (uint64_t(100 - total_distance) << 8) |- (uint64_t(total_verbatim) << 1)- );+ if (use_single_token_score || array_token_positions.empty()) {+ Match single_token_match = Match(1, 0);+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost);+ match_score = single_token_match_score;+ } else {+ uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;- /*LOG(INFO) << "Match score: " << match_score << ", for seq_id: " << seq_id- << " - total_tokens_found: " << total_tokens_found- << " - total_num_typos: " << total_num_typos- << " - total_distance: " << total_distance- << " - total_verbatim: " << total_verbatim- << " - total_cost: " << total_cost;*/- }-- const int64_t default_score = INT64_MIN; // to handle field that doesn't exist in document (e.g. optional)- int64_t scores[3] = {0};- size_t match_score_index = 0;-- // avoiding loop- if (sort_fields.size() > 0) {- if (field_values[0] == TEXT_MATCH_SENTINEL) {- scores[0] = int64_t(match_score);- match_score_index = 0;- } else if (field_values[0] == SEQ_ID_SENTINEL) {- scores[0] = seq_id;- } else {- auto it = field_values[0]->find(seq_id);- scores[0] = (it == field_values[0]->end()) ? default_score : it->second;- }- if (sort_order[0] == -1) {- scores[0] = -scores[0];+ for (const auto& kv: array_token_positions) {+ const std::vector<token_positions_t>& token_positions = kv.second;+ if (token_positions.empty()) {+ continue;}+ const Match &match = Match(seq_id, token_positions, false, prioritize_exact_match);+ uint64_t this_match_score = match.get_match_score(total_cost);++ total_tokens_found += ((this_match_score >> 24) & 0xFF);+ total_num_typos += 255 - ((this_match_score >> 16) & 0xFF);+ total_distance += 100 - ((this_match_score >> 8) & 0xFF);+ total_verbatim += (this_match_score & 0xFF);++ /*std::ostringstream os;+ os << name << ", total_cost: " << (255 - total_cost)+ << ", words_present: " << match.words_present+ << ", match_score: " << match_score+ << ", match.distance: " << match.distance+ << ", seq_id: " << seq_id << std::endl;+ LOG(INFO) << os.str();*/+ }++ match_score = (+ (uint64_t(total_tokens_found) << 24) |+ (uint64_t(255 - total_num_typos) << 16) |+ (uint64_t(100 - total_distance) << 8) |+ (uint64_t(total_verbatim) << 1)+ );++ /*LOG(INFO) << "Match score: " << match_score << ", for seq_id: " << seq_id+ << " - total_tokens_found: " << total_tokens_found+ << " - total_num_typos: " << total_num_typos+ << " - total_distance: " << total_distance+ << " - total_verbatim: " << total_verbatim+ << " - total_cost: " << total_cost;*/+ }++ const int64_t default_score = INT64_MIN; // to handle field that doesn't exist in document (e.g. optional)+ int64_t scores[3] = {0};+ size_t match_score_index = 0;++ // avoiding loop+ if (sort_fields.size() > 0) {+ if (field_values[0] == TEXT_MATCH_SENTINEL) {+ scores[0] = int64_t(match_score);+ match_score_index = 0;+ } else if (field_values[0] == SEQ_ID_SENTINEL) {+ scores[0] = seq_id;+ } else {+ auto it = field_values[0]->find(seq_id);+ scores[0] = (it == field_values[0]->end()) ? default_score : it->second;}--- if(sort_fields.size() > 1) {- if (field_values[1] == TEXT_MATCH_SENTINEL) {- scores[1] = int64_t(match_score);- match_score_index = 1;- } else if (field_values[1] == SEQ_ID_SENTINEL) {- scores[1] = seq_id;- } else {- auto it = field_values[1]->find(seq_id);- scores[1] = (it == field_values[1]->end()) ? default_score : it->second;- }-- if (sort_order[1] == -1) {- scores[1] = -scores[1];- }+ if (sort_order[0] == -1) {+ scores[0] = -scores[0];}+ }- if(sort_fields.size() > 2) {- if(field_values[2] == TEXT_MATCH_SENTINEL) {- scores[2] = int64_t(match_score);- match_score_index = 2;- } else if (field_values[2] == SEQ_ID_SENTINEL) {- scores[2] = seq_id;- } else {- auto it = field_values[2]->find(seq_id);- scores[2] = (it == field_values[2]->end()) ? default_score : it->second;- }+ if(sort_fields.size() > 1) {+ if (field_values[1] == TEXT_MATCH_SENTINEL) {+ scores[1] = int64_t(match_score);+ match_score_index = 1;+ } else if (field_values[1] == SEQ_ID_SENTINEL) {+ scores[1] = seq_id;+ } else {+ auto it = field_values[1]->find(seq_id);+ scores[1] = (it == field_values[1]->end()) ? default_score : it->second;+ }- if(sort_order[2] == -1) {- scores[2] = -scores[2];- }+ if (sort_order[1] == -1) {+ scores[1] = -scores[1];}+ }- uint64_t distinct_id = seq_id;+ if(sort_fields.size() > 2) {+ if(field_values[2] == TEXT_MATCH_SENTINEL) {+ scores[2] = int64_t(match_score);+ match_score_index = 2;+ } else if (field_values[2] == SEQ_ID_SENTINEL) {+ scores[2] = seq_id;+ } else {+ auto it = field_values[2]->find(seq_id);+ scores[2] = (it == field_values[2]->end()) ? default_score : it->second;+ }- if(group_limit != 0) {- distinct_id = get_distinct_id(group_by_fields, seq_id);- groups_processed.emplace(distinct_id);+ if(sort_order[2] == -1) {+ scores[2] = -scores[2];}+ }++ uint64_t distinct_id = seq_id;- //LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;- KV kv(field_id, query_index, token_bits, seq_id, distinct_id, match_score_index, scores);- topster->add(&kv);+ if(group_limit != 0) {+ distinct_id = get_distinct_id(group_by_fields, seq_id);+ groups_processed.emplace(distinct_id);}+ //LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;+ KV kv(field_id, query_index, token_bits, seq_id, distinct_id, match_score_index, scores);+ topster->add(&kv);+//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";}diff --git a/src/posting.cpp b/src/posting.cppindex 2a3b4745..cc5c0e03 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -418,39 +418,20 @@ void posting_t::destroy_list(void*& obj) {}void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>& raw_posting_lists,- std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec) {+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_positions) {std::vector<posting_list_t*> plists;std::vector<uint32_t> expanded_plist_indices;to_expanded_plists(raw_posting_lists, plists, expanded_plist_indices);- posting_list_t::result_iter_state_t iter_state;- iter_state.ids.push_back(id);- iter_state.num_lists = plists.size();-- std::vector<posting_list_t::block_t*>& block_vec = iter_state.blocks;- std::vector<uint32_t>& index_vec = iter_state.indices;+ std::vector<posting_list_t::iterator_t> its;for(posting_list_t* pl: plists) {- posting_list_t::block_t* block = pl->block_of(id);- block_vec.push_back(block);-- bool found_index = false;-- if(block != nullptr) {- uint32_t index = block->ids.indexOf(id);- if(index != block->ids.getLength()) {- index_vec.push_back(index);- found_index = true;- }- }-- if(!found_index) {- index_vec.push_back(UINT32_MAX);- }+ its.push_back(pl->new_iterator());+ its.back().skip_to(id);}- posting_list_t::get_offsets(iter_state, array_token_positions_vec);+ posting_list_t::get_offsets(its, array_token_positions);for(uint32_t expanded_plist_index: expanded_plist_indices) {delete plists[expanded_plist_index];diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 3cbd0189..f4501f67 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -661,96 +661,44 @@ void posting_list_t::intersect(const std::vector<posting_list_t*>& posting_lists}}-bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting_lists, const size_t batch_size,- std::vector<posting_list_t::iterator_t>& its,- result_iter_state_t& iter_state) {-- if(posting_lists.empty()) {- return false;- }-- if(its.empty()) {- its.reserve(posting_lists.size());-- for(const auto& posting_list: posting_lists) {- its.push_back(posting_list->new_iterator());+bool posting_list_t::take_id(result_iter_state_t& istate, uint32_t id) {+ // decide if this result id should be excluded+ if(istate.excluded_result_ids_size != 0) {+ while(istate.excluded_result_ids_index < istate.excluded_result_ids_size &&+ istate.excluded_result_ids[istate.excluded_result_ids_index] < id) {+ istate.excluded_result_ids_index++;}- iter_state.num_lists = posting_lists.size();- iter_state.blocks.reserve(100);- iter_state.indices.reserve(100);- } else {- // already in the middle of iteration: prepare for next batch- iter_state.ids.clear();- iter_state.indices.clear();- iter_state.blocks.clear();+ if(istate.excluded_result_ids_index < istate.excluded_result_ids_size &&+ id == istate.excluded_result_ids[istate.excluded_result_ids_index]) {+ istate.excluded_result_ids_index++;+ return false;+ }}- size_t num_lists = its.size();-- switch (num_lists) {- case 1:- while(its[0].valid()) {- iter_state.ids.push_back(its[0].id());- iter_state.blocks.push_back(its[0].block());- iter_state.indices.push_back(its[0].index());+ bool id_found_in_filter = true;- its[0].next();+ // decide if this result be matched with filter results+ if(istate.filter_ids_length != 0) {+ id_found_in_filter = false;- if(iter_state.ids.size() == batch_size) {- return its[0].valid();- }- }- break;- case 2:- while(!at_end2(its)) {- if(equals2(its)) {- // still need to ensure that the ID exists in inclusion list but NOT in exclusion list- iter_state.ids.push_back(its[0].id());+ // e.g. [1, 3] vs [2, 3]- iter_state.blocks.push_back(its[0].block());- iter_state.blocks.push_back(its[1].block());-- iter_state.indices.push_back(its[0].index());- iter_state.indices.push_back(its[1].index());-- advance_all2(its);- } else {- advance_non_largest2(its);- }-- if(iter_state.ids.size() == batch_size) {- return !at_end2(its);- }- }- break;- default:- while(!at_end(its)) {- if(equals(its)) {- //LOG(INFO) << its[0].id();- iter_state.ids.push_back(its[0].id());-- for(size_t i = 0; i < its.size(); i++) {- iter_state.blocks.push_back(its[i].block());- iter_state.indices.push_back(its[i].index());- }-- advance_all(its);- } else {- advance_non_largest(its);- }+ while(istate.filter_ids_index < istate.filter_ids_length && istate.filter_ids[istate.filter_ids_index] < id) {+ istate.filter_ids_index++;+ }- if(iter_state.ids.size() == batch_size) {- return !at_end(its);- }- }+ if(istate.filter_ids_index < istate.filter_ids_length && istate.filter_ids[istate.filter_ids_index] == id) {+ istate.filter_ids_index++;+ id_found_in_filter = true;+ }}- return false;+ return id_found_in_filter;}-bool posting_list_t::get_offsets(posting_list_t::result_iter_state_t& iter_state,- std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>>& array_token_positions_vec) {+bool posting_list_t::get_offsets(std::vector<iterator_t>& its,+ std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos) {// Plain string format:// offset1, offset2, ... , 0 (if token is the last offset for the document)@@ -763,78 +711,84 @@ bool posting_list_t::get_offsets(posting_list_t::result_iter_state_t& iter_statesize_t id_block_index = 0;- for(size_t i = 0; i < iter_state.ids.size(); i++) {- uint32_t id = iter_state.ids[i];- array_token_positions_vec.emplace_back();- std::unordered_map<size_t, std::vector<token_positions_t>>& array_tok_pos = array_token_positions_vec.back();+ for(size_t j = 0; j < its.size(); j++) {+ block_t* curr_block = its[j].block();+ uint32_t curr_index = its[j].index();- for(size_t j = 0; j < iter_state.num_lists; j++) {- block_t* curr_block = iter_state.blocks[id_block_index];- uint32_t curr_index = iter_state.indices[id_block_index++];+ if(curr_block == nullptr || curr_index == UINT32_MAX) {+ continue;+ }- if(curr_block == nullptr || curr_index == UINT32_MAX) {- continue;- }+ /*uint32_t* offsets = curr_block->offsets.uncompress();- uint32_t* offsets = curr_block->offsets.uncompress();+ uint32_t start_offset = curr_block->offset_index.at(curr_index);+ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?+ curr_block->offsets.getLength() :+ curr_block->offset_index.at(curr_index + 1);*/- uint32_t start_offset = curr_block->offset_index.at(curr_index);- uint32_t end_offset = (curr_index == curr_block->size() - 1) ?- curr_block->offsets.getLength() :- curr_block->offset_index.at(curr_index + 1);+ uint32_t* offsets = its[j].offsets;- std::vector<uint16_t> positions;- int prev_pos = -1;- bool is_last_token = false;+ uint32_t start_offset = its[j].offset_index[curr_index];+ uint32_t end_offset = (curr_index == curr_block->size() - 1) ?+ curr_block->offsets.getLength() :+ its[j].offset_index[curr_index + 1];- while(start_offset < end_offset) {- int pos = offsets[start_offset];- start_offset++;+ std::vector<uint16_t> positions;+ int prev_pos = -1;+ bool is_last_token = false;- if(pos == 0) {- // indicates that token is the last token on the doc- is_last_token = true;- start_offset++;- continue;- }+ /*LOG(INFO) << "id: " << its[j].id() << ", start_offset: " << start_offset << ", end_offset: " << end_offset;+ for(size_t x = 0; x < end_offset; x++) {+ LOG(INFO) << "x: " << x << ", pos: " << offsets[x];+ }*/- if(pos == prev_pos) { // indicates end of array index- if(!positions.empty()) {- size_t array_index = (size_t) offsets[start_offset];- is_last_token = false;-- if(start_offset+1 < end_offset) {- size_t next_offset = (size_t) offsets[start_offset + 1];- if(next_offset == 0) {- // indicates that token is the last token on the doc- is_last_token = true;- start_offset++;- }- }+ while(start_offset < end_offset) {+ int pos = offsets[start_offset];+ start_offset++;- array_tok_pos[array_index].push_back(token_positions_t{is_last_token, positions});- positions.clear();+ if(pos == 0) {+ // indicates that token is the last token on the doc+ is_last_token = true;+ start_offset++;+ continue;+ }++ if(pos == prev_pos) { // indicates end of array index+ if(!positions.empty()) {+ size_t array_index = (size_t) offsets[start_offset];+ is_last_token = false;++ if(start_offset+1 < end_offset) {+ size_t next_offset = (size_t) offsets[start_offset + 1];+ if(next_offset == 0) {+ // indicates that token is the last token on the doc+ is_last_token = true;+ start_offset++;+ }}- start_offset++; // skip current value which is the array index or flag for last index- prev_pos = -1;- continue;+ array_token_pos[array_index].push_back(token_positions_t{is_last_token, positions});+ positions.clear();}- prev_pos = pos;- positions.push_back((uint16_t)pos - 1);+ start_offset++; // skip current value which is the array index or flag for last index+ prev_pos = -1;+ continue;}- if(!positions.empty()) {- // for plain string fields- array_tok_pos[0].push_back(token_positions_t{is_last_token, positions});- }+ prev_pos = pos;+ positions.push_back((uint16_t)pos - 1);+ }- delete [] offsets;+ if(!positions.empty()) {+ // for plain string fields+ array_token_pos[0].push_back(token_positions_t{is_last_token, positions});}++ //delete [] offsets;}- return false;+ return true;}bool posting_list_t::at_end(const std::vector<posting_list_t::iterator_t>& its) {@@ -982,8 +936,13 @@ bool posting_list_t::contains_atleast_one(const uint32_t* target_ids, size_t tar/* iterator_t operations */posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root):- curr_block(root), curr_index(0), uncompressed_block(nullptr), ids(nullptr) {+ curr_block(root), curr_index(0) {+ if(curr_block != nullptr) {+ ids = curr_block->ids.uncompress();+ offset_index = curr_block->offset_index.uncompress();+ offsets = curr_block->offsets.uncompress();+ }}bool posting_list_t::iterator_t::valid() const {@@ -995,21 +954,22 @@ void posting_list_t::iterator_t::next() {if(curr_index == curr_block->size()) {curr_index = 0;curr_block = curr_block->next;- }-}--uint32_t posting_list_t::iterator_t::id() {- if(uncompressed_block != curr_block) {- uncompressed_block = curr_block;delete [] ids;- ids = nullptr;+ delete [] offset_index;+ delete [] offsets;++ ids = offset_index = offsets = nullptr;if(curr_block != nullptr) {ids = curr_block->ids.uncompress();+ offset_index = curr_block->offset_index.uncompress();+ offsets = curr_block->offsets.uncompress();}}+}+uint32_t posting_list_t::iterator_t::id() const {return ids[curr_index];}@@ -1025,6 +985,20 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {bool skipped_block = false;while(curr_block != nullptr && curr_block->ids.last() < id) {curr_block = curr_block->next;++ // FIXME: remove duplication+ delete [] ids;+ delete [] offset_index;+ delete [] offsets;++ ids = offset_index = offsets = nullptr;++ if(curr_block != nullptr) {+ ids = curr_block->ids.uncompress();+ offset_index = curr_block->offset_index.uncompress();+ offsets = curr_block->offsets.uncompress();+ }+skipped_block = true;}@@ -1045,10 +1019,12 @@ posting_list_t::iterator_t::~iterator_t() {posting_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {curr_block = rhs.curr_block;curr_index = rhs.curr_index;- uncompressed_block = rhs.uncompressed_block;ids = rhs.ids;+ offset_index = rhs.offset_index;+ offsets = rhs.offsets;rhs.curr_block = nullptr;- rhs.uncompressed_block = nullptr;rhs.ids = nullptr;+ rhs.offset_index = nullptr;+ rhs.offsets = nullptr;}diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 6e8d67e1..53eb2414 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -608,28 +608,13 @@ TEST(PostingListTest, IntersectionBasics) {std::vector<posting_list_t::iterator_t> its;posting_list_t::result_iter_state_t iter_state;- bool has_more = posting_list_t::block_intersect(lists, 2, its, iter_state);- ASSERT_FALSE(has_more);- ASSERT_EQ(2, iter_state.ids.size());- ASSERT_EQ(3, iter_state.ids[0]);- ASSERT_EQ(20, iter_state.ids[1]);-- ASSERT_EQ(6, iter_state.blocks.size());- ASSERT_EQ(6, iter_state.indices.size());-- // try with smaller batch size-- std::vector<posting_list_t::iterator_t> its2;- posting_list_t::result_iter_state_t iter_state2;- has_more = posting_list_t::block_intersect(lists, 1, its2, iter_state2);- ASSERT_TRUE(has_more);- ASSERT_EQ(1, iter_state2.ids.size());- ASSERT_EQ(3, iter_state2.ids[0]);-- has_more = posting_list_t::block_intersect(lists, 1, its2, iter_state2);- ASSERT_FALSE(has_more);- ASSERT_EQ(1, iter_state2.ids.size());- ASSERT_EQ(20, iter_state2.ids[0]);+ result_ids.clear();+ posting_list_t::block_intersect(lists, its, iter_state, [&](auto id, auto& its){+ result_ids.push_back(id);+ });+ ASSERT_EQ(2, result_ids.size());+ ASSERT_EQ(3, result_ids[0]);+ ASSERT_EQ(20, result_ids[1]);// single item itersectionstd::vector<posting_list_t*> single_item_list = {&p1};@@ -644,23 +629,15 @@ TEST(PostingListTest, IntersectionBasics) {std::vector<posting_list_t::iterator_t> its3;posting_list_t::result_iter_state_t iter_state3;- has_more = posting_list_t::block_intersect(single_item_list, 2, its3, iter_state3);- ASSERT_TRUE(has_more);- ASSERT_EQ(2, iter_state3.ids.size());- ASSERT_EQ(0, iter_state3.ids[0]);- ASSERT_EQ(2, iter_state3.ids[1]);- ASSERT_EQ(2, iter_state3.blocks.size());- ASSERT_EQ(2, iter_state3.indices.size());- ASSERT_EQ(0, iter_state3.indices[0]);- ASSERT_EQ(1, iter_state3.indices[1]);-- has_more = posting_list_t::block_intersect(single_item_list, 2, its3, iter_state3);- ASSERT_FALSE(has_more);- ASSERT_EQ(2, iter_state3.ids.size());- ASSERT_EQ(2, iter_state3.blocks.size());- ASSERT_EQ(2, iter_state3.indices.size());- ASSERT_EQ(0, iter_state3.indices[0]);- ASSERT_EQ(1, iter_state3.indices[1]);+ result_ids.clear();+ posting_list_t::block_intersect(single_item_list, its3, iter_state3, [&](auto id, auto& its){+ result_ids.push_back(id);+ });+ ASSERT_EQ(4, result_ids.size());+ ASSERT_EQ(0, result_ids[0]);+ ASSERT_EQ(2, result_ids[1]);+ ASSERT_EQ(3, result_ids[2]);+ ASSERT_EQ(20, result_ids[3]);// empty intersection liststd::vector<posting_list_t*> empty_list;@@ -670,11 +647,11 @@ TEST(PostingListTest, IntersectionBasics) {std::vector<posting_list_t::iterator_t> its4;posting_list_t::result_iter_state_t iter_state4;- has_more = posting_list_t::block_intersect(empty_list, 1, its4, iter_state4);- ASSERT_FALSE(has_more);- ASSERT_EQ(0, iter_state4.ids.size());- ASSERT_EQ(0, iter_state4.blocks.size());- ASSERT_EQ(0, iter_state4.indices.size());+ result_ids.clear();+ posting_list_t::block_intersect(empty_list, its4, iter_state4, [&](auto id, auto& its){+ result_ids.push_back(id);+ });+ ASSERT_EQ(0, result_ids.size());}TEST(PostingListTest, ResultsAndOffsetsBasics) {@@ -729,6 +706,7 @@ TEST(PostingListTest, ResultsAndOffsetsBasics) {lists.push_back(&p2);lists.push_back(&p3);+ /*std::vector<posting_list_t::iterator_t> its;posting_list_t::result_iter_state_t iter_state;bool has_more = posting_list_t::block_intersect(lists, 2, its, iter_state);@@ -745,6 +723,7 @@ TEST(PostingListTest, ResultsAndOffsetsBasics) {ASSERT_EQ(actual_offsets_20[0].positions, array_token_positions_vec[1].at(0)[0].positions);ASSERT_EQ(actual_offsets_20[1].positions, array_token_positions_vec[1].at(0)[1].positions);ASSERT_EQ(actual_offsets_20[2].positions, array_token_positions_vec[1].at(0)[2].positions);+ */}TEST(PostingListTest, IntersectionSkipBlocks) {@@ -1269,15 +1248,15 @@ TEST(PostingListTest, BlockIntersectionOnMixedLists) {std::vector<void*> raw_posting_lists = {SET_COMPACT_POSTING(list1), &p1};posting_list_t::result_iter_state_t iter_state;- posting_t::block_intersector_t intersector(raw_posting_lists, 1, iter_state);-- ASSERT_TRUE(intersector.intersect());- ASSERT_EQ(1, iter_state.ids.size());- ASSERT_EQ(5, iter_state.ids[0]);+ posting_t::block_intersector_t intersector(raw_posting_lists, iter_state);- ASSERT_FALSE(intersector.intersect());- ASSERT_EQ(1, iter_state.ids.size());- ASSERT_EQ(8, iter_state.ids[0]);+ std::vector<uint32_t> result_ids;+ intersector.intersect([&](auto seq_id, auto& its) {+ result_ids.push_back(seq_id);+ });+ ASSERT_EQ(2, result_ids.size());+ ASSERT_EQ(5, result_ids[0]);+ ASSERT_EQ(8, result_ids[1]);free(list1);}