Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Bake concurrency into a single index.
Kishore Nallan7b4450bbf99c
diff --git a/include/collection.h b/include/collection.hindex 04a7f06f..f6e5a244 100644--- a/include/collection.h+++ b/include/collection.h@@ -317,8 +317,6 @@ private:const std::string default_sorting_field;- std::atomic<size_t> num_memory_shards;-const float max_memory_ratio;const std::string fallback_field_type;@@ -329,7 +327,7 @@ private:std::vector<char> token_separators;- const std::vector<Index*> indices;+ Index* index;// methods@@ -391,7 +389,7 @@ private:std::set<uint64_t>& processed_syn_hashes,std::vector<std::vector<std::string>>& results) const;- std::vector<Index *> init_indices();+ Index* init_index();static std::vector<char> to_char_array(std::vector<std::string> strs);@@ -428,7 +426,7 @@ public:Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,const uint32_t next_seq_id, Store *store, const std::vector<field>& fields,- const std::string& default_sorting_field, const size_t num_memory_shards,+ const std::string& default_sorting_field,const float max_memory_ratio, const std::string& fallback_field_type,const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators);@@ -483,7 +481,7 @@ public:static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,const spp::sparse_hash_set<std::string> & exclude_fields);- const std::vector<Index *> &_get_indexes() const;+ const Index* _get_index() const;bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,std::string &value) const;@@ -492,8 +490,7 @@ public:static void populate_result_kvs(Topster *topster, std::vector<std::vector<KV *>> &result_kvs);- void batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,- size_t &num_indexed);+ void batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out, size_t &num_indexed);bool is_exceeding_memory_threshold() const;@@ -505,7 +502,7 @@ public:nlohmann::json get_summary_json() const;- size_t par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch, std::vector<size_t>& indexed_counts);+ size_t batch_index_in_memory(std::vector<index_record>& index_records);Option<nlohmann::json> add(const std::string & json_str,const index_operation_t& operation=CREATE, const std::string& id="",@@ -555,8 +552,6 @@ public:bool facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count, const nlohmann::json &document,std::string &value);- size_t get_num_memory_shards();-size_t get_num_documents() const;DIRTY_VALUES parse_dirty_values_option(std::string& dirty_values) const;diff --git a/include/field.h b/include/field.hindex 03d82ec8..a36807d9 100644--- a/include/field.h+++ b/include/field.h@@ -564,7 +564,7 @@ struct facet {std::unordered_map<uint64_t, facet_count_t> result_map;facet_stats_t stats;- facet(const std::string & field_name): field_name(field_name) {+ explicit facet(const std::string & field_name): field_name(field_name) {}};diff --git a/include/index.h b/include/index.hindex 208ac025..8358b67e 100644--- a/include/index.h+++ b/include/index.h@@ -21,6 +21,7 @@#include "magic_enum.hpp"#include "match_score.h"#include "posting_list.h"+#include "threadpool.h"struct token_t {size_t position;@@ -50,7 +51,7 @@ struct search_args {std::vector<query_tokens_t> field_query_tokens;std::vector<search_field_t> search_fields;std::vector<filter> filters;- std::vector<facet> facets;+ std::vector<facet>& facets;std::map<size_t, std::map<size_t, uint32_t>> included_ids;std::vector<uint32_t> excluded_ids;std::vector<sort_by> sort_fields_std;@@ -69,6 +70,8 @@ struct search_args {bool prioritize_exact_match;size_t all_result_ids_len;bool exhaustive_search;+ size_t concurrency;+spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;Topster* topster;@@ -76,13 +79,10 @@ struct search_args {std::vector<std::vector<KV*>> raw_result_kvs;std::vector<std::vector<KV*>> override_result_kvs;- search_args() {-- }-search_args(std::vector<query_tokens_t> field_query_tokens,std::vector<search_field_t> search_fields, std::vector<filter> filters,- std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,+ std::vector<facet>& facets,+ std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,std::vector<sort_by> sort_fields_std, facet_query_t facet_query, const std::vector<uint32_t>& num_typos,size_t max_facet_values, size_t max_hits, size_t per_page, size_t page, token_ordering token_order,const std::vector<bool>& prefixes,@@ -90,7 +90,8 @@ struct search_args {const std::vector<std::string>& group_by_fields, size_t group_limit,const std::string& default_sorting_field,bool prioritize_exact_match,- bool exhaustive_search):+ bool exhaustive_search,+ size_t concurrency):field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -99,7 +100,7 @@ struct search_args {drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),- exhaustive_search(exhaustive_search) {+ exhaustive_search(exhaustive_search), concurrency(concurrency) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -162,6 +163,8 @@ class Index {private:mutable std::shared_mutex mutex;+ ThreadPool* thread_pool;+const uint64_t FACET_ARRAY_DELIMETER = std::numeric_limits<uint64_t>::max();std::string name;@@ -230,6 +233,7 @@ private:const size_t group_limit,const std::vector<std::string>& group_by_fields,bool prioritize_exact_match,+ size_t concurrency,const token_ordering token_order = FREQUENCY, const bool prefix = false,const size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,const size_t typo_tokens_threshold = Index::TYPO_TOKENS_THRESHOLD,@@ -250,7 +254,8 @@ private:const size_t group_limit, const std::vector<std::string>& group_by_fields,const std::vector<token_t>& query_tokens,bool prioritize_exact_match,- size_t combination_limit) const;+ size_t combination_limit,+ size_t concurrency) const;void insert_doc(const int64_t score, art_tree *t, uint32_t seq_id,const std::unordered_map<std::string, std::vector<uint32_t>> &token_to_offsets) const;@@ -316,8 +321,11 @@ public:Index() = delete;- Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,- std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema,+ Index(const std::string& name,+ ThreadPool* thread_pool,+ const std::unordered_map<std::string, field>& search_schema,+ const std::map<std::string, field>& facet_schema,+ const std::unordered_map<std::string, field>& sort_schema,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators);~Index();@@ -341,7 +349,7 @@ public:const std::vector<std::string> &group_by_fields, uint32_t token_bits,bool prioritize_exact_match,bool single_exact_query_token,- std::vector<posting_list_t::iterator_t>& posting_lists) const;+ const std::vector<posting_list_t::iterator_t>& posting_lists) const;static int64_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);@@ -385,7 +393,8 @@ public:const std::vector<std::string>& group_by_fields,const std::string& default_sorting_field,bool prioritize_exact_match,- const size_t combination_limit) const;+ const size_t combination_limit,+ size_t concurrency) const;Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);diff --git a/include/posting.h b/include/posting.hindex cd9d48aa..682135b3 100644--- a/include/posting.h+++ b/include/posting.h@@ -3,6 +3,7 @@#include <cstdint>#include <vector>#include "posting_list.h"+#include "threadpool.h"#define IS_COMPACT_POSTING(x) (((uintptr_t)x & 1))#define SET_COMPACT_POSTING(x) ((void*)((uintptr_t)x | 1))@@ -48,21 +49,24 @@ private:public:struct block_intersector_t {- std::vector<posting_list_t::iterator_t> its;std::vector<posting_list_t*> plists;std::vector<uint32_t> expanded_plist_indices;posting_list_t::result_iter_state_t& iter_state;+ ThreadPool* thread_pool;+ size_t parallelize_min_ids;block_intersector_t(const std::vector<void*>& raw_posting_lists,- posting_list_t::result_iter_state_t& iter_state):- iter_state(iter_state) {+ posting_list_t::result_iter_state_t& iter_state,+ ThreadPool* thread_pool,+ size_t parallelize_min_ids = 1):+ iter_state(iter_state), thread_pool(thread_pool),+ parallelize_min_ids(parallelize_min_ids) {to_expanded_plists(raw_posting_lists, plists, expanded_plist_indices);- its.reserve(plists.size());- for(const auto& posting_list: plists) {- its.push_back(posting_list->new_iterator());- }+ std::sort(this->plists.begin(), this->plists.end(), [](posting_list_t* a, posting_list_t* b) {+ return a->num_blocks() < b->num_blocks();+ });}~block_intersector_t() {@@ -72,7 +76,9 @@ public:}template<class T>- bool intersect(T func);+ bool intersect(T func, size_t concurrency=4);++ void split_lists(size_t concurrency, std::vector<std::vector<posting_list_t::iterator_t>>& partial_its_vec);};static void upsert(void*& obj, uint32_t id, const std::vector<uint32_t>& offsets);@@ -101,7 +107,86 @@ public:};template<class T>-bool posting_t::block_intersector_t::intersect(T func) {- posting_list_t::block_intersect<T>(plists, its, iter_state, func);+bool posting_t::block_intersector_t::intersect(T func, size_t concurrency) {+ // Split posting lists into N chunks and intersect them in-parallel+ // 1. Sort posting lists by number of blocks+ // 2. Iterate on the posting list with least number of blocks on N-block windows+ // 3. On each window, pick the last block's last ID and identify blocks from other lists containing that ID+ // 4. Construct N groups of iterators this way (the last block must overlap on both sides of the window)++ if(plists.empty()) {+ return true;+ }++ if(plists[0]->num_ids() < parallelize_min_ids) {+ std::vector<posting_list_t::iterator_t> its;+ its.reserve(plists.size());++ for(const auto& posting_list: plists) {+ its.push_back(posting_list->new_iterator());+ }++ posting_list_t::block_intersect<T>(its, iter_state, func);+ return true;+ }++ std::vector<std::vector<posting_list_t::iterator_t>> partial_its_vec(concurrency);+ split_lists(concurrency, partial_its_vec);++ /*for(size_t i = 0; i < partial_its_vec.size(); i++) {+ auto& partial_its = partial_its_vec[i];++ if (partial_its.empty()) {+ continue;+ }++ LOG(INFO) << "Vec " << i;++ for (auto& it: partial_its) {+ while (it.valid()) {+ LOG(INFO) << it.id();+ it.next();+ }++ LOG(INFO) << "---";+ }+ }*/++ size_t num_processed = 0;+ std::mutex m_process;+ std::condition_variable cv_process;+ size_t num_non_empty = 0;++ for(size_t i = 0; i < partial_its_vec.size(); i++) {+ auto& partial_its = partial_its_vec[i];++ if(partial_its.empty()) {+ continue;+ }++ num_non_empty++;++ /*for(auto& it: partial_its) {+ while(it.valid()) {+ LOG(INFO) << it.id();+ it.next();+ }++ LOG(INFO) << "---";+ }*/++ thread_pool->enqueue([this, i, &func, &partial_its, &num_processed, &m_process, &cv_process]() {+ auto iter_state_copy = iter_state;+ iter_state_copy.index = i;+ posting_list_t::block_intersect<T>(partial_its, iter_state_copy, func);+ std::unique_lock<std::mutex> lock(m_process);+ num_processed++;+ cv_process.notify_one();+ });+ }++ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_non_empty; });+return true;}diff --git a/include/posting_list.h b/include/posting_list.hindex 8d7ea6c7..316f395e 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -47,6 +47,7 @@ public:block_t* curr_block;uint32_t curr_index;+ block_t* end_block;public:// uncompressed data structures for performance@@ -54,7 +55,7 @@ public:uint32_t* offset_index = nullptr;uint32_t* offsets = nullptr;- explicit iterator_t(block_t* root);+ explicit iterator_t(block_t* start, block_t* end);iterator_t(iterator_t&& rhs) noexcept;~iterator_t();[[nodiscard]] bool valid() const;@@ -73,8 +74,7 @@ public:size_t excluded_result_ids_index = 0;size_t filter_ids_index = 0;-- std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;+ size_t index = 0;result_iter_state_t() = default;@@ -130,7 +130,7 @@ public:block_t* get_root();- size_t num_blocks();+ size_t num_blocks() const;size_t num_ids();@@ -142,7 +142,7 @@ public:bool contains_atleast_one(const uint32_t* target_ids, size_t target_ids_size);- iterator_t new_iterator();+ iterator_t new_iterator(block_t* start_block = nullptr, block_t* end_block = nullptr);static void merge(const std::vector<posting_list_t*>& posting_lists, std::vector<uint32_t>& result_ids);@@ -150,7 +150,6 @@ public:template<class T>static bool block_intersect(- const std::vector<posting_list_t*>& posting_lists,std::vector<posting_list_t::iterator_t>& its,result_iter_state_t& istate,T func@@ -159,45 +158,24 @@ public:static bool take_id(result_iter_state_t& istate, uint32_t id);static bool get_offsets(- std::vector<iterator_t>& its,+ const std::vector<iterator_t>& its,std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos);- static bool is_single_token_verbatim_match(- posting_list_t::iterator_t& it,- bool field_is_array- );+ static bool is_single_token_verbatim_match(const posting_list_t::iterator_t& it, bool field_is_array);};template<class T>-bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& posting_lists,- std::vector<posting_list_t::iterator_t>& its,- result_iter_state_t& istate,+bool posting_list_t::block_intersect(std::vector<posting_list_t::iterator_t>& its, result_iter_state_t& istate,T func) {- if(posting_lists.empty()) {- return false;- }-- if(its.empty()) {- its.reserve(posting_lists.size());-- for(const auto& posting_list: posting_lists) {- its.push_back(posting_list->new_iterator());- }-- } else {- // already in the middle of iteration: prepare for next batch-- }-- size_t num_lists = its.size();-- switch (num_lists) {+ switch (its.size()) {+ case 0:+ break;case 1:while(its[0].valid()) {if(posting_list_t::take_id(istate, its[0].id())) {- func(its[0].id(), its);+ func(its[0].id(), its, istate.index);}its[0].next();@@ -207,7 +185,7 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& postingwhile(!at_end2(its)) {if(equals2(its)) {if(posting_list_t::take_id(istate, its[0].id())) {- func(its[0].id(), its);+ func(its[0].id(), its, istate.index);}advance_all2(its);@@ -221,7 +199,7 @@ bool posting_list_t::block_intersect(const std::vector<posting_list_t*>& postingif(equals(its)) {//LOG(INFO) << its[0].id();if(posting_list_t::take_id(istate, its[0].id())) {- func(its[0].id(), its);+ func(its[0].id(), its, istate.index);}advance_all(its);diff --git a/src/collection.cpp b/src/collection.cppindex ab516ce2..1fc39fdd 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -43,27 +43,23 @@ struct match_index_t {Collection::Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,- const std::string& default_sorting_field, const size_t num_memory_shards,+ const std::string& default_sorting_field,const float max_memory_ratio, const std::string& fallback_field_type,const std::vector<std::string>& symbols_to_index, const std::vector<std::string>& token_separators):name(name), collection_id(collection_id), created_at(created_at),next_seq_id(next_seq_id), store(store),fields(fields), default_sorting_field(default_sorting_field),- num_memory_shards(num_memory_shards),max_memory_ratio(max_memory_ratio),fallback_field_type(fallback_field_type), dynamic_fields({}),symbols_to_index(to_char_array(symbols_to_index)), token_separators(to_char_array(token_separators)),- indices(init_indices()) {+ index(init_index()) {this->num_documents = 0;}Collection::~Collection() {std::unique_lock lock(mutex);-- for(size_t i = 0; i < indices.size(); i++) {- delete indices[i];- }+ delete index;}uint32_t Collection::get_next_seq_id() {@@ -151,7 +147,6 @@ nlohmann::json Collection::get_summary_json() const {nlohmann::json json_response;json_response["name"] = name;- json_response["num_memory_shards"] = num_memory_shards.load();json_response["num_documents"] = num_documents.load();json_response["created_at"] = created_at.load();@@ -200,11 +195,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaconst index_operation_t& operation, const std::string& id,const DIRTY_VALUES& dirty_values) {//LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();- std::vector<std::vector<index_record>> iter_batch;-- for(size_t i = 0; i < num_memory_shards; i++) {- iter_batch.emplace_back(std::vector<index_record>());- }+ std::vector<index_record> index_records;const size_t index_batch_size = 1000;size_t num_indexed = 0;@@ -254,18 +245,17 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma}*/- iter_batch[seq_id % this->get_num_memory_shards()].emplace_back(record);+ index_records.emplace_back(record);if((i+1) % index_batch_size == 0 || i == json_lines.size()-1) {- batch_index(iter_batch, json_lines, num_indexed);- for(size_t i_index = 0; i_index < get_num_memory_shards(); i_index++) {- // to return the document for the single doc add cases- if(iter_batch[i_index].size() == 1) {- const auto& rec = iter_batch[i_index][0];- document = rec.is_update ? rec.new_doc : rec.doc;- }- iter_batch[i_index].clear();+ batch_index(index_records, json_lines, num_indexed);++ // to return the document for the single doc add cases+ if(index_records.size() == 1) {+ const auto& rec = index_records[0];+ document = rec.is_update ? rec.new_doc : rec.doc;}+ index_records.clear();}}@@ -280,70 +270,67 @@ bool Collection::is_exceeding_memory_threshold() const {return SystemMetrics::used_memory_ratio() > max_memory_ratio;}-void Collection::batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,+void Collection::batch_index(std::vector<index_record>& index_records, std::vector<std::string>& json_out,size_t &num_indexed) {- std::vector<size_t> indexed_counts;- indexed_counts.reserve(index_batches.size());- par_index_in_memory(index_batches, indexed_counts);- // store only documents that were indexed in-memory successfully- for(auto& index_batch: index_batches) {- for(auto& index_record: index_batch) {- nlohmann::json res;-- if(index_record.indexed.ok()) {- if(index_record.is_update) {- const std::string& serialized_json = index_record.new_doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);- bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);-- if(!write_ok) {- // we will attempt to reindex the old doc on a best-effort basis- LOG(ERROR) << "Update to disk failed. Will restore old document";- remove_document(index_record.new_doc, index_record.seq_id, false);- index_in_memory(index_record.old_doc, index_record.seq_id, index_record.operation, index_record.dirty_values);- index_record.index_failure(500, "Could not write to on-disk storage.");- } else {- num_indexed++;- index_record.index_success();- }+ batch_index_in_memory(index_records);+ // store only documents that were indexed in-memory successfully+ for(auto& index_record: index_records) {+ nlohmann::json res;++ if(index_record.indexed.ok()) {+ if(index_record.is_update) {+ const std::string& serialized_json = index_record.new_doc.dump(-1, ' ', false, nlohmann::detail::error_handler_t::ignore);+ bool write_ok = store->insert(get_seq_id_key(index_record.seq_id), serialized_json);++ if(!write_ok) {+ // we will attempt to reindex the old doc on a best-effort basis+ LOG(ERROR) << "Update to disk failed. Will restore old document";+ remove_document(index_record.new_doc, index_record.seq_id, false);+ index_in_memory(index_record.old_doc, index_record.seq_id, index_record.operation, index_record.dirty_values);+ index_record.index_failure(500, "Could not write to on-disk storage.");} else {- const std::string& seq_id_str = std::to_string(index_record.seq_id);- const std::string& serialized_json = index_record.doc.dump(-1, ' ', false,- nlohmann::detail::error_handler_t::ignore);-- rocksdb::WriteBatch batch;- batch.Put(get_doc_id_key(index_record.doc["id"]), seq_id_str);- batch.Put(get_seq_id_key(index_record.seq_id), serialized_json);- bool write_ok = store->batch_write(batch);-- if(!write_ok) {- // remove from in-memory store to keep the state synced- LOG(ERROR) << "Write to disk failed. Will restore old document";- remove_document(index_record.doc, index_record.seq_id, false);- index_record.index_failure(500, "Could not write to on-disk storage.");- } else {- num_indexed++;- index_record.index_success();- }+ num_indexed++;+ index_record.index_success();}- res["success"] = index_record.indexed.ok();- if(!index_record.indexed.ok()) {- res["document"] = json_out[index_record.position];- res["error"] = index_record.indexed.error();- res["code"] = index_record.indexed.code();- }} else {- res["success"] = false;+ const std::string& seq_id_str = std::to_string(index_record.seq_id);+ const std::string& serialized_json = index_record.doc.dump(-1, ' ', false,+ nlohmann::detail::error_handler_t::ignore);++ rocksdb::WriteBatch batch;+ batch.Put(get_doc_id_key(index_record.doc["id"]), seq_id_str);+ batch.Put(get_seq_id_key(index_record.seq_id), serialized_json);+ bool write_ok = store->batch_write(batch);++ if(!write_ok) {+ // remove from in-memory store to keep the state synced+ LOG(ERROR) << "Write to disk failed. Will restore old document";+ remove_document(index_record.doc, index_record.seq_id, false);+ index_record.index_failure(500, "Could not write to on-disk storage.");+ } else {+ num_indexed++;+ index_record.index_success();+ }+ }++ res["success"] = index_record.indexed.ok();+ if(!index_record.indexed.ok()) {res["document"] = json_out[index_record.position];res["error"] = index_record.indexed.error();res["code"] = index_record.indexed.code();}-- json_out[index_record.position] = res.dump(-1, ' ', false,- nlohmann::detail::error_handler_t::ignore);+ } else {+ res["success"] = false;+ res["document"] = json_out[index_record.position];+ res["error"] = index_record.indexed.error();+ res["code"] = index_record.indexed.code();}++ json_out[index_record.position] = res.dump(-1, ' ', false,+ nlohmann::detail::error_handler_t::ignore);}}@@ -359,55 +346,17 @@ Option<uint32_t> Collection::index_in_memory(nlohmann::json &document, uint32_treturn validation_op;}- Index* index = indices[seq_id % num_memory_shards];index->index_in_memory(document, seq_id, default_sorting_field, op);num_documents += 1;return Option<>(200);}-size_t Collection::par_index_in_memory(std::vector<std::vector<index_record>> & iter_batch,- std::vector<size_t>& indexed_counts) {+size_t Collection::batch_index_in_memory(std::vector<index_record>& index_records) {std::unique_lock lock(mutex);-- size_t num_processed = 0;- std::mutex m_process;- std::condition_variable cv_process;-- std::vector<size_t> num_indexed_vec(indices.size());-- for(size_t index_id = 0; index_id < indices.size(); index_id++) {- Index* index = indices[index_id];- auto parent_write_log_index = write_log_index;-- CollectionManager::get_instance().get_thread_pool()->enqueue(- [parent_write_log_index, index, index_id, &num_indexed_vec, &iter_batch, this,- &m_process, &num_processed, &cv_process]() {-- // ensures that a crash can be traced back to the write log index- write_log_index = parent_write_log_index;-- size_t num_indexed = Index::batch_memory_index(index, std::ref(iter_batch[index_id]), default_sorting_field,- search_schema, facet_schema, fallback_field_type);- std::unique_lock<std::mutex> lock(m_process);- num_indexed_vec[index_id] = num_indexed;- num_processed++;- cv_process.notify_one();- });- }-- const size_t num_indices = indices.size();- std::unique_lock<std::mutex> lock_process(m_process);- cv_process.wait(lock_process, [&](){ return num_processed == num_indices; });-- size_t num_indexed = 0;-- for(size_t index_id = 0; index_id < indices.size(); index_id++) {- num_documents += num_indexed_vec[index_id];- num_indexed += num_indexed_vec[index_id];- indexed_counts[index_id] = num_indexed_vec[index_id];- }-+ size_t num_indexed = Index::batch_memory_index(index, index_records, default_sorting_field,+ search_schema, facet_schema, fallback_field_type);+ num_documents += num_indexed;return num_indexed;}@@ -591,8 +540,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}*/- std::map<uint32_t, std::map<size_t, std::map<size_t, uint32_t>>> index_to_included_ids;- std::map<uint32_t, std::vector<uint32_t>> index_to_excluded_ids;+ std::map<size_t, std::map<size_t, uint32_t>> included_ids;for(const auto& pos_ids: include_ids) {size_t outer_pos = pos_ids.first;@@ -600,17 +548,11 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::for(size_t inner_pos = 0; inner_pos < std::min(ids_per_pos, pos_ids.second.size()); inner_pos++) {auto seq_id = pos_ids.second[inner_pos];- auto index_id = (seq_id % num_memory_shards);- index_to_included_ids[index_id][outer_pos][inner_pos] = seq_id;+ included_ids[outer_pos][inner_pos] = seq_id;//LOG(INFO) << "Adding seq_id " << seq_id << " to index_id " << index_id;}}- for(auto seq_id: excluded_ids) {- auto index_id = (seq_id % num_memory_shards);- index_to_excluded_ids[index_id].push_back(seq_id);- }-// process weights for search fieldsstd::vector<search_field_t> weighted_search_fields;size_t max_weight = 100;@@ -903,7 +845,6 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}}- std::vector<std::vector<art_leaf*>> searched_queries; // search queries used for generating the resultsstd::vector<std::vector<KV*>> raw_result_kvs;std::vector<std::vector<KV*>> override_result_kvs;@@ -934,95 +875,27 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}// search all indices- std::vector<search_args*> search_args_vec;size_t num_processed = 0;std::mutex m_process;std::condition_variable cv_process;size_t index_id = 0;- for(Index* index: indices) {- search_args* search_params = new search_args(field_query_tokens, weighted_search_fields,- filters, facets, index_to_included_ids[index_id],- index_to_excluded_ids[index_id],- sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,- per_page, page, token_order, prefixes,- drop_tokens_threshold, typo_tokens_threshold,- group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,- exhaustive_search);+ search_args* search_params = new search_args(field_query_tokens, weighted_search_fields,+ filters, facets, included_ids, excluded_ids,+ sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,+ per_page, page, token_order, prefixes,+ drop_tokens_threshold, typo_tokens_threshold,+ group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,+ exhaustive_search,+ 4);- search_args_vec.push_back(search_params);-- CollectionManager::get_instance().get_thread_pool()->enqueue(- [index, search_params, &m_process, &num_processed, &cv_process]() {- index->run_search(search_params);- std::unique_lock<std::mutex> lock(m_process);- num_processed++;- cv_process.notify_one();- });-- index_id++;- }-- const size_t num_indices = indices.size();- std::unique_lock<std::mutex> lock_process(m_process);- cv_process.wait(lock_process, [&](){ return num_processed == num_indices; });+ index->run_search(search_params);// for grouping we have to re-aggregate- const size_t topster_size = std::max((size_t)1, max_hits);- Topster topster(topster_size, group_limit);- Topster curated_topster(topster_size, group_limit);-- for(const search_args* search_params: search_args_vec) {- aggregate_topster(searched_queries.size(), topster, search_params->topster);- aggregate_topster(searched_queries.size(), curated_topster, search_params->curated_topster);-- searched_queries.insert(searched_queries.end(), search_params->searched_queries.begin(),- search_params->searched_queries.end());-- for(size_t fi = 0; fi < search_params->facets.size(); fi++) {- auto & this_facet = search_params->facets[fi];- auto & acc_facet = facets[fi];-- for(auto & facet_kv: this_facet.result_map) {- if(search_params->group_limit) {- // we have to add all group sets- acc_facet.result_map[facet_kv.first].groups.insert(- facet_kv.second.groups.begin(), facet_kv.second.groups.end()- );- } else {- size_t count = 0;- if(acc_facet.result_map.count(facet_kv.first) == 0) {- // not found, so set it- count = facet_kv.second.count;- } else {- count = acc_facet.result_map[facet_kv.first].count + facet_kv.second.count;- }- acc_facet.result_map[facet_kv.first].count = count;- }-- acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;- acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;- acc_facet.result_map[facet_kv.first].query_token_pos = facet_kv.second.query_token_pos;- }-- if(this_facet.stats.fvcount != 0) {- acc_facet.stats.fvcount += this_facet.stats.fvcount;- acc_facet.stats.fvsum += this_facet.stats.fvsum;- acc_facet.stats.fvmax = std::max(acc_facet.stats.fvmax, this_facet.stats.fvmax);- acc_facet.stats.fvmin = std::min(acc_facet.stats.fvmin, this_facet.stats.fvmin);- }- }-- if(group_limit) {- groups_processed.insert(- search_params->groups_processed.begin(),- search_params->groups_processed.end()- );- } else {- total_found += search_params->all_result_ids_len;- }- }+ Topster& topster = *search_params->topster;+ Topster& curated_topster = *search_params->curated_topster;+ const std::vector<std::vector<art_leaf*>>& searched_queries = search_params->searched_queries;topster.sort();curated_topster.sort();@@ -1038,7 +911,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}}- total_found = groups_processed.size() + override_result_kvs.size();+ total_found = search_params->groups_processed.size() + override_result_kvs.size();+ } else {+ total_found = search_params->all_result_ids_len;}// All fields are sorted descending@@ -1344,9 +1219,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}// free search params- for(auto search_params: search_args_vec) {- delete search_params;- }+ delete search_params;result["request_params"] = nlohmann::json::object();;result["request_params"]["collection_name"] = name;@@ -1465,11 +1338,9 @@ Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,return filter_op;}- for(auto& index: indices) {- uint32_t* filter_ids = nullptr;- size_t filter_ids_len = index->do_filtering(&filter_ids, filters);- index_ids.emplace_back(filter_ids_len, filter_ids);- }+ uint32_t* filter_ids = nullptr;+ size_t filter_ids_len = index->do_filtering(&filter_ids, filters);+ index_ids.emplace_back(filter_ids_len, filter_ids);return Option<bool>(true);}@@ -1569,7 +1440,6 @@ void Collection::highlight_result(const field &search_field,// Must search for the token string fresh on that field for the given document since `token_leaf`// is from the best matched field and need not be present in other fields of a document.- Index* index = indices[field_order_kv->key % num_memory_shards];art_leaf* actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len);if(actual_leaf != nullptr && posting_t::contains(actual_leaf->values, field_order_kv->key)) {@@ -1589,7 +1459,6 @@ void Collection::highlight_result(const field &search_field,continue;}- Index* index = indices[field_order_kv->key % num_memory_shards];art_leaf *actual_leaf = index->get_token_leaf(search_field.name,reinterpret_cast<const unsigned char *>(q_token.c_str()),q_token.size() + 1);@@ -1879,7 +1748,6 @@ void Collection::remove_document(const nlohmann::json & document, const uint32_t{std::unique_lock lock(mutex);- Index* index = indices[seq_id % num_memory_shards];index->remove(seq_id, document, false);num_documents -= 1;}@@ -1977,10 +1845,6 @@ Option<uint32_t> Collection::remove_override(const std::string & id) {return Option<uint32_t>(404, "Could not find that `id`.");}-size_t Collection::get_num_memory_shards() {- return num_memory_shards.load();-}-uint32_t Collection::get_seq_id_from_key(const std::string & key) {// last 4 bytes of the key would be the serialized version of the sequence idstd::string serialized_seq_id = key.substr(key.length() - 4);@@ -2127,8 +1991,8 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,return Option<bool>(true);}-const std::vector<Index *> &Collection::_get_indexes() const {- return indices;+const Index* Collection::_get_index() const {+ return index;}Option<bool> Collection::parse_geopoint_filter_value(std::string& raw_value,@@ -2755,9 +2619,7 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constreturn Option<bool>(500, "Could not persist collection meta to store.");}- for(auto index: indices) {- index->refresh_schemas(new_fields);- }+ index->refresh_schemas(new_fields);} catch(...) {return Option<bool>(500, "Unable to parse collection meta.");@@ -2767,9 +2629,7 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, constreturn Option<bool>(true);}-std::vector<Index *> Collection::init_indices() {- std::vector<Index*> index_list;-+Index* Collection::init_index() {for(const field& field: fields) {if(field.is_dynamic()) {// regexp fields are treated as dynamic fields@@ -2788,13 +2648,10 @@ std::vector<Index *> Collection::init_indices() {}}- for(size_t i = 0; i < num_memory_shards; i++) {- Index* index = new Index(name+std::to_string(i), search_schema, facet_schema, sort_schema,- symbols_to_index, token_separators);- index_list.push_back(index);- }-- return index_list;+ return new Index(name+std::to_string(0),+ CollectionManager::get_instance().get_thread_pool(),+ search_schema, facet_schema, sort_schema,+ symbols_to_index, token_separators);}DIRTY_VALUES Collection::parse_dirty_values_option(std::string& dirty_values) const {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex d941234d..76588e38 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -72,7 +72,6 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectionstore,fields,default_sorting_field,- num_memory_shards,max_memory_ratio,fallback_field_type,symbols_to_index,@@ -271,7 +270,7 @@ Option<Collection*> CollectionManager::create_collection(const std::string& namecollection_meta[Collection::COLLECTION_SEPARATORS] = token_separators;Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,- default_sorting_field, num_memory_shards,+ default_sorting_field,this->max_memory_ratio, fallback_field_type,symbols_to_index, token_separators);next_collection_id++;@@ -1032,11 +1031,7 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionrocksdb::Iterator* iter = cm.store->scan(seq_id_prefix);std::unique_ptr<rocksdb::Iterator> iter_guard(iter);- std::vector<std::vector<index_record>> iter_batch;-- for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {- iter_batch.emplace_back(std::vector<index_record>());- }+ std::vector<index_record> index_records;size_t num_found_docs = 0;size_t num_valid_docs = 0;@@ -1059,7 +1054,7 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionnum_valid_docs++;- iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));+ index_records.emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));// Peek and check for last record right here so that we handle batched indexing correctly// Without doing this, the "last batch" would have to be indexed outside the loop.@@ -1067,27 +1062,21 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionbool last_record = !(iter->Valid() && iter->key().starts_with(seq_id_prefix));// batch must match atleast the number of shards- const size_t batch_size = std::max(init_batch_size, collection->get_num_memory_shards());+ const size_t batch_size = 1;if((num_valid_docs % batch_size == 0) || last_record) {- std::vector<size_t> indexed_counts;- indexed_counts.reserve(iter_batch.size());-- collection->par_index_in_memory(iter_batch, indexed_counts);+ size_t num_records = index_records.size();+ size_t num_indexed = collection->batch_index_in_memory(index_records);- for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {- size_t num_records = iter_batch[i].size();- size_t num_indexed = indexed_counts[i];-- if(num_indexed != num_records) {- const Option<std::string> & index_error_op = get_first_index_error(iter_batch[i]);- if(!index_error_op.ok()) {- return Option<bool>(false, index_error_op.get());- }+ if(num_indexed != num_records) {+ const Option<std::string> & index_error_op = get_first_index_error(index_records);+ if(!index_error_op.ok()) {+ return Option<bool>(false, index_error_op.get());}- iter_batch[i].clear();- num_indexed_docs += num_indexed;}++ index_records.clear();+ num_indexed_docs += num_indexed;}}diff --git a/src/index.cpp b/src/index.cppindex 42a4f508..ceabf9fc 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -23,10 +23,12 @@spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;-Index::Index(const std::string name, const std::unordered_map<std::string, field> & search_schema,- std::map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema,+Index::Index(const std::string& name, ThreadPool* thread_pool,+ const std::unordered_map<std::string, field> & search_schema,+ const std::map<std::string, field>& facet_schema, const std::unordered_map<std::string, field>& sort_schema,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):- name(name), search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),+ name(name), thread_pool(thread_pool),+ search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),symbols_to_index(symbols_to_index), token_separators(token_separators) {for(const auto & fname_field: search_schema) {@@ -696,7 +698,7 @@ void Index::compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::s}void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,- size_t group_limit, const std::vector<std::string>& group_by_fields,+ const size_t group_limit, const std::vector<std::string>& group_by_fields,const uint32_t* result_ids, size_t results_size) const {struct facet_info_t {@@ -785,7 +787,6 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,for(size_t i = 0; i < results_size; i++) {uint32_t doc_seq_id = result_ids[i];-const auto& facet_hashes_it = field_facet_mapping->find(doc_seq_id);if(facet_hashes_it == field_facet_mapping->end()) {@@ -883,13 +884,29 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}}+void aggregate_topster(Topster& agg_topster, Topster* index_topster) {+ if(index_topster->distinct) {+ for(auto &group_topster_entry: index_topster->group_kv_map) {+ Topster* group_topster = group_topster_entry.second;+ for(const auto& map_kv: group_topster->kv_map) {+ agg_topster.add(map_kv.second);+ }+ }+ } else {+ for(const auto& map_kv: index_topster->kv_map) {+ agg_topster.add(map_kv.second);+ }+ }+}+void Index::search_candidates(const uint8_t & field_id, bool field_is_array,uint32_t* filter_ids, size_t filter_ids_length,const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,const std::vector<uint32_t>& curated_ids,const std::vector<sort_by> & sort_fields,std::vector<token_candidates> & token_candidates_vec,- std::vector<std::vector<art_leaf*>> & searched_queries, Topster* topster,+ std::vector<std::vector<art_leaf*>> & searched_queries,+ Topster* topster,spp::sparse_hash_set<uint64_t>& groups_processed,uint32_t** all_result_ids, size_t & all_result_ids_len,size_t& field_num_results,@@ -898,7 +915,8 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,const std::vector<std::string>& group_by_fields,const std::vector<token_t>& query_tokens,bool prioritize_exact_match,- const size_t combination_limit) const {+ const size_t combination_limit,+ const size_t concurrency) const {auto product = []( long long a, token_candidates & b ) { return a*b.candidates.size(); };long long int N = std::accumulate(token_candidates_vec.begin(), token_candidates_vec.end(), 1LL, product);@@ -971,36 +989,52 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,single_exact_query_token = true;}- std::vector<uint32_t> result_id_vec;- posting_t::block_intersector_t intersector(posting_lists, iter_state);+ std::vector<uint32_t> result_id_vecs[concurrency];++ Topster* topsters[concurrency];+ for(size_t i = 0; i < concurrency; i++) {+ topsters[i] = new Topster(topster->MAX_SIZE, topster->distinct);+ }++ std::vector<spp::sparse_hash_set<uint64_t>> groups_processed_vec(concurrency);- intersector.intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its) {- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, field_is_array,- total_cost, topster, query_suggestion, groups_processed,+ posting_t::block_intersector_t(+ posting_lists, iter_state, thread_pool, 100+ )+ .intersect([&](uint32_t seq_id, std::vector<posting_list_t::iterator_t>& its, size_t index) {+ score_results(sort_fields, searched_queries.size(), field_id, field_is_array,+ total_cost, topsters[index], query_suggestion, groups_processed_vec[index],seq_id, sort_order, field_values, geopoint_indices,group_limit, group_by_fields, token_bits,prioritize_exact_match, single_exact_query_token, its);- result_id_vec.push_back(seq_id);- });+ result_id_vecs[index].push_back(seq_id);+ }, concurrency);- if(result_id_vec.empty()) {- continue;- }+ delete [] excluded_result_ids;- uint32_t* result_ids = &result_id_vec[0];- size_t result_size = result_id_vec.size();+ size_t num_result_ids = 0;- field_num_results += result_size;+ for(size_t i = 0; i < concurrency; i++) {+ aggregate_topster(*topster, topsters[i]);+ delete topsters[i];- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,- result_size, &new_all_result_ids);- delete [] *all_result_ids;- *all_result_ids = new_all_result_ids;+ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, &result_id_vecs[i][0],+ result_id_vecs[i].size(), &new_all_result_ids);+ delete [] *all_result_ids;+ *all_result_ids = new_all_result_ids;- delete [] excluded_result_ids;+ num_result_ids += result_id_vecs[i].size();++ groups_processed.insert(groups_processed_vec[i].begin(), groups_processed_vec[i].end());+ }++ if(num_result_ids == 0) {+ continue;+ }+ field_num_results += num_result_ids;searched_queries.push_back(actual_query_suggestion);}}@@ -1353,7 +1387,8 @@ void Index::run_search(search_args* search_params) {search_params->group_limit, search_params->group_by_fields,search_params->default_sorting_field,search_params->prioritize_exact_match,- search_params->exhaustive_search);+ search_params->exhaustive_search,+ search_params->concurrency);}void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,@@ -1446,7 +1481,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,const std::vector<std::string>& group_by_fields,const std::string& default_sorting_field,bool prioritize_exact_match,- const size_t combination_limit) const {+ const size_t combination_limit,+ const size_t concurrency) const {std::shared_lock lock(mutex);@@ -1630,7 +1666,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,- field_num_results, group_limit, group_by_fields, prioritize_exact_match, token_order, field_prefix,+ field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,+ token_order, field_prefix,drop_tokens_threshold, typo_tokens_threshold, combination_limit);// do synonym based searches@@ -1642,7 +1679,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,field_name, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,- field_num_results, group_limit, group_by_fields, prioritize_exact_match, token_order, field_prefix,+ field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,+ token_order, field_prefix,drop_tokens_threshold, typo_tokens_threshold, combination_limit);}@@ -1835,7 +1873,86 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,delete [] exclude_token_ids;- do_facets(facets, facet_query, group_limit, group_by_fields, all_result_ids, all_result_ids_len);+ const size_t num_threads = std::min(concurrency, all_result_ids_len);+ const size_t window_size = (num_threads == 0) ? 0 :+ (all_result_ids_len + num_threads - 1) / num_threads; // rounds up+ size_t result_index = 0;++ size_t num_processed = 0;+ std::mutex m_process;+ std::condition_variable cv_process;++ std::vector<std::vector<facet>> facet_batches(num_threads);+ for(size_t i = 0; i < num_threads; i++) {+ for(const auto& this_facet: facets) {+ facet_batches[i].emplace_back(facet(this_facet.field_name));+ }+ }++ size_t num_queued = 0;++ for(size_t thread_id = 0; thread_id < num_threads && result_index < all_result_ids_len; thread_id++) {+ size_t batch_res_len = window_size;++ if(result_index + window_size > all_result_ids_len) {+ batch_res_len = (result_index + window_size) - all_result_ids_len;+ }++ uint32_t* batch_result_ids = all_result_ids + result_index;+ num_queued++;++ thread_pool->enqueue([this, thread_id, &facet_batches, &facet_query, group_limit, group_by_fields,+ batch_result_ids, batch_res_len, &num_processed, &m_process, &cv_process]() {+ auto fq = facet_query;+ do_facets(facet_batches[thread_id], fq, group_limit, group_by_fields,+ batch_result_ids, batch_res_len);+ std::unique_lock<std::mutex> lock(m_process);+ num_processed++;+ cv_process.notify_one();+ });++ result_index += batch_res_len;+ }++ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_queued; });++ for(const auto& facet_batch: facet_batches) {+ for(size_t fi = 0; fi < facet_batch.size(); fi++) {+ auto& this_facet = facet_batch[fi];+ auto& acc_facet = facets[fi];++ for(auto & facet_kv: this_facet.result_map) {+ if(group_limit) {+ // we have to add all group sets+ acc_facet.result_map[facet_kv.first].groups.insert(+ facet_kv.second.groups.begin(), facet_kv.second.groups.end()+ );+ } else {+ size_t count = 0;+ if(acc_facet.result_map.count(facet_kv.first) == 0) {+ // not found, so set it+ count = facet_kv.second.count;+ } else {+ count = acc_facet.result_map[facet_kv.first].count + facet_kv.second.count;+ }+ acc_facet.result_map[facet_kv.first].count = count;+ }++ acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;+ acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;+ acc_facet.result_map[facet_kv.first].query_token_pos = facet_kv.second.query_token_pos;+ }++ if(this_facet.stats.fvcount != 0) {+ acc_facet.stats.fvcount += this_facet.stats.fvcount;+ acc_facet.stats.fvsum += this_facet.stats.fvsum;+ acc_facet.stats.fvmax = std::max(acc_facet.stats.fvmax, this_facet.stats.fvmax);+ acc_facet.stats.fvmin = std::min(acc_facet.stats.fvmin, this_facet.stats.fvmin);+ }+ }+ }+do_facets(facets, facet_query, group_limit, group_by_fields, &included_ids[0], included_ids.size());all_result_ids_len += curated_topster->size;@@ -1876,6 +1993,7 @@ void Index::search_field(const uint8_t & field_id,uint32_t** all_result_ids, size_t & all_result_ids_len, size_t& field_num_results,const size_t group_limit, const std::vector<std::string>& group_by_fields,bool prioritize_exact_match,+ const size_t concurrency,const token_ordering token_order, const bool prefix,const size_t drop_tokens_threshold,const size_t typo_tokens_threshold,@@ -2004,7 +2122,7 @@ void Index::search_field(const uint8_t & field_id,curated_ids, sort_fields, token_candidates_vec, searched_queries, topster,groups_processed, all_result_ids, all_result_ids_len, field_num_results,typo_tokens_threshold, group_limit, group_by_fields, query_tokens,- prioritize_exact_match, combination_limit);+ prioritize_exact_match, combination_limit, concurrency);}resume_typo_loop:@@ -2047,7 +2165,8 @@ void Index::search_field(const uint8_t & field_id,return search_field(field_id, query_tokens, truncated_tokens, exclude_token_ids, exclude_token_ids_size,num_tokens_dropped, field, filter_ids, filter_ids_length, curated_ids,facets,sort_fields, num_typos,searched_queries, topster, groups_processed, all_result_ids,- all_result_ids_len, field_num_results, group_limit, group_by_fields, prioritize_exact_match,+ all_result_ids_len, field_num_results, group_limit, group_by_fields,+ prioritize_exact_match, concurrency,token_order, prefix, drop_tokens_threshold, typo_tokens_threshold, combination_limit);}}@@ -2087,10 +2206,10 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3> field_values,const std::vector<size_t>& geopoint_indices,const size_t group_limit, const std::vector<std::string>& group_by_fields,- uint32_t token_bits,+ const uint32_t token_bits,bool prioritize_exact_match,bool single_exact_query_token,- std::vector<posting_list_t::iterator_t>& posting_lists) const {+ const std::vector<posting_list_t::iterator_t>& posting_lists) const {spp::sparse_hash_map<uint32_t, int64_t>* TEXT_MATCH_SENTINEL = &text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t>* SEQ_ID_SENTINEL = &seq_id_sentinel_value;diff --git a/src/posting.cpp b/src/posting.cppindex cc5c0e03..6b2ef148 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -436,4 +436,58 @@ void posting_t::get_array_token_positions(uint32_t id, const std::vector<void*>&for(uint32_t expanded_plist_index: expanded_plist_indices) {delete plists[expanded_plist_index];}-}\ No newline at end of file+}++void posting_t::block_intersector_t::split_lists(size_t concurrency,+ std::vector<std::vector<posting_list_t::iterator_t>>& partial_its_vec) {+ const size_t num_blocks = this->plists[0]->num_blocks();+ const size_t window_size = (num_blocks + concurrency - 1) / concurrency; // rounds up++ size_t blocks_traversed = 0;+ posting_list_t::block_t* start_block = this->plists[0]->get_root();+ posting_list_t::block_t* curr_block = start_block;++ size_t window_index = 0;++ while(curr_block != nullptr) {+ blocks_traversed++;+ if(blocks_traversed % window_size == 0 || blocks_traversed == num_blocks) {+ // construct partial iterators and intersect within them++ std::vector<posting_list_t::iterator_t>& partial_its = partial_its_vec[window_index];++ for(size_t i = 0; i < this->plists.size(); i++) {+ posting_list_t::block_t* p_start_block = nullptr;+ posting_list_t::block_t* p_end_block = nullptr;++ // [1, 2] [3, 4] [5, 6]+ // [3, 5] [6]++ if(i == 0) {+ auto& plist = this->plists[i];+ p_start_block = start_block;+ p_end_block = curr_block->next;+ } else {+ auto start_block_first_id = start_block->ids.at(0);+ auto end_block_last_id = curr_block->ids.last();++ p_start_block = this->plists[i]->block_of(start_block_first_id);+ posting_list_t::block_t* last_block = this->plists[i]->block_of(end_block_last_id);++ if(p_start_block == last_block && p_start_block != nullptr) {+ p_end_block = p_start_block->next;+ } else {+ p_end_block = last_block == nullptr ? nullptr : last_block->next;+ }+ }++ partial_its.emplace_back(p_start_block, p_end_block);+ }++ start_block = curr_block->next;+ window_index++;+ }++ curr_block = curr_block->next;+ }+}diff --git a/src/posting_list.cpp b/src/posting_list.cppindex 03bcc85e..bd048cbf 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -531,7 +531,7 @@ posting_list_t::block_t* posting_list_t::get_root() {return &root_block;}-size_t posting_list_t::num_blocks() {+size_t posting_list_t::num_blocks() const {return id_block_map.size();}@@ -697,7 +697,7 @@ bool posting_list_t::take_id(result_iter_state_t& istate, uint32_t id) {return id_found_in_filter;}-bool posting_list_t::get_offsets(std::vector<iterator_t>& its,+bool posting_list_t::get_offsets(const std::vector<iterator_t>& its,std::unordered_map<size_t, std::vector<token_positions_t>>& array_token_pos) {// Plain string format:@@ -791,7 +791,7 @@ bool posting_list_t::get_offsets(std::vector<iterator_t>& its,return true;}-bool posting_list_t::is_single_token_verbatim_match(posting_list_t::iterator_t& it, bool field_is_array) {+bool posting_list_t::is_single_token_verbatim_match(const posting_list_t::iterator_t& it, bool field_is_array) {block_t* curr_block = it.block();uint32_t curr_index = it.index();@@ -864,8 +864,9 @@ bool posting_list_t::equals2(std::vector<posting_list_t::iterator_t>& its) {return its[0].id() == its[1].id();}-posting_list_t::iterator_t posting_list_t::new_iterator() {- return posting_list_t::iterator_t(&root_block);+posting_list_t::iterator_t posting_list_t::new_iterator(block_t* start_block, block_t* end_block) {+ start_block = (start_block == nullptr) ? &root_block : start_block;+ return posting_list_t::iterator_t(start_block, end_block);}void posting_list_t::advance_all(std::vector<posting_list_t::iterator_t>& its) {@@ -978,10 +979,10 @@ bool posting_list_t::contains_atleast_one(const uint32_t* target_ids, size_t tar/* iterator_t operations */-posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root):- curr_block(root), curr_index(0) {+posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* start, posting_list_t::block_t* end):+ curr_block(start), curr_index(0), end_block(end) {- if(curr_block != nullptr) {+ if(curr_block != end_block) {ids = curr_block->ids.uncompress();offset_index = curr_block->offset_index.uncompress();offsets = curr_block->offsets.uncompress();@@ -989,7 +990,7 @@ posting_list_t::iterator_t::iterator_t(posting_list_t::block_t* root):}bool posting_list_t::iterator_t::valid() const {- return (curr_block != nullptr) && (curr_index < curr_block->size());+ return (curr_block != end_block) && (curr_index < curr_block->size());}void posting_list_t::iterator_t::next() {@@ -1004,7 +1005,7 @@ void posting_list_t::iterator_t::next() {ids = offset_index = offsets = nullptr;- if(curr_block != nullptr) {+ if(curr_block != end_block) {ids = curr_block->ids.uncompress();offset_index = curr_block->offset_index.uncompress();offsets = curr_block->offsets.uncompress();@@ -1026,7 +1027,7 @@ posting_list_t::block_t* posting_list_t::iterator_t::block() const {void posting_list_t::iterator_t::skip_to(uint32_t id) {bool skipped_block = false;- while(curr_block != nullptr && curr_block->ids.last() < id) {+ while(curr_block != end_block && curr_block->ids.last() < id) {curr_block = curr_block->next;// FIXME: remove duplication@@ -1036,7 +1037,7 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {ids = offset_index = offsets = nullptr;- if(curr_block != nullptr) {+ if(curr_block != end_block) {ids = curr_block->ids.uncompress();offset_index = curr_block->offset_index.uncompress();offsets = curr_block->offsets.uncompress();@@ -1049,7 +1050,7 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {curr_index = 0;}- while(curr_block != nullptr && curr_index < curr_block->size() && this->id() < id) {+ while(curr_block != end_block && curr_index < curr_block->size() && this->id() < id) {curr_index++;}}@@ -1057,16 +1058,24 @@ void posting_list_t::iterator_t::skip_to(uint32_t id) {posting_list_t::iterator_t::~iterator_t() {delete [] ids;ids = nullptr;++ delete [] offsets;+ offsets = nullptr;++ delete [] offset_index;+ offset_index = nullptr;}posting_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {curr_block = rhs.curr_block;curr_index = rhs.curr_index;+ end_block = rhs.end_block;ids = rhs.ids;offset_index = rhs.offset_index;offsets = rhs.offsets;rhs.curr_block = nullptr;+ rhs.end_block = nullptr;rhs.ids = nullptr;rhs.offset_index = nullptr;rhs.offsets = nullptr;diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 95b382b1..34f33ab2 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -957,7 +957,7 @@ TEST_F(CollectionAllFieldsTest, DoNotIndexFieldMarkedAsNonIndex) {auto add_op = coll1->add(doc.dump(), CREATE);ASSERT_TRUE(add_op.ok());- ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().count("post"));+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().count("post"));auto res_op = coll1->search("Amazon", {"description_txt"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false});ASSERT_FALSE(res_op.ok());@@ -972,7 +972,7 @@ TEST_F(CollectionAllFieldsTest, DoNotIndexFieldMarkedAsNonIndex) {auto update_op = coll1->add(doc.dump(), UPDATE, "0");ASSERT_TRUE(add_op.ok());- ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().count("post"));+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().count("post"));auto res = coll1->search("Amazon", {"company_name"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ("Some post updated.", res["hits"][0]["document"]["post"].get<std::string>());diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 228e01c5..97e56f84 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -417,10 +417,10 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);- ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 245aa490..aecdfc4c 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -165,11 +165,11 @@ TEST_F(CollectionSortingTest, NoDefaultSortingField) {// without a default sorting field, matches should be sorted by (text_match, seq_id)auto results = coll1->search("rocket", {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {false}).get();- ASSERT_EQ(5, results["found"]);- ASSERT_EQ(5, results["hits"].size());+ ASSERT_EQ(4, results["found"].get<size_t>());+ ASSERT_EQ(4, results["hits"].size());ASSERT_EQ(24, results["out_of"]);- std::vector<std::string> ids = {"16", "15", "7", "0", "22"};+ std::vector<std::string> ids = {"16", "15", "7", "0"};for(size_t i=0; i < results["hits"].size(); i++) {ASSERT_EQ(ids[i], results["hits"][i]["document"]["id"].get<std::string>());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 42e1e9d5..3891bd04 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -284,6 +284,23 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// with 2 indexed wordsresults = collection->search("from DoesNotExist insTruments", query_fields, "", facets, sort_fields, {1}, 10).get();+ ASSERT_EQ(1, results["hits"].size());+ ids = {"2"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string id = ids.at(i);+ std::string result_id = result["document"]["id"];+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // exhaustive search should throw more results+ results = collection->search("from DoesNotExist insTruments", query_fields, "", facets, sort_fields, {1}, 10,+ 1, FREQUENCY, {true},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 1, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000,+ true, false, true, "", true).get();ASSERT_EQ(2, results["hits"].size());ids = {"2", "17"};@@ -941,9 +958,9 @@ TEST_F(CollectionTest, MultipleFields) {query_fields = {"cast"};results = coll_mul_fields->search("chris pine", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();- ASSERT_EQ(3, results["hits"].size());+ ASSERT_EQ(1, results["hits"].size());- ids = {"7", "6", "1"};+ ids = {"7"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);std::string result_id = result["document"]["id"];@@ -1873,7 +1890,7 @@ TEST_F(CollectionTest, DeletionOfDocumentSingularFields) {ASSERT_EQ(0, res["found"].get<int32_t>());// also assert against the actual index- Index *index = coll1->_get_indexes()[0]; // seq id will always be zero for first document+ const Index *index = coll1->_get_index(); // seq id will always be zero for first documentauto search_index = index->_get_search_index();auto numerical_index = index->_get_numerical_index();@@ -1939,7 +1956,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {ASSERT_EQ(0, res["found"].get<int32_t>());// also assert against the actual index- Index *index = coll1->_get_indexes()[0]; // seq id will always be zero for first document+ const Index *index = coll1->_get_index(); // seq id will always be zero for first documentauto search_index = index->_get_search_index();auto numerical_index = index->_get_numerical_index();@@ -2010,7 +2027,8 @@ TEST_F(CollectionTest, SearchLargeTextField) {ASSERT_TRUE(res_op.ok());results = res_op.get();- ASSERT_EQ(2, results["hits"].size());++ ASSERT_EQ(1, results["hits"].size());ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());@@ -2279,7 +2297,6 @@ TEST_F(CollectionTest, UpdateDocument) {// upserting without a mandatory field should be an errorpartial_doc = doc;partial_doc.erase("title");- LOG(INFO) << partial_doc.dump();add_op = coll1->add(partial_doc.dump(), UPSERT);ASSERT_FALSE(add_op.ok());ASSERT_EQ("Field `title` has been declared in the schema, but is not found in the document.", add_op.error());@@ -2698,9 +2715,9 @@ TEST_F(CollectionTest, OptionalFieldCanBeNull) {ASSERT_TRUE(coll1->add(doc.dump()).ok());- ASSERT_EQ(2, coll1->_get_indexes()[0]->_get_search_index().at("title")->size);- ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("artist")->size);- ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("genres")->size);+ ASSERT_EQ(2, coll1->_get_index()->_get_search_index().at("title")->size);+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().at("artist")->size);+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().at("genres")->size);auto results = coll1->search("beat",{"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY).get();@@ -2744,11 +2761,11 @@ TEST_F(CollectionTest, EmptyStringNotIndexed) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ(2, coll1->_get_indexes()[0]->_get_search_index().at("title")->size);- ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("artist")->size);- ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("launch_year")->size);- ASSERT_EQ(0, coll1->_get_indexes()[0]->_get_search_index().at("genres")->size);- ASSERT_EQ(1, coll1->_get_indexes()[0]->_get_search_index().at("labels")->size);+ ASSERT_EQ(2, coll1->_get_index()->_get_search_index().at("title")->size);+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().at("artist")->size);+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().at("launch_year")->size);+ ASSERT_EQ(0, coll1->_get_index()->_get_search_index().at("genres")->size);+ ASSERT_EQ(1, coll1->_get_index()->_get_search_index().at("labels")->size);collectionManager.drop_collection("coll1");}diff --git a/test/index_test.cpp b/test/index_test.cppindex 957b376c..8ca0ef07 100644--- a/test/index_test.cpp+++ b/test/index_test.cpp@@ -10,7 +10,9 @@ TEST(IndexTest, ScrubReindexDoc) {search_schema.emplace("cast", field("cast", field_types::STRING_ARRAY, false));search_schema.emplace("movie", field("movie", field_types::BOOL, false));- Index index("index", search_schema, {}, {}, {}, {});+ ThreadPool pool(4);++ Index index("index", &pool, search_schema, {}, {}, {}, {});nlohmann::json old_doc;old_doc["id"] = "1";old_doc["title"] = "One more thing.";@@ -57,6 +59,8 @@ TEST(IndexTest, ScrubReindexDoc) {ASSERT_STREQ("1", del_doc3["id"].get<std::string>().c_str());ASSERT_STREQ("The Lawyer", del_doc3["title"].get<std::string>().c_str());ASSERT_STREQ("Bar", del_doc3["foo"].get<std::string>().c_str());++ pool.shutdown();}TEST(IndexTest, PointInPolygon180thMeridian) {diff --git a/test/posting_list_test.cpp b/test/posting_list_test.cppindex 53eb2414..675b2762 100644--- a/test/posting_list_test.cpp+++ b/test/posting_list_test.cpp@@ -4,7 +4,21 @@#include <chrono>#include <vector>-TEST(PostingListTest, Insert) {+class PostingListTest : public ::testing::Test {+protected:+ ThreadPool* pool;++ virtual void SetUp() {+ pool = new ThreadPool(4);+ }++ virtual void TearDown() {+ pool->shutdown();+ delete pool;+ }+};++TEST_F(PostingListTest, Insert) {std::vector<uint32_t> offsets = {0, 1, 3};posting_list_t pl(5);@@ -115,7 +129,7 @@ TEST(PostingListTest, Insert) {}}-TEST(PostingListTest, InplaceUpserts) {+TEST_F(PostingListTest, InplaceUpserts) {std::vector<uint32_t> offsets = {1, 2, 3};posting_list_t pl(5);@@ -272,7 +286,7 @@ TEST(PostingListTest, InplaceUpserts) {ASSERT_EQ(6, pl.get_root()->offset_index.at(2));}-TEST(PostingListTest, RemovalsOnFirstBlock) {+TEST_F(PostingListTest, RemovalsOnFirstBlock) {std::vector<uint32_t> offsets = {0, 1, 3};posting_list_t pl(5);@@ -369,7 +383,7 @@ TEST(PostingListTest, RemovalsOnFirstBlock) {}}-TEST(PostingListTest, RemovalsOnLaterBlocks) {+TEST_F(PostingListTest, RemovalsOnLaterBlocks) {std::vector<uint32_t> offsets = {0, 1, 3};posting_list_t pl(5);@@ -459,7 +473,7 @@ TEST(PostingListTest, RemovalsOnLaterBlocks) {}}-TEST(PostingListTest, OutOfOrderUpserts) {+TEST_F(PostingListTest, OutOfOrderUpserts) {std::vector<uint32_t> offsets = {0, 1, 3};posting_list_t pl(5);@@ -492,7 +506,7 @@ TEST(PostingListTest, OutOfOrderUpserts) {}}-TEST(PostingListTest, RandomInsertAndDeletes) {+TEST_F(PostingListTest, RandomInsertAndDeletes) {time_t t;srand((unsigned) time(&t));@@ -522,7 +536,7 @@ TEST(PostingListTest, RandomInsertAndDeletes) {ASSERT_LT(pl.num_blocks(), 1000);}-TEST(PostingListTest, MergeBasics) {+TEST_F(PostingListTest, MergeBasics) {std::vector<uint32_t> offsets = {0, 1, 3};std::vector<posting_list_t*> lists;@@ -566,9 +580,8 @@ TEST(PostingListTest, MergeBasics) {}}-TEST(PostingListTest, IntersectionBasics) {+TEST_F(PostingListTest, SplittingOfListsSimple) {std::vector<uint32_t> offsets = {0, 1, 3};- std::vector<posting_list_t*> lists;// [0, 2] [3, 20]// [1, 3], [5, 10], [20]@@ -594,13 +607,95 @@ TEST(PostingListTest, IntersectionBasics) {p3.upsert(7, offsets);p3.upsert(20, offsets);- lists.push_back(&p1);- lists.push_back(&p2);- lists.push_back(&p3);+ std::vector<void*> raw_lists = {&p1, &p2, &p3};++ std::vector<posting_list_t::iterator_t> its;+ posting_list_t::result_iter_state_t iter_state;+ posting_t::block_intersector_t intersector(raw_lists, iter_state, pool);++ std::vector<std::vector<posting_list_t::iterator_t>> partial_its_vec(4);+ intersector.split_lists(4, partial_its_vec);++ /*for(size_t i = 0; i < partial_its_vec.size(); i++) {+ auto& partial_its = partial_its_vec[i];++ if (partial_its.empty()) {+ continue;+ }++ LOG(INFO) << "Vec " << i;++ for (auto& it: partial_its) {+ while (it.valid()) {+ LOG(INFO) << it.id();+ it.next();+ }++ LOG(INFO) << "---";+ }+ }++ return ;*/++ std::vector<std::vector<std::vector<uint32_t>>> split_ids = {+ {{0, 2}, {1, 3}, {2, 3}},+ {{3, 20}, {1, 3, 5, 10, 20}, {2, 3, 5, 7, 20}}+ };++ ASSERT_EQ(4, partial_its_vec.size());+ ASSERT_EQ(3, partial_its_vec[0].size());+ ASSERT_EQ(3, partial_its_vec[1].size());++ for(size_t i = 0; i < partial_its_vec.size(); i++) {+ auto& partial_its = partial_its_vec[i];+ for (size_t j = 0; j < partial_its.size(); j++) {+ auto& it = partial_its[j];+ size_t k = 0;++ while (it.valid()) {+ //LOG(INFO) << it.id();+ ASSERT_EQ(split_ids[i][j][k], it.id());+ k++;+ it.next();+ }++ //LOG(INFO) << "---";+ }+ }+}++TEST_F(PostingListTest, IntersectionBasics) {+ std::vector<uint32_t> offsets = {0, 1, 3};++ // [0, 2] [3, 20]+ // [1, 3], [5, 10], [20]+ // [2, 3], [5, 7], [20]+ posting_list_t p1(2);+ p1.upsert(0, offsets);+ p1.upsert(2, offsets);+ p1.upsert(3, offsets);+ p1.upsert(20, offsets);++ posting_list_t p2(2);+ p2.upsert(1, offsets);+ p2.upsert(3, offsets);+ p2.upsert(5, offsets);+ p2.upsert(10, offsets);+ p2.upsert(20, offsets);++ posting_list_t p3(2);+ p3.upsert(2, offsets);+ p3.upsert(3, offsets);+ p3.upsert(5, offsets);+ p3.upsert(7, offsets);+ p3.upsert(20, offsets);++ std::vector<void*> raw_lists = {&p1, &p2, &p3};+ std::vector<posting_list_t*> posting_lists = {&p1, &p2, &p3};std::vector<uint32_t> result_ids;- posting_list_t::intersect(lists, result_ids);+ posting_list_t::intersect(posting_lists, result_ids);ASSERT_EQ(2, result_ids.size());ASSERT_EQ(3, result_ids[0]);@@ -609,9 +704,11 @@ TEST(PostingListTest, IntersectionBasics) {std::vector<posting_list_t::iterator_t> its;posting_list_t::result_iter_state_t iter_state;result_ids.clear();- posting_list_t::block_intersect(lists, its, iter_state, [&](auto id, auto& its){++ posting_t::block_intersector_t(raw_lists, iter_state, pool).intersect([&](auto id, auto& its, size_t index){result_ids.push_back(id);});+ASSERT_EQ(2, result_ids.size());ASSERT_EQ(3, result_ids[0]);ASSERT_EQ(20, result_ids[1]);@@ -627,12 +724,14 @@ TEST(PostingListTest, IntersectionBasics) {ASSERT_EQ(expected_ids[i], result_ids[i]);}- std::vector<posting_list_t::iterator_t> its3;- posting_list_t::result_iter_state_t iter_state3;+ posting_list_t::result_iter_state_t iter_state2;result_ids.clear();- posting_list_t::block_intersect(single_item_list, its3, iter_state3, [&](auto id, auto& its){+ raw_lists = {&p1};++ posting_t::block_intersector_t(raw_lists, iter_state2, pool).intersect([&](auto id, auto& its, size_t index){result_ids.push_back(id);});+ASSERT_EQ(4, result_ids.size());ASSERT_EQ(0, result_ids[0]);ASSERT_EQ(2, result_ids[1]);@@ -645,16 +744,18 @@ TEST(PostingListTest, IntersectionBasics) {posting_list_t::intersect(empty_list, result_ids);ASSERT_EQ(0, result_ids.size());- std::vector<posting_list_t::iterator_t> its4;- posting_list_t::result_iter_state_t iter_state4;+ posting_list_t::result_iter_state_t iter_state3;result_ids.clear();- posting_list_t::block_intersect(empty_list, its4, iter_state4, [&](auto id, auto& its){+ raw_lists.clear();++ posting_t::block_intersector_t(raw_lists, iter_state3, pool).intersect([&](auto id, auto& its, size_t index){result_ids.push_back(id);});+ASSERT_EQ(0, result_ids.size());}-TEST(PostingListTest, ResultsAndOffsetsBasics) {+TEST_F(PostingListTest, ResultsAndOffsetsBasics) {// NOTE: due to the way offsets1 are parsed, the actual positions are 1 less than the offset values stored// (to account for the special offset `0` which indicates last offsetstd::vector<uint32_t> offsets1 = {1, 2, 4};@@ -726,7 +827,7 @@ TEST(PostingListTest, ResultsAndOffsetsBasics) {*/}-TEST(PostingListTest, IntersectionSkipBlocks) {+TEST_F(PostingListTest, IntersectionSkipBlocks) {std::vector<uint32_t> offsets = {0, 1, 3};std::vector<posting_list_t*> lists;@@ -777,7 +878,7 @@ TEST(PostingListTest, IntersectionSkipBlocks) {delete [] final_results;}-TEST(PostingListTest, PostingListContainsAtleastOne) {+TEST_F(PostingListTest, PostingListContainsAtleastOne) {// when posting list is larger than target IDsposting_list_t p1(100);@@ -813,7 +914,7 @@ TEST(PostingListTest, PostingListContainsAtleastOne) {ASSERT_FALSE(p2.contains_atleast_one(&target_ids2[0], target_ids2.size()));}-TEST(PostingListTest, PostingListMergeAdjancentBlocks) {+TEST_F(PostingListTest, PostingListMergeAdjancentBlocks) {// when posting list is larger than target IDsposting_list_t p1(6);@@ -902,7 +1003,7 @@ TEST(PostingListTest, PostingListMergeAdjancentBlocks) {ASSERT_EQ(0, block2->offsets.getLength());}-TEST(PostingListTest, PostingListSplitBlock) {+TEST_F(PostingListTest, PostingListSplitBlock) {posting_list_t p1(6);for (size_t i = 0; i < 6; i++) {@@ -952,7 +1053,7 @@ TEST(PostingListTest, PostingListSplitBlock) {}}-TEST(PostingListTest, CompactPostingListUpsertAppends) {+TEST_F(PostingListTest, CompactPostingListUpsertAppends) {uint32_t ids[] = {0, 1000, 1002};uint32_t offset_index[] = {0, 3, 6};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1011,7 +1112,7 @@ TEST(PostingListTest, CompactPostingListUpsertAppends) {delete ((posting_list_t*)(obj));}-TEST(PostingListTest, CompactPostingListUpserts) {+TEST_F(PostingListTest, CompactPostingListUpserts) {uint32_t ids[] = {3, 1000, 1002};uint32_t offset_index[] = {0, 3, 6};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1055,7 +1156,7 @@ TEST(PostingListTest, CompactPostingListUpserts) {free(COMPACT_POSTING_PTR(obj));}-TEST(PostingListTest, CompactPostingListUpdateWithLessOffsets) {+TEST_F(PostingListTest, CompactPostingListUpdateWithLessOffsets) {uint32_t ids[] = {0, 1000, 1002};uint32_t offset_index[] = {0, 3, 6};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1103,7 +1204,7 @@ TEST(PostingListTest, CompactPostingListUpdateWithLessOffsets) {free(list);}-TEST(PostingListTest, CompactPostingListUpdateWithMoreOffsets) {+TEST_F(PostingListTest, CompactPostingListUpdateWithMoreOffsets) {uint32_t ids[] = {0, 1000, 1002};uint32_t offset_index[] = {0, 3, 6};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1152,7 +1253,7 @@ TEST(PostingListTest, CompactPostingListUpdateWithMoreOffsets) {free(list);}-TEST(PostingListTest, CompactPostingListErase) {+TEST_F(PostingListTest, CompactPostingListErase) {uint32_t ids[] = {0, 1000, 1002};uint32_t offset_index[] = {0, 3, 6};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1192,7 +1293,7 @@ TEST(PostingListTest, CompactPostingListErase) {free(list);}-TEST(PostingListTest, CompactPostingListContainsAtleastOne) {+TEST_F(PostingListTest, CompactPostingListContainsAtleastOne) {uint32_t ids[] = {5, 6, 7, 8};uint32_t offset_index[] = {0, 3, 6, 9};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1219,7 +1320,7 @@ TEST(PostingListTest, CompactPostingListContainsAtleastOne) {posting_t::destroy_list(obj);}-TEST(PostingListTest, CompactToFullPostingListConversion) {+TEST_F(PostingListTest, CompactToFullPostingListConversion) {uint32_t ids[] = {5, 6, 7, 8};uint32_t offset_index[] = {0, 3, 6, 9};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1231,7 +1332,7 @@ TEST(PostingListTest, CompactToFullPostingListConversion) {ASSERT_EQ(4, p1->num_ids());}-TEST(PostingListTest, BlockIntersectionOnMixedLists) {+TEST_F(PostingListTest, BlockIntersectionOnMixedLists) {uint32_t ids[] = {5, 6, 7, 8};uint32_t offset_index[] = {0, 3, 6, 9};uint32_t offsets[] = {0, 3, 4, 0, 3, 4, 0, 3, 4, 0, 3, 4};@@ -1248,12 +1349,13 @@ TEST(PostingListTest, BlockIntersectionOnMixedLists) {std::vector<void*> raw_posting_lists = {SET_COMPACT_POSTING(list1), &p1};posting_list_t::result_iter_state_t iter_state;- posting_t::block_intersector_t intersector(raw_posting_lists, iter_state);-std::vector<uint32_t> result_ids;- intersector.intersect([&](auto seq_id, auto& its) {++ posting_t::block_intersector_t(raw_posting_lists, iter_state, pool)+ .intersect([&](auto seq_id, auto& its, size_t index) {result_ids.push_back(seq_id);});+ASSERT_EQ(2, result_ids.size());ASSERT_EQ(5, result_ids[0]);ASSERT_EQ(8, result_ids[1]);@@ -1261,7 +1363,7 @@ TEST(PostingListTest, BlockIntersectionOnMixedLists) {free(list1);}-TEST(PostingListTest, DISABLED_Benchmark) {+TEST_F(PostingListTest, DISABLED_Benchmark) {std::vector<uint32_t> offsets = {0, 1, 3};posting_list_t pl(4096);sorted_array arr;@@ -1295,7 +1397,7 @@ TEST(PostingListTest, DISABLED_Benchmark) {LOG(INFO) << "Time taken for 5 sorted array updates: " << timeMicros;}-TEST(PostingListTest, DISABLED_BenchmarkIntersection) {+TEST_F(PostingListTest, DISABLED_BenchmarkIntersection) {std::vector<uint32_t> offsets = {0, 1, 3};time_t t;