Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle batch indexer for reference collections. (#1612)
Harpreet Sangar84573668a45f
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex 04afba58..e7e5621a 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -38,6 +38,15 @@ private:std::condition_variable cv;};+ struct refq_entry {+ uint64_t queue_id;+ uint64_t start_ts;++ refq_entry(uint64_t qid, uint64_t sts): queue_id(qid), start_ts(sts) {++ }+ };+HttpServer* server;Store* store;Store* meta_store;@@ -47,11 +56,15 @@ private:await_t* qmutuxes;std::vector<std::deque<uint64_t>> queues;+ std::unordered_map<std::string, std::unordered_set<std::string>> coll_to_references;+ await_t refq_wait;+ std::list<refq_entry> reference_q;+/* Variables to be serialized on snapshot /--------------------------------------------------------- */std::mutex mutex;- std::unordered_map<uint64_t, req_res_t> req_res_map;+ std::map<uint64_t, req_res_t> req_res_map;std::atomic<int64_t> queued_writes = 0;diff --git a/include/collection_manager.h b/include/collection_manager.hindex 380b2738..4400376f 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -78,8 +78,6 @@ private:std::atomic<bool>* quit;- BatchedIndexer* batch_indexer;-// All the references to a particular collection are stored until it is created.std::map<std::string, std::set<reference_pair>> referenced_in_backlog;@@ -103,7 +101,6 @@ public:static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";static constexpr const char* SYMLINK_PREFIX = "$SL";static constexpr const char* PRESET_PREFIX = "$PS";- static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";static CollectionManager & get_instance() {static CollectionManager instance;@@ -147,7 +144,7 @@ public:// PUBLICLY EXPOSED APIvoid init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,- const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer);+ const std::string & auth_key, std::atomic<bool>& quit);// only for tests!void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);@@ -238,4 +235,6 @@ public:static void _populate_referenced_ins(const std::string& collection_meta_json,std::map<std::string, spp::sparse_hash_map<std::string, std::string>>& referenced_ins);++ std::unordered_set<std::string> get_collection_references(const std::string& coll_name);};diff --git a/include/core_api.h b/include/core_api.hindex 08df91b9..780f7b3a 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -180,6 +180,10 @@ void get_collections_for_auth(std::map<std::string, std::string>& req_params, cobool is_doc_import_route(uint64_t route_hash);+bool is_coll_create_route(uint64_t route_hash);++bool is_drop_collection_route(uint64_t route_hash);+bool is_doc_write_route(uint64_t route_hash);bool is_doc_del_route(uint64_t route_hash);diff --git a/include/raft_server.h b/include/raft_server.hindex 61e2b761..7d0ab843 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -100,6 +100,7 @@ class ReplicationState : public braft::StateMachine {private:static constexpr const char* db_snapshot_name = "db_snapshot";static constexpr const char* analytics_db_snapshot_name = "analytics_db_snapshot";+ static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";mutable std::shared_mutex node_mutex;diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex 15f0a54e..44715329 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -2,6 +2,7 @@#include "core_api.h"#include "thread_local_vars.h"#include "cached_resource_stat.h"+#include "collection_manager.h"BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, const size_t num_threads,const Config& config, const std::atomic<bool>& skip_writes):@@ -13,6 +14,28 @@ BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_stoskip_index_iter_upper_bound = new rocksdb::Slice(skip_index_upper_bound_key);}+std::string get_ref_coll_names(const std::string& body, std::unordered_set<std::string>& referenced_collections) {+ std::string collection_name;+ auto const& obj = nlohmann::json::parse(body, nullptr, false);++ if (!obj.is_discarded() && obj.is_object() && obj.contains("name") && obj["name"].is_string() &&+ obj.contains("fields")) {+ collection_name = obj["name"];++ for (const auto &field: obj["fields"]) {+ if (!field.contains("reference")) {+ continue;+ }++ std::vector<std::string> split_result;+ StringUtils::split(field["reference"], split_result, ".");+ referenced_collections.insert(split_result.front());+ }+ }++ return collection_name;+}+void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {// Called by the raft write thread: goal is to quickly send the request to a queue and move on// NOTE: it's ok to access `req` and `res` in this function without synchronization@@ -48,6 +71,7 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::shbool is_old_serialized_request = (req->start_ts == 0);bool read_more_input = (req->_req != nullptr && req->_req->proceed_req);+ bool is_live_req = res->is_alive;if(req->last_chunk_aggregate) {//LOG(INFO) << "Last chunk for req_id: " << req->start_ts;@@ -56,19 +80,54 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh{const std::string& coll_name = get_collection_name(req);uint64_t queue_id = StringUtils::hash_wy(coll_name.c_str(), coll_name.size()) % num_threads;- req->body = "";+ req->params["collection"] = coll_name;{std::unique_lock lk2(mutex);req_res_map[req->start_ts].is_complete = true;}- {- std::lock_guard lk1(qmutuxes[queue_id].mcv);- queues[queue_id].emplace_back(req->start_ts);+ bool queue_write = true;++ if(!is_live_req) {+ if (is_coll_create_route(req->route_hash)) {+ // Save reference mapping to take care of ordering of requests+ std::unordered_set<std::string> referenced_collections;+ get_ref_coll_names(req->body, referenced_collections);+ if (!referenced_collections.empty()) {+ std::lock_guard lock(mutex);+ coll_to_references[coll_name] = std::move(referenced_collections);+ }+ } else if (is_drop_collection_route(req->route_hash)) {+ std::lock_guard lock(mutex);+ coll_to_references.erase(coll_name);+ } else {+ auto ref_colls_it = coll_to_references.find(coll_name);+ const auto& ref_collections = (ref_colls_it != coll_to_references.end()) ? ref_colls_it->second :+ CollectionManager::get_instance().get_collection_references(coll_name);++ if(!ref_collections.empty()) {+ // If this request involves a collection that references other collection(s), we have to wait+ // for the other collection(s) request(s) that arrived before this request to finish by pushing+ // this request onto a waiting queue.+ std::unique_lock lk(refq_wait.mcv);+ reference_q.emplace_back(queue_id, req->start_ts);+ lk.unlock();+ refq_wait.cv.notify_one();+ queue_write = false;+ }++ }}- qmutuxes[queue_id].cv.notify_one();+ req->body = "";++ if(queue_write) {+ std::unique_lock qlk(qmutuxes[queue_id].mcv);+ queues[queue_id].emplace_back(req->start_ts);+ qlk.unlock();+ qmutuxes[queue_id].cv.notify_one();+ }}// IMPORTANT: must not read `req` variables (except _req) henceforth to prevent data races with indexing thread@@ -267,11 +326,79 @@ void BatchedIndexer::run() {store->delete_range(req_key_prefix, req_key_prefix + StringUtils::serialize_uint32_t(UINT32_MAX));std::unique_lock lk(mutex);+req_res_map.erase(req_id);+ lk.unlock();+ refq_wait.cv.notify_one();}});}+ std::thread ref_sequence_thread([&]() {+ // Waits for dependent requests that are ahead to finish before pushing a request onto main indexing queue.+ while(!quit) {+ std::unique_lock ref_qlk(refq_wait.mcv);+ refq_wait.cv.wait(ref_qlk, [&] {+ return quit || !reference_q.empty();+ });++ if(quit) {+ break;+ }++ std::lock_guard lock(mutex);++ // We will iterate on the reference queue and check if there are any ongoing requests that have been+ // sent prior to this request.+ auto reference_q_it = reference_q.begin();+ while(reference_q_it != reference_q.end()) {+ bool found_ref_coll = false;++ auto req_res_it = req_res_map.find(reference_q_it->start_ts);+ if(req_res_it == req_res_map.end()) {+ reference_q_it = reference_q.erase(reference_q_it);+ continue;+ }++ auto const& coll_name = req_res_it->second.req->params["collection"];+ auto ref_colls_it = coll_to_references.find(coll_name);+ const auto& ref_collections = (ref_colls_it != coll_to_references.end()) ? ref_colls_it->second :+ CollectionManager::get_instance().get_collection_references(coll_name);++ if(ref_collections.empty()) {+ // This request is not dependent on any other request. Push this request onto main processing queue+ // and remove node from queue.+ std::unique_lock qlk(qmutuxes[reference_q_it->queue_id].mcv);+ queues[reference_q_it->queue_id].emplace_back(reference_q_it->start_ts);+ qlk.unlock();+ qmutuxes[reference_q_it->queue_id].cv.notify_one();+ reference_q_it = reference_q.erase(reference_q_it);+ continue;+ }++ for (auto it = req_res_map.begin(); it != req_res_it; it++) {+ auto const& req_coll_name = it->second.req->params["collection"];+ if(ref_collections.count(req_coll_name) != 0) {+ found_ref_coll = true;+ break;+ }+ }++ if(!found_ref_coll) {+ // All the dependent requests have been completed. Push this request onto main processing queue and+ // remove node from queue.+ std::unique_lock qlk(qmutuxes[reference_q_it->queue_id].mcv);+ queues[reference_q_it->queue_id].emplace_back(reference_q_it->start_ts);+ qlk.unlock();+ qmutuxes[reference_q_it->queue_id].cv.notify_one();+ reference_q_it = reference_q.erase(reference_q_it);+ } else {+ reference_q_it++;+ }+ }+ }+ });+uint64_t stuck_counter = 0;uint64_t prev_count = 0;@@ -341,10 +468,12 @@ void BatchedIndexer::run() {LOG(INFO) << "Notifying batch indexer threads about shutdown...";for(size_t i = 0; i < num_threads; i++) {await_t& queue_mutex = qmutuxes[i];- std::lock_guard<std::mutex> lk(queue_mutex.mcv);queue_mutex.cv.notify_one();}+ LOG(INFO) << "Notifying reference sequence thread about shutdown...";+ refq_wait.cv.notify_one();+ ref_sequence_thread.join();LOG(INFO) << "Batched indexer threadpool shutdown...";thread_pool->shutdown();@@ -418,6 +547,14 @@ void BatchedIndexer::serialize_state(nlohmann::json& state) {//LOG(INFO) << "req_key: " << req_key << ", next_chunk_index: " << kv.second.next_chunk_index;}+ state["reference_q"] = nlohmann::json::array();+ for(auto& ref_req: reference_q) {+ nlohmann::json ref_req_obj;+ ref_req_obj["queue_id"] = ref_req.queue_id;+ ref_req_obj["start_ts"] = ref_req.start_ts;+ state["reference_q"].push_back(ref_req_obj);+ }+LOG(INFO) << "Serialized " << num_reqs_stored << " in-flight requests for snapshot.";}@@ -461,6 +598,15 @@ void BatchedIndexer::load_state(const nlohmann::json& state) {num_reqs_restored++;}+ if(state.contains("reference_q")) {+ for(const auto& item: state["reference_q"].items()) {+ const nlohmann::json& ref_entry = item.value();+ reference_q.emplace_back(ref_entry["queue_id"], ref_entry["start_ts"]);+ }++ refq_wait.cv.notify_one();+ }+// need to sort on `start_ts` to preserve original order before notifying queuesfor(auto queue_id: queue_ids) {std::unique_lock lk(qmutuxes[queue_id].mcv);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex a38841b3..148185ee 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -210,8 +210,7 @@ void CollectionManager::add_to_collections(Collection* collection) {void CollectionManager::init(Store *store, ThreadPool* thread_pool,const float max_memory_ratio,const std::string & auth_key,- std::atomic<bool>& quit,- BatchedIndexer* batch_indexer) {+ std::atomic<bool>& quit) {std::unique_lock lock(mutex);this->store = store;@@ -219,14 +218,13 @@ void CollectionManager::init(Store *store, ThreadPool* thread_pool,this->bootstrap_auth_key = auth_key;this->max_memory_ratio = max_memory_ratio;this->quit = &quit;- this->batch_indexer = batch_indexer;}// used only in tests!void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,std::atomic<bool>& quit) {ThreadPool* thread_pool = new ThreadPool(8);- init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr);+ init(store, thread_pool, max_memory_ratio, auth_key, quit);}void CollectionManager::_populate_referenced_ins(const std::string& collection_meta_json,@@ -428,16 +426,6 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sloading_pool.shutdown();- LOG(INFO) << "Initializing batched indexer from snapshot state...";- if(batch_indexer != nullptr) {- std::string batched_indexer_state_str;- StoreStatus s = store->get(BATCHED_INDEXER_STATE_KEY, batched_indexer_state_str);- if(s == FOUND) {- nlohmann::json batch_indexer_state = nlohmann::json::parse(batched_indexer_state_str);- batch_indexer->load_state(batch_indexer_state);- }- }-return Option<bool>(true);}@@ -2394,4 +2382,20 @@ void CollectionManager::process_embedding_field_delete(const std::string& model_EmbedderManager::get_instance().delete_text_embedder(model_name);EmbedderManager::get_instance().delete_image_embedder(model_name);}-}\ No newline at end of file+}++std::unordered_set<std::string> CollectionManager::get_collection_references(const std::string& coll_name) {+ std::shared_lock lock(mutex);++ std::unordered_set<std::string> references;+ auto it = collections.find(coll_name);+ if (it == collections.end()) {+ return references;+ }++ for (const auto& item: it->second->get_reference_fields()) {+ const auto& ref_pair = item.second;+ references.insert(ref_pair.collection);+ }+ return references;+}diff --git a/src/core_api.cpp b/src/core_api.cppindex fce57150..36aabae4 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2241,6 +2241,18 @@ bool is_doc_import_route(uint64_t route_hash) {return found && (rpath->handler == post_import_documents);}+bool is_coll_create_route(uint64_t route_hash) {+ route_path* rpath;+ bool found = server->get_route(route_hash, &rpath);+ return found && (rpath->handler == post_create_collection);+}++bool is_drop_collection_route(uint64_t route_hash) {+ route_path* rpath;+ bool found = server->get_route(route_hash, &rpath);+ return found && (rpath->handler == del_drop_collection);+}+bool is_doc_write_route(uint64_t route_hash) {route_path* rpath;bool found = server->get_route(route_hash, &rpath);diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 7307b54f..af260eb0 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -563,7 +563,7 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clnlohmann::json batch_index_state;batched_indexer->serialize_state(batch_index_state);- store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());+ store->insert(BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());// we will delete all the skip indices in meta store and flush that DB// this will block writes, but should be pretty fast@@ -579,7 +579,7 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl}if(analytics_store) {- analytics_store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());+ analytics_store->insert(BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());rocksdb::Checkpoint* checkpoint2 = nullptr;status = analytics_store->create_check_point(&checkpoint2, analytics_db_snapshot_path);std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint2);@@ -626,6 +626,16 @@ int ReplicationState::init_db() {return 1;}+ if(batched_indexer != nullptr) {+ LOG(INFO) << "Initializing batched indexer from snapshot state...";+ std::string batched_indexer_state_str;+ StoreStatus s = store->get(BATCHED_INDEXER_STATE_KEY, batched_indexer_state_str);+ if(s == FOUND) {+ nlohmann::json batch_indexer_state = nlohmann::json::parse(batched_indexer_state_str);+ batched_indexer->load_state(batch_indexer_state);+ }+ }+return 0;}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex cef97f6c..37459cb5 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -436,7 +436,7 @@ int run_server(const Config & config, const std::string & version, void (*masterCollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),- config.get_api_key(), quit_raft_service, batch_indexer);+ config.get_api_key(), quit_raft_service);StopwordsManager& stopwordsManager = StopwordsManager::get_instance();stopwordsManager.init(&store);