Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Drop support for rocksdb based replication.
kishorencd3af2bff39f9
diff --git a/include/replicator.h b/include/replicator.hdeleted file mode 100644index 09f361b5..00000000--- a/include/replicator.h+++ /dev/null@@ -1,40 +0,0 @@-#pragma once--#include <string>-#include "http_data.h"-#include "store.h"--static constexpr const char* REPLICATION_EVENT_MSG = "replication_event";--struct ReplicationEvent {- std::string type;- std::string key;- std::string value;-- ReplicationEvent(const std::string& type, const uint32_t collection_id,- const std::string& key, const std::string& value): type(type), key(key), value(value) {-- }-};--class IterateBatchHandler: public rocksdb::WriteBatch::Handler {-private:- http_message_dispatcher* message_dispatcher;-public:- IterateBatchHandler(http_message_dispatcher* message_dispatcher): message_dispatcher(message_dispatcher) {-- }-- void Put(const rocksdb::Slice& key, const rocksdb::Slice& value);-- void Delete(const rocksdb::Slice& key);-- void Merge(const rocksdb::Slice& key, const rocksdb::Slice& value);-};--class Replicator {-public:- static void start(http_message_dispatcher* message_dispatcher, const std::string & master_host_port, const std::string & api_key, Store& store);-- static bool on_replication_event(void *data);-};diff --git a/include/typesense_server_utils.h b/include/typesense_server_utils.hindex ea114220..4b0af691 100644--- a/include/typesense_server_utils.h+++ b/include/typesense_server_utils.h@@ -7,7 +7,6 @@#include "logger.h"#include "store.h"#include "collection_manager.h"-#include "replicator.h"#include <csignal>#include <sys/types.h>#include <sys/stat.h>@@ -27,4 +26,4 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv);int init_logger(Config & config, const std::string & server_version);int run_server(const Config & config, const std::string & version,- void (*master_server_routes)(), void (*replica_server_routes)());+ void (*master_server_routes)());diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 28e23e00..800d27b3 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -72,5 +72,5 @@ int main(int argc, char **argv) {return ret_code;}- return run_server(config, TYPESENSE_VERSION, &master_server_routes, &replica_server_routes);+ return run_server(config, TYPESENSE_VERSION, &master_server_routes);}\ No newline at end of filediff --git a/src/replicator.cpp b/src/replicator.cppdeleted file mode 100644index 1cc8d05d..00000000--- a/src/replicator.cpp+++ /dev/null@@ -1,213 +0,0 @@-#include "replicator.h"-#include <iostream>-#include "http_client.h"-#include "collection_manager.h"-#include "collection.h"-#include "string_utils.h"-#include <json.hpp>-#include <thread>-#include "logger.h"---void IterateBatchHandler::Put(const rocksdb::Slice& key, const rocksdb::Slice& value) {- std::vector<std::string> parts;- StringUtils::split(key.ToString(), parts, "_");-- std::string val = value.ToString() + "\n";-- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {- // nothing to do here as this is called only when a new collection is created and it's always "0"- }-- if(parts.size() == 1 && parts[0] == CollectionManager::NEXT_COLLECTION_ID_KEY) {- ReplicationEvent* replication_event = new ReplicationEvent("UPDATE_NEXT_COLLECTION_ID", 0,- key.ToString(), value.ToString());- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }-- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {- ReplicationEvent* replication_event = new ReplicationEvent("ADD_COLLECTION_META", 0,- key.ToString(), value.ToString());- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }-- if(parts.size() == 3 && parts[1] == Collection::SEQ_ID_PREFIX) {- ReplicationEvent* replication_event = new ReplicationEvent("ADD_DOCUMENT", std::stoi(parts[0]),- key.ToString(), value.ToString());- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }-- if(parts.size() >= 2 && parts[0] == CollectionManager::SYMLINK_PREFIX) {- ReplicationEvent* replication_event = new ReplicationEvent("ADD_SYMLINK", 0,- key.ToString(), value.ToString());- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }-}--void IterateBatchHandler::Delete(const rocksdb::Slice& key) {- std::vector<std::string> parts;- StringUtils::split(key.ToString(), parts, "_");-- if(parts.size() == 3 && parts[1] == Collection::DOC_ID_PREFIX) {- ReplicationEvent* replication_event = new ReplicationEvent("REMOVE_DOCUMENT", 0, key.ToString(), "");- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }-- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {- ReplicationEvent* replication_event = new ReplicationEvent("DROP_COLLECTION", 0, key.ToString(), "");- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }-- if(parts.size() >= 2 && parts[0] == CollectionManager::SYMLINK_PREFIX) {- ReplicationEvent* replication_event = new ReplicationEvent("REMOVE_SYMLINK", 0,- key.ToString(), "");- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }--}--void IterateBatchHandler::Merge(const rocksdb::Slice& key, const rocksdb::Slice& value) {- std::vector<std::string> parts;- StringUtils::split(key.ToString(), parts, "_");-- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {- ReplicationEvent* replication_event = new ReplicationEvent("INCR_COLLECTION_NEXT_SEQ", 0,- key.ToString(), value.ToString());- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);- }-}--void Replicator::start(http_message_dispatcher* message_dispatcher, const std::string & master_host_port,- const std::string & api_key, Store& store) {- size_t total_runs = 0;-- while(true) {- IterateBatchHandler handler(message_dispatcher);- uint64_t latest_seq_num = store.get_latest_seq_number();-- if(total_runs++ % 20 == 0) {- // roughly every 60 seconds- LOG(INFO) << "Replica's latest sequence number: " << latest_seq_num;- }-- std::string url = master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1);-- std::string json_response;- long status_code = HttpClient::get_response(url, json_response, 15000);-- if(status_code == 200) {- nlohmann::json json_content = nlohmann::json::parse(json_response);- nlohmann::json updates = json_content["updates"];-- // first write to memory- for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {- const std::string update_decoded = StringUtils::base64_decode(*update);- rocksdb::WriteBatch write_batch(update_decoded);- write_batch.Iterate(&handler);- }-- // now write to store- for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {- const std::string update_decoded = StringUtils::base64_decode(*update);- rocksdb::WriteBatch write_batch(update_decoded);- store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);- }-- if(updates.size() > 0) {- LOG(INFO) << "Replica has consumed " << store.get_latest_seq_number() << "/"- << json_content["latest_seq_num"] << " updates from master.";- }-- } else {- LOG(ERROR) << "Replication error while fetching records from master, status_code=" << status_code- << ", replica's latest sequence number: " << latest_seq_num;-- if(status_code != 0) {- LOG(ERROR) << json_response;- }- }-- std::this_thread::sleep_for(std::chrono::milliseconds(3000));- }-}--bool Replicator::on_replication_event(void *data) {- ReplicationEvent* replication_event = static_cast<ReplicationEvent*>(data);-- if(replication_event->type == "UPDATE_NEXT_COLLECTION_ID") {- CollectionManager & collection_manager = CollectionManager::get_instance();- collection_manager.set_next_collection_id(std::stoi(replication_event->value));- }-- if(replication_event->type == "ADD_COLLECTION_META") {- nlohmann::json collection_meta;- try {- collection_meta = nlohmann::json::parse(replication_event->value);- } catch(...) {- LOG(ERROR) << "Failed to parse collection meta JSON.";- LOG(ERROR) << "Replication event value: " << replication_event->value;- delete replication_event;- exit(1);- }-- CollectionManager & collection_manager = CollectionManager::get_instance();- Collection* collection = collection_manager.init_collection(collection_meta, 0);- collection_manager.add_to_collections(collection);- }-- if(replication_event->type == "ADD_DOCUMENT") {- CollectionManager & collection_manager = CollectionManager::get_instance();- std::vector<std::string> parts;- StringUtils::split(replication_event->key, parts, "_"); // collection_id, seq_id_prefix, seq_id- Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));- nlohmann::json document = nlohmann::json::parse(replication_event->value);-- uint32_t seq_id = Collection::get_seq_id_from_key(replication_event->key);- collection->index_in_memory(document, seq_id);- }-- if(replication_event->type == "ADD_SYMLINK") {- CollectionManager & collection_manager = CollectionManager::get_instance();- std::vector<std::string> parts;- std::string symlink_prefix_key = std::string(CollectionManager::SYMLINK_PREFIX) + "_";- StringUtils::split(replication_event->key, parts, symlink_prefix_key); // symlink_prefix, symlink_name- std::string & symlink_name = parts[0];- spp::sparse_hash_map<std::string, std::string> & simlinks = collection_manager.get_symlinks();- simlinks[symlink_name] = replication_event->value;- }-- if(replication_event->type == "INCR_COLLECTION_NEXT_SEQ") {- CollectionManager & collection_manager = CollectionManager::get_instance();- const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);- Collection* collection = collection_manager.get_collection(collection_name);- collection->increment_next_seq_id_field();- }-- if(replication_event->type == "REMOVE_DOCUMENT") {- std::vector<std::string> parts;- StringUtils::split(replication_event->key, parts, "_"); // collection_id, doc_id_prefix, doc_id- CollectionManager & collection_manager = CollectionManager::get_instance();- Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));- collection->remove(parts[2], false);- }-- if(replication_event->type == "REMOVE_SYMLINK") {- CollectionManager & collection_manager = CollectionManager::get_instance();- std::vector<std::string> parts;- std::string symlink_prefix_key = std::string(CollectionManager::SYMLINK_PREFIX) + "_";- StringUtils::split(replication_event->key, parts, symlink_prefix_key); // symlink_prefix, symlink_name- std::string & symlink_name = parts[0];- collection_manager.delete_symlink(symlink_name);- }-- if(replication_event->type == "DROP_COLLECTION") {- CollectionManager & collection_manager = CollectionManager::get_instance();- // <collection_meta_prefix>_<collection_name>- const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_META_PREFIX)+1);- collection_manager.drop_collection(collection_name, false);- }-- delete replication_event;-- return true;-}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex d8a8280e..fca636b1 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -244,8 +244,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& streturn 0;}-int run_server(const Config & config, const std::string & version,- void (*master_server_routes)(), void (*replica_server_routes)()) {+int run_server(const Config & config, const std::string & version, void (*master_server_routes)()) {LOG(INFO) << "Starting Typesense " << version << std::flush;quit_raft_service = false;@@ -256,6 +255,12 @@ int run_server(const Config & config, const std::string & version,return 1;}+ if(!config.get_master().empty()) {+ LOG(ERROR) << "The --master option has been deprecated. Please use clustering for high availability. "+ << "Look for the --peers configuration in the documentation.";+ return 1;+ }+std::string data_dir = config.get_data_dir();std::string db_dir = config.get_data_dir() + "/db";std::string state_dir = config.get_data_dir() + "/state";@@ -301,7 +306,6 @@ int run_server(const Config & config, const std::string & version,server->set_auth_handler(handle_authentication);server->on(SEND_RESPONSE_MSG, on_send_response);- server->on(REPLICATION_EVENT_MSG, Replicator::on_replication_event);server->on(ReplicationState::REPLICATION_MSG, async_write_request);// first we start the raft service@@ -321,28 +325,7 @@ int run_server(const Config & config, const std::string & version,std::this_thread::sleep_for(std::chrono::milliseconds(500));}- if(config.get_master().empty()) {- master_server_routes();- } else {- replica_server_routes();-- const std::string & master_host_port = config.get_master();- std::vector<std::string> parts;- StringUtils::split(master_host_port, parts, ":");- if(parts.size() != 3) {- LOG(ERROR) << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>";- return 1;- }-- LOG(INFO) << "Typesense is starting as a read-only replica... Master URL is: " << master_host_port;- LOG(INFO) << "Spawning replication thread...";-- std::thread replication_thread([&store, &config]() {- Replicator::start(server->get_message_dispatcher(), config.get_master(), config.get_api_key(), store);- });-- replication_thread.detach();- }+ master_server_routes();int ret_code = server->run(&replication_state);