Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Drop support for rocksdb based replication.

kishorencd3af2bff39f9
diff --git a/include/replicator.h b/include/replicator.h
deleted file mode 100644
index 09f361b5..00000000
--- a/include/replicator.h
+++ /dev/null
@@ -1,40 +0,0 @@
-#pragma once
-
-#include <string>
-#include "http_data.h"
-#include "store.h"
-
-static constexpr const char* REPLICATION_EVENT_MSG = "replication_event";
-
-struct ReplicationEvent {
- std::string type;
- std::string key;
- std::string value;
-
- ReplicationEvent(const std::string& type, const uint32_t collection_id,
- const std::string& key, const std::string& value): type(type), key(key), value(value) {
-
- }
-};
-
-class IterateBatchHandler: public rocksdb::WriteBatch::Handler {
-private:
- http_message_dispatcher* message_dispatcher;
-public:
- IterateBatchHandler(http_message_dispatcher* message_dispatcher): message_dispatcher(message_dispatcher) {
-
- }
-
- void Put(const rocksdb::Slice& key, const rocksdb::Slice& value);
-
- void Delete(const rocksdb::Slice& key);
-
- void Merge(const rocksdb::Slice& key, const rocksdb::Slice& value);
-};
-
-class Replicator {
-public:
- static void start(http_message_dispatcher* message_dispatcher, const std::string & master_host_port, const std::string & api_key, Store& store);
-
- static bool on_replication_event(void *data);
-};
diff --git a/include/typesense_server_utils.h b/include/typesense_server_utils.h
index ea114220..4b0af691 100644
--- a/include/typesense_server_utils.h
+++ b/include/typesense_server_utils.h
@@ -7,7 +7,6 @@
#include "logger.h"
#include "store.h"
#include "collection_manager.h"
-#include "replicator.h"
#include <csignal>
#include <sys/types.h>
#include <sys/stat.h>
@@ -27,4 +26,4 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv);
int init_logger(Config & config, const std::string & server_version);
int run_server(const Config & config, const std::string & version,
- void (*master_server_routes)(), void (*replica_server_routes)());
+ void (*master_server_routes)());
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 28e23e00..800d27b3 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -72,5 +72,5 @@ int main(int argc, char **argv) {
return ret_code;
}
- return run_server(config, TYPESENSE_VERSION, &master_server_routes, &replica_server_routes);
+ return run_server(config, TYPESENSE_VERSION, &master_server_routes);
}
\ No newline at end of file
diff --git a/src/replicator.cpp b/src/replicator.cpp
deleted file mode 100644
index 1cc8d05d..00000000
--- a/src/replicator.cpp
+++ /dev/null
@@ -1,213 +0,0 @@
-#include "replicator.h"
-#include <iostream>
-#include "http_client.h"
-#include "collection_manager.h"
-#include "collection.h"
-#include "string_utils.h"
-#include <json.hpp>
-#include <thread>
-#include "logger.h"
-
-
-void IterateBatchHandler::Put(const rocksdb::Slice& key, const rocksdb::Slice& value) {
- std::vector<std::string> parts;
- StringUtils::split(key.ToString(), parts, "_");
-
- std::string val = value.ToString() + "\n";
-
- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {
- // nothing to do here as this is called only when a new collection is created and it's always "0"
- }
-
- if(parts.size() == 1 && parts[0] == CollectionManager::NEXT_COLLECTION_ID_KEY) {
- ReplicationEvent* replication_event = new ReplicationEvent("UPDATE_NEXT_COLLECTION_ID", 0,
- key.ToString(), value.ToString());
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-
- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {
- ReplicationEvent* replication_event = new ReplicationEvent("ADD_COLLECTION_META", 0,
- key.ToString(), value.ToString());
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-
- if(parts.size() == 3 && parts[1] == Collection::SEQ_ID_PREFIX) {
- ReplicationEvent* replication_event = new ReplicationEvent("ADD_DOCUMENT", std::stoi(parts[0]),
- key.ToString(), value.ToString());
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-
- if(parts.size() >= 2 && parts[0] == CollectionManager::SYMLINK_PREFIX) {
- ReplicationEvent* replication_event = new ReplicationEvent("ADD_SYMLINK", 0,
- key.ToString(), value.ToString());
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-}
-
-void IterateBatchHandler::Delete(const rocksdb::Slice& key) {
- std::vector<std::string> parts;
- StringUtils::split(key.ToString(), parts, "_");
-
- if(parts.size() == 3 && parts[1] == Collection::DOC_ID_PREFIX) {
- ReplicationEvent* replication_event = new ReplicationEvent("REMOVE_DOCUMENT", 0, key.ToString(), "");
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-
- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {
- ReplicationEvent* replication_event = new ReplicationEvent("DROP_COLLECTION", 0, key.ToString(), "");
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-
- if(parts.size() >= 2 && parts[0] == CollectionManager::SYMLINK_PREFIX) {
- ReplicationEvent* replication_event = new ReplicationEvent("REMOVE_SYMLINK", 0,
- key.ToString(), "");
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-
-}
-
-void IterateBatchHandler::Merge(const rocksdb::Slice& key, const rocksdb::Slice& value) {
- std::vector<std::string> parts;
- StringUtils::split(key.ToString(), parts, "_");
-
- if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {
- ReplicationEvent* replication_event = new ReplicationEvent("INCR_COLLECTION_NEXT_SEQ", 0,
- key.ToString(), value.ToString());
- message_dispatcher->send_message(REPLICATION_EVENT_MSG, replication_event);
- }
-}
-
-void Replicator::start(http_message_dispatcher* message_dispatcher, const std::string & master_host_port,
- const std::string & api_key, Store& store) {
- size_t total_runs = 0;
-
- while(true) {
- IterateBatchHandler handler(message_dispatcher);
- uint64_t latest_seq_num = store.get_latest_seq_number();
-
- if(total_runs++ % 20 == 0) {
- // roughly every 60 seconds
- LOG(INFO) << "Replica's latest sequence number: " << latest_seq_num;
- }
-
- std::string url = master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1);
-
- std::string json_response;
- long status_code = HttpClient::get_response(url, json_response, 15000);
-
- if(status_code == 200) {
- nlohmann::json json_content = nlohmann::json::parse(json_response);
- nlohmann::json updates = json_content["updates"];
-
- // first write to memory
- for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {
- const std::string update_decoded = StringUtils::base64_decode(*update);
- rocksdb::WriteBatch write_batch(update_decoded);
- write_batch.Iterate(&handler);
- }
-
- // now write to store
- for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {
- const std::string update_decoded = StringUtils::base64_decode(*update);
- rocksdb::WriteBatch write_batch(update_decoded);
- store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);
- }
-
- if(updates.size() > 0) {
- LOG(INFO) << "Replica has consumed " << store.get_latest_seq_number() << "/"
- << json_content["latest_seq_num"] << " updates from master.";
- }
-
- } else {
- LOG(ERROR) << "Replication error while fetching records from master, status_code=" << status_code
- << ", replica's latest sequence number: " << latest_seq_num;
-
- if(status_code != 0) {
- LOG(ERROR) << json_response;
- }
- }
-
- std::this_thread::sleep_for(std::chrono::milliseconds(3000));
- }
-}
-
-bool Replicator::on_replication_event(void *data) {
- ReplicationEvent* replication_event = static_cast<ReplicationEvent*>(data);
-
- if(replication_event->type == "UPDATE_NEXT_COLLECTION_ID") {
- CollectionManager & collection_manager = CollectionManager::get_instance();
- collection_manager.set_next_collection_id(std::stoi(replication_event->value));
- }
-
- if(replication_event->type == "ADD_COLLECTION_META") {
- nlohmann::json collection_meta;
- try {
- collection_meta = nlohmann::json::parse(replication_event->value);
- } catch(...) {
- LOG(ERROR) << "Failed to parse collection meta JSON.";
- LOG(ERROR) << "Replication event value: " << replication_event->value;
- delete replication_event;
- exit(1);
- }
-
- CollectionManager & collection_manager = CollectionManager::get_instance();
- Collection* collection = collection_manager.init_collection(collection_meta, 0);
- collection_manager.add_to_collections(collection);
- }
-
- if(replication_event->type == "ADD_DOCUMENT") {
- CollectionManager & collection_manager = CollectionManager::get_instance();
- std::vector<std::string> parts;
- StringUtils::split(replication_event->key, parts, "_"); // collection_id, seq_id_prefix, seq_id
- Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));
- nlohmann::json document = nlohmann::json::parse(replication_event->value);
-
- uint32_t seq_id = Collection::get_seq_id_from_key(replication_event->key);
- collection->index_in_memory(document, seq_id);
- }
-
- if(replication_event->type == "ADD_SYMLINK") {
- CollectionManager & collection_manager = CollectionManager::get_instance();
- std::vector<std::string> parts;
- std::string symlink_prefix_key = std::string(CollectionManager::SYMLINK_PREFIX) + "_";
- StringUtils::split(replication_event->key, parts, symlink_prefix_key); // symlink_prefix, symlink_name
- std::string & symlink_name = parts[0];
- spp::sparse_hash_map<std::string, std::string> & simlinks = collection_manager.get_symlinks();
- simlinks[symlink_name] = replication_event->value;
- }
-
- if(replication_event->type == "INCR_COLLECTION_NEXT_SEQ") {
- CollectionManager & collection_manager = CollectionManager::get_instance();
- const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);
- Collection* collection = collection_manager.get_collection(collection_name);
- collection->increment_next_seq_id_field();
- }
-
- if(replication_event->type == "REMOVE_DOCUMENT") {
- std::vector<std::string> parts;
- StringUtils::split(replication_event->key, parts, "_"); // collection_id, doc_id_prefix, doc_id
- CollectionManager & collection_manager = CollectionManager::get_instance();
- Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));
- collection->remove(parts[2], false);
- }
-
- if(replication_event->type == "REMOVE_SYMLINK") {
- CollectionManager & collection_manager = CollectionManager::get_instance();
- std::vector<std::string> parts;
- std::string symlink_prefix_key = std::string(CollectionManager::SYMLINK_PREFIX) + "_";
- StringUtils::split(replication_event->key, parts, symlink_prefix_key); // symlink_prefix, symlink_name
- std::string & symlink_name = parts[0];
- collection_manager.delete_symlink(symlink_name);
- }
-
- if(replication_event->type == "DROP_COLLECTION") {
- CollectionManager & collection_manager = CollectionManager::get_instance();
- // <collection_meta_prefix>_<collection_name>
- const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_META_PREFIX)+1);
- collection_manager.drop_collection(collection_name, false);
- }
-
- delete replication_event;
-
- return true;
-}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index d8a8280e..fca636b1 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -244,8 +244,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
return 0;
}
-int run_server(const Config & config, const std::string & version,
- void (*master_server_routes)(), void (*replica_server_routes)()) {
+int run_server(const Config & config, const std::string & version, void (*master_server_routes)()) {
LOG(INFO) << "Starting Typesense " << version << std::flush;
quit_raft_service = false;
@@ -256,6 +255,12 @@ int run_server(const Config & config, const std::string & version,
return 1;
}
+ if(!config.get_master().empty()) {
+ LOG(ERROR) << "The --master option has been deprecated. Please use clustering for high availability. "
+ << "Look for the --peers configuration in the documentation.";
+ return 1;
+ }
+
std::string data_dir = config.get_data_dir();
std::string db_dir = config.get_data_dir() + "/db";
std::string state_dir = config.get_data_dir() + "/state";
@@ -301,7 +306,6 @@ int run_server(const Config & config, const std::string & version,
server->set_auth_handler(handle_authentication);
server->on(SEND_RESPONSE_MSG, on_send_response);
- server->on(REPLICATION_EVENT_MSG, Replicator::on_replication_event);
server->on(ReplicationState::REPLICATION_MSG, async_write_request);
// first we start the raft service
@@ -321,28 +325,7 @@ int run_server(const Config & config, const std::string & version,
std::this_thread::sleep_for(std::chrono::milliseconds(500));
}
- if(config.get_master().empty()) {
- master_server_routes();
- } else {
- replica_server_routes();
-
- const std::string & master_host_port = config.get_master();
- std::vector<std::string> parts;
- StringUtils::split(master_host_port, parts, ":");
- if(parts.size() != 3) {
- LOG(ERROR) << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>";
- return 1;
- }
-
- LOG(INFO) << "Typesense is starting as a read-only replica... Master URL is: " << master_host_port;
- LOG(INFO) << "Spawning replication thread...";
-
- std::thread replication_thread([&store, &config]() {
- Replicator::start(server->get_message_dispatcher(), config.get_master(), config.get_api_key(), store);
- });
-
- replication_thread.detach();
- }
+ master_server_routes();
int ret_code = server->run(&replication_state);