Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow customization of peering address.

kishorenc8829c5aebc09
diff --git a/include/config.h b/include/config.h
index 73ab9046..a2e35f5b 100644
--- a/include/config.h
+++ b/include/config.h
@@ -13,11 +13,12 @@ private:
std::string api_key;
std::string search_only_api_key;
- std::string listen_address;
- uint32_t listen_port;
+ std::string api_address;
+ uint32_t api_port;
- uint32_t raft_port;
- std::string raft_peers;
+ std::string peering_address;
+ uint32_t peering_port;
+ std::string peers;
std::string master;
@@ -34,9 +35,9 @@ private:
public:
Config() {
- this->listen_address = "0.0.0.0";
- this->listen_port = 8108;
- this->raft_port = 8107;
+ this->api_address = "0.0.0.0";
+ this->api_port = 8108;
+ this->peering_port = 8107;
this->enable_cors = false;
this->indices_per_collection = 4;
}
@@ -60,11 +61,11 @@ public:
}
void set_listen_address(const std::string & listen_address) {
- this->listen_address = listen_address;
+ this->api_address = listen_address;
}
void set_listen_port(int listen_port) {
- this->listen_port = listen_port;
+ this->api_port = listen_port;
}
void set_master(const std::string & master) {
@@ -88,11 +89,11 @@ public:
}
void set_raft_port(int raft_port) {
- this->raft_port = raft_port;
+ this->peering_port = raft_port;
}
void set_raft_peers(const std::string & raft_peers) {
- this->raft_peers = raft_peers;
+ this->peers = raft_peers;
}
// getters
@@ -113,12 +114,12 @@ public:
return this->search_only_api_key;
}
- std::string get_listen_address() const {
- return this->listen_address;
+ std::string get_api_address() const {
+ return this->api_address;
}
- int get_listen_port() const {
- return this->listen_port;
+ int get_api_port() const {
+ return this->api_port;
}
std::string get_master() const {
@@ -145,12 +146,16 @@ public:
return indices_per_collection;
}
- int get_raft_port() const {
- return this->raft_port;
+ std::string get_peering_address() const {
+ return this->peering_address;
}
- std::string get_raft_peers() const {
- return this->raft_peers;
+ int get_peering_port() const {
+ return this->peering_port;
+ }
+
+ std::string get_peers() const {
+ return this->peers;
}
// loaders
@@ -171,18 +176,30 @@ public:
this->search_only_api_key = get_env("TYPESENSE_SEARCH_ONLY_API_KEY");
if(!get_env("TYPESENSE_LISTEN_ADDRESS").empty()) {
- this->listen_address = get_env("TYPESENSE_LISTEN_ADDRESS");
+ this->api_address = get_env("TYPESENSE_LISTEN_ADDRESS");
}
if(!get_env("TYPESENSE_LISTEN_PORT").empty()) {
- this->listen_port = std::stoi(get_env("TYPESENSE_LISTEN_PORT"));
+ this->api_port = std::stoi(get_env("TYPESENSE_LISTEN_PORT"));
+ }
+
+ if(!get_env("TYPESENSE_API_ADDRESS").empty()) {
+ this->api_address = get_env("TYPESENSE_LISTEN_ADDRESS");
+ }
+
+ if(!get_env("TYPESENSE_API_PORT").empty()) {
+ this->api_port = std::stoi(get_env("TYPESENSE_API_PORT"));
+ }
+
+ if(!get_env("TYPESENSE_PEERING_ADDRESS").empty()) {
+ this->api_address = get_env("TYPESENSE_PEERING_ADDRESS");
}
- if(!get_env("TYPESENSE_RAFT_PORT").empty()) {
- this->raft_port = std::stoi(get_env("TYPESENSE_RAFT_PORT"));
+ if(!get_env("TYPESENSE_PEERING_PORT").empty()) {
+ this->peering_port = std::stoi(get_env("TYPESENSE_PEERING_PORT"));
}
- this->raft_peers = get_env("TYPESENSE_RAFT_PEERS");
+ this->peers = get_env("TYPESENSE_PEERS");
this->master = get_env("TYPESENSE_MASTER");
this->ssl_certificate = get_env("TYPESENSE_SSL_CERTIFICATE");
@@ -229,7 +246,11 @@ public:
}
if(reader.Exists("server", "listen-address")) {
- this->listen_address = reader.Get("server", "listen-address", "");
+ this->api_address = reader.Get("server", "listen-address", "");
+ }
+
+ if(reader.Exists("server", "api-address")) {
+ this->api_address = reader.Get("server", "api-address", "");
}
if(reader.Exists("server", "master")) {
@@ -245,19 +266,27 @@ public:
}
if(reader.Exists("server", "listen-port")) {
- this->listen_port = reader.GetInteger("server", "listen-port", 8108);
+ this->api_port = reader.GetInteger("server", "listen-port", 8108);
+ }
+
+ if(reader.Exists("server", "api-port")) {
+ this->api_port = reader.GetInteger("server", "api-port", 8108);
}
if(reader.Exists("server", "enable-cors")) {
this->enable_cors = reader.GetBoolean("server", "enable-cors", false);
}
+ if(reader.Exists("server", "peering-address")) {
+ this->peering_address = reader.Get("server", "peering-address", "");
+ }
+
if(reader.Exists("server", "peering-port")) {
- this->raft_port = reader.GetInteger("server", "raft-port", 8107);
+ this->peering_port = reader.GetInteger("server", "peering-port", 8107);
}
if(reader.Exists("server", "peers")) {
- this->raft_peers = reader.Get("server", "peers", "");
+ this->peers = reader.Get("server", "peers", "");
}
}
@@ -279,7 +308,11 @@ public:
}
if(options.exist("listen-address")) {
- this->listen_address = options.get<std::string>("listen-address");
+ this->api_address = options.get<std::string>("listen-address");
+ }
+
+ if(options.exist("api-address")) {
+ this->api_address = options.get<std::string>("api-address");
}
if(options.exist("master")) {
@@ -295,19 +328,27 @@ public:
}
if(options.exist("listen-port")) {
- this->listen_port = options.get<uint32_t>("listen-port");
+ this->api_port = options.get<uint32_t>("listen-port");
+ }
+
+ if(options.exist("api-port")) {
+ this->api_port = options.get<uint32_t>("api-port");
}
if(options.exist("enable-cors")) {
this->enable_cors = options.exist("enable-cors");
}
+ if(options.exist("peering-address")) {
+ this->peering_address = options.get<std::string>("peering-address");
+ }
+
if(options.exist("peering-port")) {
- this->raft_port = options.get<uint32_t>("peering-port");
+ this->peering_port = options.get<uint32_t>("peering-port");
}
if(options.exist("peers")) {
- this->raft_peers = options.get<std::string>("peers");
+ this->peers = options.get<std::string>("peers");
}
}
diff --git a/include/raft_server.h b/include/raft_server.h
index 914ad6fe..9f3b1f0a 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -103,7 +103,8 @@ public:
}
// Starts this node
- int start(int api_port, int raft_port, int election_timeout_ms, int snapshot_interval_s,
+ int start(const std::string & peering_address, int peering_port, int api_port,
+ int election_timeout_ms, int snapshot_interval_s,
const std::string & raft_dir, const std::string & peers);
// Generic write method for synchronizing all writes
@@ -191,7 +192,7 @@ private:
}
void on_error(const ::braft::Error& e) {
- LOG(ERROR) << "Met raft error " << e;
+ LOG(ERROR) << "Met peering error " << e;
}
void on_configuration_committed(const ::braft::Configuration& conf) {
diff --git a/include/store.h b/include/store.h
index c0de1cc4..9ae74f55 100644
--- a/include/store.h
+++ b/include/store.h
@@ -90,7 +90,7 @@ public:
LOG(ERROR) << "It seems like the data directory " << state_dir_path << " is already being used by "
<< "another Typesense server. ";
LOG(ERROR) << "If you are SURE that this is not the case, delete the LOCK file "
- << "in the data directory and try again.";
+ << "in the data db directory and try again.";
}
}
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 68fcf98b..739131f5 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -17,10 +17,21 @@ void ReplicationClosure::Run() {
// State machine implementation
-int ReplicationState::start(const int api_port, int raft_port, int election_timeout_ms, int snapshot_interval_s,
+int ReplicationState::start(const std::string & peering_address, int peering_port, const int api_port,
+ int election_timeout_ms, int snapshot_interval_s,
const std::string & raft_dir, const std::string & peers) {
+ butil::ip_t peering_ip;
+ if(!peering_address.empty()) {
+ int ip_conv_status = butil::str2ip(peering_address.c_str(), &peering_ip);
+ if(ip_conv_status != 0) {
+ LOG(ERROR) << "Failed to parse peering address `" << peering_address << "`";
+ return -1;
+ }
+ } else {
+ peering_ip = butil::my_ip();
+ }
- butil::EndPoint addr(butil::my_ip(), raft_port);
+ butil::EndPoint addr(peering_ip, peering_port);
braft::NodeOptions node_options;
std::string actual_peers = peers;
@@ -28,11 +39,11 @@ int ReplicationState::start(const int api_port, int raft_port, int election_time
if(actual_peers.empty()) {
char str[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &(addr.ip.s_addr), str, INET_ADDRSTRLEN);
- actual_peers = std::string(str) + ":" + std::to_string(raft_port) + ":" + std::to_string(api_port);
+ actual_peers = std::string(str) + ":" + std::to_string(peering_port) + ":" + std::to_string(api_port);
}
if(node_options.initial_conf.parse_from(actual_peers) != 0) {
- LOG(ERROR) << "Fail to parse peer configuration `" << peers << "`";
+ LOG(ERROR) << "Failed to parse peer configuration `" << peers << "`";
return -1;
}
@@ -73,7 +84,7 @@ int ReplicationState::start(const int api_port, int raft_port, int election_time
}
if (node->init(node_options) != 0) {
- LOG(ERROR) << "Fail to init raft node";
+ LOG(ERROR) << "Fail to init peering node";
delete node;
return -1;
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index fca636b1..13602894 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -43,7 +43,7 @@ void catch_crash(int sig) {
Option<std::string> fetch_file_contents(const std::string & file_path) {
if(!file_exists(file_path)) {
- return Option<std::string>(404, "Error reading file containing raft peers.");
+ return Option<std::string>(404, "Error reading file containing peers.");
}
std::ifstream infile(file_path);
@@ -103,16 +103,13 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("api-key", 'a', "API key that allows all operations.", true);
options.add<std::string>("search-only-api-key", 's', "API key that allows only searches.", false);
- options.add<std::string>("listen-address", 'h', "Address to which Typesense server binds.", false, "0.0.0.0");
- options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8108);
+ options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");
+ options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);
+ options.add<std::string>("peering-address", '\0', "Internal IP address to which Typesense peering service binds.", false, "");
options.add<uint32_t>("peering-port", '\0', "Port on which Typesense peering service listens.", false, 8107);
options.add<std::string>("peers", '\0', "Path to file with comma separated string of peer node IPs.", false);
- options.add<std::string>("master", 'm', "To start the server as read-only replica, "
- "provide the master's address in http(s)://<master_address>:<master_port> format.",
- false, "");
-
options.add<std::string>("ssl-certificate", 'c', "Path to the SSL certificate file.", false, "");
options.add<std::string>("ssl-certificate-key", 'k', "Path to the SSL certificate key file.", false, "");
@@ -121,6 +118,12 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("log-dir", '\0', "Path to the log directory.", false, "");
options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");
+
+ // DEPRECATED
+ options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
+ options.add<uint32_t>("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108);
+ options.add<std::string>("master", 'm', "[DEPRECATED: use clustering via --peers] Master's address in http(s)://<master_address>:<master_port> format "
+ "to start as read-only replica.", false, "");
}
int init_logger(Config & config, const std::string & server_version) {
@@ -170,8 +173,8 @@ bool on_send_response(void *data) {
return true;
}
-int start_raft_server(ReplicationState& replication_state, const std::string& state_dir,
- const std::string& path_to_peers, uint32_t api_port, uint32_t raft_port) {
+int start_raft_server(ReplicationState& replication_state, const std::string& state_dir, const std::string& path_to_peers,
+ const std::string& peering_address, uint32_t peering_port, uint32_t api_port) {
std::string peer_ips;
@@ -186,7 +189,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
} else {
peer_ips = peers_op.get();
if(peer_ips.empty()) {
- LOG(ERROR) << "File containing raft peers is empty.";
+ LOG(ERROR) << "File containing peers is empty.";
exit(-1);
} else {
peer_ips = peers_op.get();
@@ -194,25 +197,25 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
}
}
- // start raft server
+ // start peering server
brpc::Server raft_server;
- if (braft::add_service(&raft_server, raft_port) != 0) {
- LOG(ERROR) << "Failed to add raft service";
+ if (braft::add_service(&raft_server, peering_port) != 0) {
+ LOG(ERROR) << "Failed to add peering service";
exit(-1);
}
- if (raft_server.Start(raft_port, nullptr) != 0) {
- LOG(ERROR) << "Failed to start raft server";
+ if (raft_server.Start(peering_port, nullptr) != 0) {
+ LOG(ERROR) << "Failed to start peering service";
exit(-1);
}
- if (replication_state.start(api_port, raft_port, 1000, 600, state_dir, peer_ips) != 0) {
- LOG(ERROR) << "Failed to start raft state";
+ if (replication_state.start(peering_address, peering_port, api_port, 1000, 600, state_dir, peer_ips) != 0) {
+ LOG(ERROR) << "Failed to start peering state";
exit(-1);
}
- LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();
+ LOG(INFO) << "Typesense peering service is running on " << raft_server.listen_address();
// Wait until 'CTRL-C' is pressed. then Stop() and Join() the service
size_t raft_counter = 0;
@@ -232,7 +235,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
sleep(1);
}
- LOG(INFO) << "Typesense raft service is going to quit";
+ LOG(INFO) << "Typesense peering service is going to quit";
// Stop counter before server
replication_state.shutdown();
@@ -268,7 +271,7 @@ int run_server(const Config & config, const std::string & version, void (*master
bool create_init_db_snapshot = false; // for importing raw DB from earlier versions
if(!directory_exists(db_dir) && file_exists(data_dir+"/CURRENT") && file_exists(data_dir+"/IDENTITY")) {
- if(!config.get_raft_peers().empty()) {
+ if(!config.get_peers().empty()) {
LOG(ERROR) << "Your data directory needs to be migrated to the new format.";
LOG(ERROR) << "To do that, please start the Typesense server without the --peers argument.";
return 1;
@@ -296,8 +299,8 @@ int run_server(const Config & config, const std::string & version, void (*master
server = new HttpServer(
version,
- config.get_listen_address(),
- config.get_listen_port(),
+ config.get_api_address(),
+ config.get_api_port(),
config.get_ssl_cert(),
config.get_ssl_cert_key(),
config.get_enable_cors()
@@ -308,17 +311,18 @@ int run_server(const Config & config, const std::string & version, void (*master
server->on(SEND_RESPONSE_MSG, on_send_response);
server->on(ReplicationState::REPLICATION_MSG, async_write_request);
- // first we start the raft service
+ // first we start the peering service
ThreadPool thread_pool(4);
ReplicationState replication_state(&store, &thread_pool, server->get_message_dispatcher(), create_init_db_snapshot);
std::thread raft_thread([&replication_state, &config, &state_dir]() {
- std::string path_to_peers = config.get_raft_peers();
- start_raft_server(replication_state, state_dir, path_to_peers, config.get_listen_port(), config.get_raft_port());
+ std::string path_to_peers = config.get_peers();
+ start_raft_server(replication_state, state_dir, path_to_peers,
+ config.get_peering_address(), config.get_peering_port(), config.get_api_port());
});
- // Wait for raft service to be ready before starting http
+ // Wait for peering service to be ready before starting http
// Follower or leader must have started AND data must also have been loaded
LOG(INFO) << "Waiting for peering service to be ready before starting API service...";
while(replication_state.get_init_readiness_count() < 2) {
@@ -331,7 +335,7 @@ int run_server(const Config & config, const std::string & version, void (*master
// we are out of the event loop here
- LOG(INFO) << "Typesense API service has quit. Stopping raft service...";
+ LOG(INFO) << "Typesense API service has quit. Stopping peering service...";
quit_raft_service = true;
raft_thread.join();
diff --git a/test/config_test.cpp b/test/config_test.cpp
index 143cb1b0..82c0bdf0 100644
--- a/test/config_test.cpp
+++ b/test/config_test.cpp
@@ -34,7 +34,7 @@ TEST(ConfigTest, LoadCmdLineArguments) {
config.load_config_cmd_args(options);
ASSERT_EQ("abcd", config.get_api_key());
- ASSERT_EQ(8080, config.get_listen_port());
+ ASSERT_EQ(8080, config.get_api_port());
ASSERT_EQ("/tmp/data", config.get_data_dir());
}
@@ -46,7 +46,7 @@ TEST(ConfigTest, LoadEnvVars) {
config.load_config_env();
ASSERT_EQ("/tmp/ts", config.get_data_dir());
- ASSERT_EQ(9090, config.get_listen_port());
+ ASSERT_EQ(9090, config.get_api_port());
}
TEST(ConfigTest, BadConfigurationReturnsError) {
@@ -85,7 +85,7 @@ TEST(ConfigTest, LoadConfigFile) {
ASSERT_EQ("/tmp/ts", config.get_data_dir());
ASSERT_EQ("1234", config.get_api_key());
ASSERT_EQ("/tmp/logs", config.get_log_dir());
- ASSERT_EQ(9090, config.get_listen_port());
+ ASSERT_EQ(9090, config.get_api_port());
ASSERT_EQ(false, config.get_enable_cors());
}
@@ -157,8 +157,8 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) {
ASSERT_EQ("abcd", config.get_api_key());
ASSERT_EQ("/tmp/data", config.get_data_dir());
ASSERT_EQ("/tmp/ts_log", config.get_log_dir());
- ASSERT_EQ(9090, config.get_listen_port());
+ ASSERT_EQ(9090, config.get_api_port());
ASSERT_EQ(true, config.get_enable_cors());
- ASSERT_EQ("192.168.10.10", config.get_listen_address());
+ ASSERT_EQ("192.168.10.10", config.get_api_address());
ASSERT_EQ("supersecret", config.get_search_only_api_key());
}
\ No newline at end of file