Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow customization of peering address.
kishorenc8829c5aebc09
diff --git a/include/config.h b/include/config.hindex 73ab9046..a2e35f5b 100644--- a/include/config.h+++ b/include/config.h@@ -13,11 +13,12 @@ private:std::string api_key;std::string search_only_api_key;- std::string listen_address;- uint32_t listen_port;+ std::string api_address;+ uint32_t api_port;- uint32_t raft_port;- std::string raft_peers;+ std::string peering_address;+ uint32_t peering_port;+ std::string peers;std::string master;@@ -34,9 +35,9 @@ private:public:Config() {- this->listen_address = "0.0.0.0";- this->listen_port = 8108;- this->raft_port = 8107;+ this->api_address = "0.0.0.0";+ this->api_port = 8108;+ this->peering_port = 8107;this->enable_cors = false;this->indices_per_collection = 4;}@@ -60,11 +61,11 @@ public:}void set_listen_address(const std::string & listen_address) {- this->listen_address = listen_address;+ this->api_address = listen_address;}void set_listen_port(int listen_port) {- this->listen_port = listen_port;+ this->api_port = listen_port;}void set_master(const std::string & master) {@@ -88,11 +89,11 @@ public:}void set_raft_port(int raft_port) {- this->raft_port = raft_port;+ this->peering_port = raft_port;}void set_raft_peers(const std::string & raft_peers) {- this->raft_peers = raft_peers;+ this->peers = raft_peers;}// getters@@ -113,12 +114,12 @@ public:return this->search_only_api_key;}- std::string get_listen_address() const {- return this->listen_address;+ std::string get_api_address() const {+ return this->api_address;}- int get_listen_port() const {- return this->listen_port;+ int get_api_port() const {+ return this->api_port;}std::string get_master() const {@@ -145,12 +146,16 @@ public:return indices_per_collection;}- int get_raft_port() const {- return this->raft_port;+ std::string get_peering_address() const {+ return this->peering_address;}- std::string get_raft_peers() const {- return this->raft_peers;+ int get_peering_port() const {+ return this->peering_port;+ }++ std::string get_peers() const {+ return this->peers;}// loaders@@ -171,18 +176,30 @@ public:this->search_only_api_key = get_env("TYPESENSE_SEARCH_ONLY_API_KEY");if(!get_env("TYPESENSE_LISTEN_ADDRESS").empty()) {- this->listen_address = get_env("TYPESENSE_LISTEN_ADDRESS");+ this->api_address = get_env("TYPESENSE_LISTEN_ADDRESS");}if(!get_env("TYPESENSE_LISTEN_PORT").empty()) {- this->listen_port = std::stoi(get_env("TYPESENSE_LISTEN_PORT"));+ this->api_port = std::stoi(get_env("TYPESENSE_LISTEN_PORT"));+ }++ if(!get_env("TYPESENSE_API_ADDRESS").empty()) {+ this->api_address = get_env("TYPESENSE_LISTEN_ADDRESS");+ }++ if(!get_env("TYPESENSE_API_PORT").empty()) {+ this->api_port = std::stoi(get_env("TYPESENSE_API_PORT"));+ }++ if(!get_env("TYPESENSE_PEERING_ADDRESS").empty()) {+ this->api_address = get_env("TYPESENSE_PEERING_ADDRESS");}- if(!get_env("TYPESENSE_RAFT_PORT").empty()) {- this->raft_port = std::stoi(get_env("TYPESENSE_RAFT_PORT"));+ if(!get_env("TYPESENSE_PEERING_PORT").empty()) {+ this->peering_port = std::stoi(get_env("TYPESENSE_PEERING_PORT"));}- this->raft_peers = get_env("TYPESENSE_RAFT_PEERS");+ this->peers = get_env("TYPESENSE_PEERS");this->master = get_env("TYPESENSE_MASTER");this->ssl_certificate = get_env("TYPESENSE_SSL_CERTIFICATE");@@ -229,7 +246,11 @@ public:}if(reader.Exists("server", "listen-address")) {- this->listen_address = reader.Get("server", "listen-address", "");+ this->api_address = reader.Get("server", "listen-address", "");+ }++ if(reader.Exists("server", "api-address")) {+ this->api_address = reader.Get("server", "api-address", "");}if(reader.Exists("server", "master")) {@@ -245,19 +266,27 @@ public:}if(reader.Exists("server", "listen-port")) {- this->listen_port = reader.GetInteger("server", "listen-port", 8108);+ this->api_port = reader.GetInteger("server", "listen-port", 8108);+ }++ if(reader.Exists("server", "api-port")) {+ this->api_port = reader.GetInteger("server", "api-port", 8108);}if(reader.Exists("server", "enable-cors")) {this->enable_cors = reader.GetBoolean("server", "enable-cors", false);}+ if(reader.Exists("server", "peering-address")) {+ this->peering_address = reader.Get("server", "peering-address", "");+ }+if(reader.Exists("server", "peering-port")) {- this->raft_port = reader.GetInteger("server", "raft-port", 8107);+ this->peering_port = reader.GetInteger("server", "peering-port", 8107);}if(reader.Exists("server", "peers")) {- this->raft_peers = reader.Get("server", "peers", "");+ this->peers = reader.Get("server", "peers", "");}}@@ -279,7 +308,11 @@ public:}if(options.exist("listen-address")) {- this->listen_address = options.get<std::string>("listen-address");+ this->api_address = options.get<std::string>("listen-address");+ }++ if(options.exist("api-address")) {+ this->api_address = options.get<std::string>("api-address");}if(options.exist("master")) {@@ -295,19 +328,27 @@ public:}if(options.exist("listen-port")) {- this->listen_port = options.get<uint32_t>("listen-port");+ this->api_port = options.get<uint32_t>("listen-port");+ }++ if(options.exist("api-port")) {+ this->api_port = options.get<uint32_t>("api-port");}if(options.exist("enable-cors")) {this->enable_cors = options.exist("enable-cors");}+ if(options.exist("peering-address")) {+ this->peering_address = options.get<std::string>("peering-address");+ }+if(options.exist("peering-port")) {- this->raft_port = options.get<uint32_t>("peering-port");+ this->peering_port = options.get<uint32_t>("peering-port");}if(options.exist("peers")) {- this->raft_peers = options.get<std::string>("peers");+ this->peers = options.get<std::string>("peers");}}diff --git a/include/raft_server.h b/include/raft_server.hindex 914ad6fe..9f3b1f0a 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -103,7 +103,8 @@ public:}// Starts this node- int start(int api_port, int raft_port, int election_timeout_ms, int snapshot_interval_s,+ int start(const std::string & peering_address, int peering_port, int api_port,+ int election_timeout_ms, int snapshot_interval_s,const std::string & raft_dir, const std::string & peers);// Generic write method for synchronizing all writes@@ -191,7 +192,7 @@ private:}void on_error(const ::braft::Error& e) {- LOG(ERROR) << "Met raft error " << e;+ LOG(ERROR) << "Met peering error " << e;}void on_configuration_committed(const ::braft::Configuration& conf) {diff --git a/include/store.h b/include/store.hindex c0de1cc4..9ae74f55 100644--- a/include/store.h+++ b/include/store.h@@ -90,7 +90,7 @@ public:LOG(ERROR) << "It seems like the data directory " << state_dir_path << " is already being used by "<< "another Typesense server. ";LOG(ERROR) << "If you are SURE that this is not the case, delete the LOCK file "- << "in the data directory and try again.";+ << "in the data db directory and try again.";}}diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 68fcf98b..739131f5 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -17,10 +17,21 @@ void ReplicationClosure::Run() {// State machine implementation-int ReplicationState::start(const int api_port, int raft_port, int election_timeout_ms, int snapshot_interval_s,+int ReplicationState::start(const std::string & peering_address, int peering_port, const int api_port,+ int election_timeout_ms, int snapshot_interval_s,const std::string & raft_dir, const std::string & peers) {+ butil::ip_t peering_ip;+ if(!peering_address.empty()) {+ int ip_conv_status = butil::str2ip(peering_address.c_str(), &peering_ip);+ if(ip_conv_status != 0) {+ LOG(ERROR) << "Failed to parse peering address `" << peering_address << "`";+ return -1;+ }+ } else {+ peering_ip = butil::my_ip();+ }- butil::EndPoint addr(butil::my_ip(), raft_port);+ butil::EndPoint addr(peering_ip, peering_port);braft::NodeOptions node_options;std::string actual_peers = peers;@@ -28,11 +39,11 @@ int ReplicationState::start(const int api_port, int raft_port, int election_timeif(actual_peers.empty()) {char str[INET_ADDRSTRLEN];inet_ntop(AF_INET, &(addr.ip.s_addr), str, INET_ADDRSTRLEN);- actual_peers = std::string(str) + ":" + std::to_string(raft_port) + ":" + std::to_string(api_port);+ actual_peers = std::string(str) + ":" + std::to_string(peering_port) + ":" + std::to_string(api_port);}if(node_options.initial_conf.parse_from(actual_peers) != 0) {- LOG(ERROR) << "Fail to parse peer configuration `" << peers << "`";+ LOG(ERROR) << "Failed to parse peer configuration `" << peers << "`";return -1;}@@ -73,7 +84,7 @@ int ReplicationState::start(const int api_port, int raft_port, int election_time}if (node->init(node_options) != 0) {- LOG(ERROR) << "Fail to init raft node";+ LOG(ERROR) << "Fail to init peering node";delete node;return -1;}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex fca636b1..13602894 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -43,7 +43,7 @@ void catch_crash(int sig) {Option<std::string> fetch_file_contents(const std::string & file_path) {if(!file_exists(file_path)) {- return Option<std::string>(404, "Error reading file containing raft peers.");+ return Option<std::string>(404, "Error reading file containing peers.");}std::ifstream infile(file_path);@@ -103,16 +103,13 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("api-key", 'a', "API key that allows all operations.", true);options.add<std::string>("search-only-api-key", 's', "API key that allows only searches.", false);- options.add<std::string>("listen-address", 'h', "Address to which Typesense server binds.", false, "0.0.0.0");- options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8108);+ options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");+ options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);+ options.add<std::string>("peering-address", '\0', "Internal IP address to which Typesense peering service binds.", false, "");options.add<uint32_t>("peering-port", '\0', "Port on which Typesense peering service listens.", false, 8107);options.add<std::string>("peers", '\0', "Path to file with comma separated string of peer node IPs.", false);- options.add<std::string>("master", 'm', "To start the server as read-only replica, "- "provide the master's address in http(s)://<master_address>:<master_port> format.",- false, "");-options.add<std::string>("ssl-certificate", 'c', "Path to the SSL certificate file.", false, "");options.add<std::string>("ssl-certificate-key", 'k', "Path to the SSL certificate key file.", false, "");@@ -121,6 +118,12 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("log-dir", '\0', "Path to the log directory.", false, "");options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");++ // DEPRECATED+ options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");+ options.add<uint32_t>("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108);+ options.add<std::string>("master", 'm', "[DEPRECATED: use clustering via --peers] Master's address in http(s)://<master_address>:<master_port> format "+ "to start as read-only replica.", false, "");}int init_logger(Config & config, const std::string & server_version) {@@ -170,8 +173,8 @@ bool on_send_response(void *data) {return true;}-int start_raft_server(ReplicationState& replication_state, const std::string& state_dir,- const std::string& path_to_peers, uint32_t api_port, uint32_t raft_port) {+int start_raft_server(ReplicationState& replication_state, const std::string& state_dir, const std::string& path_to_peers,+ const std::string& peering_address, uint32_t peering_port, uint32_t api_port) {std::string peer_ips;@@ -186,7 +189,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st} else {peer_ips = peers_op.get();if(peer_ips.empty()) {- LOG(ERROR) << "File containing raft peers is empty.";+ LOG(ERROR) << "File containing peers is empty.";exit(-1);} else {peer_ips = peers_op.get();@@ -194,25 +197,25 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st}}- // start raft server+ // start peering serverbrpc::Server raft_server;- if (braft::add_service(&raft_server, raft_port) != 0) {- LOG(ERROR) << "Failed to add raft service";+ if (braft::add_service(&raft_server, peering_port) != 0) {+ LOG(ERROR) << "Failed to add peering service";exit(-1);}- if (raft_server.Start(raft_port, nullptr) != 0) {- LOG(ERROR) << "Failed to start raft server";+ if (raft_server.Start(peering_port, nullptr) != 0) {+ LOG(ERROR) << "Failed to start peering service";exit(-1);}- if (replication_state.start(api_port, raft_port, 1000, 600, state_dir, peer_ips) != 0) {- LOG(ERROR) << "Failed to start raft state";+ if (replication_state.start(peering_address, peering_port, api_port, 1000, 600, state_dir, peer_ips) != 0) {+ LOG(ERROR) << "Failed to start peering state";exit(-1);}- LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();+ LOG(INFO) << "Typesense peering service is running on " << raft_server.listen_address();// Wait until 'CTRL-C' is pressed. then Stop() and Join() the servicesize_t raft_counter = 0;@@ -232,7 +235,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& stsleep(1);}- LOG(INFO) << "Typesense raft service is going to quit";+ LOG(INFO) << "Typesense peering service is going to quit";// Stop counter before serverreplication_state.shutdown();@@ -268,7 +271,7 @@ int run_server(const Config & config, const std::string & version, void (*masterbool create_init_db_snapshot = false; // for importing raw DB from earlier versionsif(!directory_exists(db_dir) && file_exists(data_dir+"/CURRENT") && file_exists(data_dir+"/IDENTITY")) {- if(!config.get_raft_peers().empty()) {+ if(!config.get_peers().empty()) {LOG(ERROR) << "Your data directory needs to be migrated to the new format.";LOG(ERROR) << "To do that, please start the Typesense server without the --peers argument.";return 1;@@ -296,8 +299,8 @@ int run_server(const Config & config, const std::string & version, void (*masterserver = new HttpServer(version,- config.get_listen_address(),- config.get_listen_port(),+ config.get_api_address(),+ config.get_api_port(),config.get_ssl_cert(),config.get_ssl_cert_key(),config.get_enable_cors()@@ -308,17 +311,18 @@ int run_server(const Config & config, const std::string & version, void (*masterserver->on(SEND_RESPONSE_MSG, on_send_response);server->on(ReplicationState::REPLICATION_MSG, async_write_request);- // first we start the raft service+ // first we start the peering serviceThreadPool thread_pool(4);ReplicationState replication_state(&store, &thread_pool, server->get_message_dispatcher(), create_init_db_snapshot);std::thread raft_thread([&replication_state, &config, &state_dir]() {- std::string path_to_peers = config.get_raft_peers();- start_raft_server(replication_state, state_dir, path_to_peers, config.get_listen_port(), config.get_raft_port());+ std::string path_to_peers = config.get_peers();+ start_raft_server(replication_state, state_dir, path_to_peers,+ config.get_peering_address(), config.get_peering_port(), config.get_api_port());});- // Wait for raft service to be ready before starting http+ // Wait for peering service to be ready before starting http// Follower or leader must have started AND data must also have been loadedLOG(INFO) << "Waiting for peering service to be ready before starting API service...";while(replication_state.get_init_readiness_count() < 2) {@@ -331,7 +335,7 @@ int run_server(const Config & config, const std::string & version, void (*master// we are out of the event loop here- LOG(INFO) << "Typesense API service has quit. Stopping raft service...";+ LOG(INFO) << "Typesense API service has quit. Stopping peering service...";quit_raft_service = true;raft_thread.join();diff --git a/test/config_test.cpp b/test/config_test.cppindex 143cb1b0..82c0bdf0 100644--- a/test/config_test.cpp+++ b/test/config_test.cpp@@ -34,7 +34,7 @@ TEST(ConfigTest, LoadCmdLineArguments) {config.load_config_cmd_args(options);ASSERT_EQ("abcd", config.get_api_key());- ASSERT_EQ(8080, config.get_listen_port());+ ASSERT_EQ(8080, config.get_api_port());ASSERT_EQ("/tmp/data", config.get_data_dir());}@@ -46,7 +46,7 @@ TEST(ConfigTest, LoadEnvVars) {config.load_config_env();ASSERT_EQ("/tmp/ts", config.get_data_dir());- ASSERT_EQ(9090, config.get_listen_port());+ ASSERT_EQ(9090, config.get_api_port());}TEST(ConfigTest, BadConfigurationReturnsError) {@@ -85,7 +85,7 @@ TEST(ConfigTest, LoadConfigFile) {ASSERT_EQ("/tmp/ts", config.get_data_dir());ASSERT_EQ("1234", config.get_api_key());ASSERT_EQ("/tmp/logs", config.get_log_dir());- ASSERT_EQ(9090, config.get_listen_port());+ ASSERT_EQ(9090, config.get_api_port());ASSERT_EQ(false, config.get_enable_cors());}@@ -157,8 +157,8 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) {ASSERT_EQ("abcd", config.get_api_key());ASSERT_EQ("/tmp/data", config.get_data_dir());ASSERT_EQ("/tmp/ts_log", config.get_log_dir());- ASSERT_EQ(9090, config.get_listen_port());+ ASSERT_EQ(9090, config.get_api_port());ASSERT_EQ(true, config.get_enable_cors());- ASSERT_EQ("192.168.10.10", config.get_listen_address());+ ASSERT_EQ("192.168.10.10", config.get_api_address());ASSERT_EQ("supersecret", config.get_search_only_api_key());}\ No newline at end of file