Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Migrate to butil logger.

kishorence057b64d9de2
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c825f956..56d1bc41 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -49,10 +49,12 @@ include(cmake/H2O.cmake)
include(cmake/RocksDB.cmake)
include(cmake/GoogleTest.cmake)
include(cmake/TestResources.cmake)
-include(cmake/g3log.cmake)
include(cmake/Iconv.cmake)
-include(cmake/brpc.cmake)
-include(cmake/braft.cmake)
+
+if (APPLE)
+ include(cmake/brpc.cmake)
+ include(cmake/braft.cmake)
+endif()
FILE(GLOB SRC_FILES src/*.cpp)
FILE(GLOB TEST_FILES test/*.cpp)
@@ -65,8 +67,6 @@ include_directories(${DEP_ROOT_DIR}/${FOR_NAME})
include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)
include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
-include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)
-include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build/include)
include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)
@@ -76,8 +76,6 @@ file(WRITE ${DEP_ROOT_DIR}/includes.txt "")
file(APPEND ${DEP_ROOT_DIR}/includes.txt "${DEP_ROOT_DIR}/${FOR_NAME}")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/src")
-file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/include")
@@ -86,7 +84,6 @@ link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
-link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)
link_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/lib)
link_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/lib)
@@ -96,7 +93,6 @@ file(WRITE ${DEP_ROOT_DIR}/libs.txt "")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${FOR_NAME}")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")
-file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/lib")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/lib")
diff --git a/cmake/g3log.cmake b/cmake/g3log.cmake
deleted file mode 100644
index fd7e8b5f..00000000
--- a/cmake/g3log.cmake
+++ /dev/null
@@ -1,44 +0,0 @@
-# Download and build g3log
-
-set(G3LOG_VERSION 1.3.2)
-set(G3LOG_NAME g3log-${G3LOG_VERSION})
-set(G3LOG_TAR_PATH ${DEP_ROOT_DIR}/${G3LOG_NAME}.tar.gz)
-
-# Specifically pick the static version since the build generates both static and dynamic libraries
-set(G3LOGGER_LIBRARIES "${DEP_ROOT_DIR}/${G3LOG_NAME}/build/libg3logger.a")
-
-if(NOT EXISTS ${G3LOG_TAR_PATH})
- message(STATUS "Downloading G3log...")
- file(DOWNLOAD https://github.com/KjellKod/g3log/archive/${G3LOG_VERSION}.tar.gz ${G3LOG_TAR_PATH})
-endif()
-
-if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME})
- message(STATUS "Extracting G3log...")
- execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${G3LOG_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
-endif()
-
-if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
- message("Configuring G3log...")
- file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
- execute_process(COMMAND ${CMAKE_COMMAND}
- "-DCMAKE_BUILD_TYPE=Release"
- "-DG3_SHARED_LIB=OFF"
- "-H${DEP_ROOT_DIR}/${G3LOG_NAME}"
- "-B${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
- RESULT_VARIABLE
- G3LOG_CONFIGURE)
- if(NOT G3LOG_CONFIGURE EQUAL 0)
- message(FATAL_ERROR "G3log Configure failed!")
- endif()
-
- if(BUILD_DEPS STREQUAL "yes")
- message("Building G3log locally...")
- execute_process(COMMAND ${CMAKE_COMMAND} --build
- "${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
- RESULT_VARIABLE
- G3LOG_BUILD)
- if(NOT G3LOG_BUILD EQUAL 0)
- message(FATAL_ERROR "G3log build failed!")
- endif()
- endif()
-endif()
diff --git a/docker/debugger.Dockerfile b/docker/debugger.Dockerfile
index ac216bfc..8fa66da4 100644
--- a/docker/debugger.Dockerfile
+++ b/docker/debugger.Dockerfile
@@ -1,4 +1,4 @@
FROM ubuntu:18.04
RUN apt-get update
-RUN apt-get install -y gdb valgrind
\ No newline at end of file
+RUN apt-get install -y gdb valgrindinit_db
\ No newline at end of file
diff --git a/docker/development.Dockerfile b/docker/development.Dockerfile
index 6ee701a2..cafb9495 100644
--- a/docker/development.Dockerfile
+++ b/docker/development.Dockerfile
@@ -64,7 +64,13 @@ ADD https://github.com/apache/incubator-brpc/archive/0.9.7-rc03.tar.gz /opt/brpc
RUN tar -C /opt -xf /opt/brpc-0.9.7-rc03.tar.gz
COPY patches/brpc_cmakelists.txt /opt/incubator-brpc-0.9.7-rc03/src/CMakeLists.txt
RUN mkdir -p /opt/incubator-brpc-0.9.7-rc03/build && cd /opt/incubator-brpc-0.9.7-rc03/build && \
- cmake -DWITH_DEBUG_SYMBOLS=OFF .. && make -j8 && make install && rm -rf /usr/local/lib/*.so*
+ cmake -DWITH_DEBUG_SYMBOLS=ON .. && make -j8 && make install && rm -rf /usr/local/lib/*.so*
+
+ADD https://github.com/baidu/braft/archive/v1.1.0.tar.gz /opt/braft-v1.1.0.tar.gz
+RUN tar -C /opt -xf /opt/braft-v1.1.0.tar.gz
+COPY patches/braft_cmakelists.txt /opt/braft-1.1.0/src/CMakeLists.txt
+RUN mkdir -p /opt/braft-1.1.0/build && cd /opt/braft-1.1.0/build && \
+ cmake -DWITH_DEBUG_SYMBOLS=ON .. && make -j8 && make install && rm -rf /usr/local/lib/*.so*
ENV CC /usr/local/gcc-6.4.0/bin/gcc
ENV CXX /usr/local/gcc-6.4.0/bin/g++
diff --git a/docker/patches/braft_cmakelists.txt b/docker/patches/braft_cmakelists.txt
new file mode 100644
index 00000000..a39c7732
--- /dev/null
+++ b/docker/patches/braft_cmakelists.txt
@@ -0,0 +1,33 @@
+if(BUILD_UNIT_TESTS)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUNIT_TEST")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUNIT_TEST")
+elseif(NOT DEBUG)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNDEBUG")
+endif()
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+add_library(OBJ_LIB OBJECT ${SOURCES})
+
+set_property(TARGET ${OBJ_LIB} PROPERTY POSITION_INDEPENDENT_CODE 1)
+#add_library(braft-shared SHARED $<TARGET_OBJECTS:OBJ_LIB>)
+add_library(braft-static STATIC $<TARGET_OBJECTS:OBJ_LIB>)
+#target_link_libraries(braft-shared ${DYNAMIC_LIB})
+target_link_libraries(braft-static ${DYNAMIC_LIB})
+
+SET_TARGET_PROPERTIES(braft-static PROPERTIES OUTPUT_NAME braft CLEAN_DIRECT_OUTPUT 1)
+#SET_TARGET_PROPERTIES(braft-shared PROPERTIES OUTPUT_NAME braft CLEAN_DIRECT_OUTPUT 1)
+
+install(TARGETS braft-static
+ RUNTIME DESTINATION bin
+ LIBRARY DESTINATION lib${LIBSUFFIX}
+ ARCHIVE DESTINATION lib${LIBSUFFIX}
+ )
+
+#install(TARGETS braft-shared
+# RUNTIME DESTINATION bin
+# LIBRARY DESTINATION lib${LIBSUFFIX}
+# ARCHIVE DESTINATION lib${LIBSUFFIX}
+# )
diff --git a/include/http_client.h b/include/http_client.h
index bc141bfd..2ea24f88 100644
--- a/include/http_client.h
+++ b/include/http_client.h
@@ -53,7 +53,7 @@ public:
if(!ca_cert_path.empty()) {
curl_easy_setopt(curl, CURLOPT_CAINFO, ca_cert_path.c_str());
} else {
- LOG(ERR) << "Unable to locate system SSL certificates.";
+ LOG(ERROR) << "Unable to locate system SSL certificates.";
return 0;
}
diff --git a/include/logger.h b/include/logger.h
index cf5f8a4a..c8a9e62e 100644
--- a/include/logger.h
+++ b/include/logger.h
@@ -2,25 +2,4 @@
#include <string>
#include <iostream>
-#include <g3log/g3log.hpp>
-#include <g3log/logworker.hpp>
-#include <g3log/logmessage.hpp>
-
-/* Custom logging levels for use with g3log */
-const LEVELS ERR {WARNING.value + 10, {"ERROR"}};
-
-/* https://github.com/KjellKod/g3sinks/blob/master/snippets/ColorCoutSink.hpp */
-struct ConsoleLoggingSink {
- enum FG_Color {YELLOW = 33, RED = 31, GREEN=32, WHITE = 97};
-
- void ReceiveLogMessage(g3::LogMessageMover logEntry) {
- auto level = logEntry.get()._level;
-
- if (level.value >= ERR.value) {
- std::cerr << "\033[" << RED << "m"
- << logEntry.get().toString() << "\033[m" << std::flush;
- } else {
- std::cout << logEntry.get().toString() << std::flush;
- }
- }
-};
\ No newline at end of file
+#include <butil/logging.h>
diff --git a/include/raft_server.h b/include/raft_server.h
index 87dee9ed..abdd3855 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -80,10 +80,7 @@ private:
braft::Node* volatile node;
butil::atomic<int64_t> leader_term;
- rocksdb::DB* db;
- std::string db_path;
- rocksdb::Options db_options;
-
+ Store *store;
http_message_dispatcher* message_dispatcher;
butil::atomic<bool> has_initialized;
@@ -138,8 +135,6 @@ public:
void reset_db();
- rocksdb::DB *get_db() const;
-
static constexpr const char* REPLICATION_MSG = "raft_replication";
private:
diff --git a/include/store.h b/include/store.h
index 0df94024..c0de1cc4 100644
--- a/include/store.h
+++ b/include/store.h
@@ -74,24 +74,27 @@ public:
write_options.disableWAL = disable_wal;
// open DB
- rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
+ const rocksdb::Status& s = init_db();
+ assert(s.ok());
+ }
+
+ ~Store() {
+ close();
+ }
+ rocksdb::Status init_db() {
+ rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
if(!s.ok()) {
- LOG(ERR) << "Error while initializing store: " << s.ToString();
+ LOG(ERROR) << "Error while initializing store: " << s.ToString();
if(s.code() == rocksdb::Status::Code::kIOError) {
- LOG(ERR) << "It seems like the data directory " << state_dir_path << " is already being used by "
- << "another Typesense server. ";
- LOG(ERR) << "If you are SURE that this is not the case, delete the LOCK file "
- << "in the data directory and try again.";
- exit(1);
+ LOG(ERROR) << "It seems like the data directory " << state_dir_path << " is already being used by "
+ << "another Typesense server. ";
+ LOG(ERROR) << "If you are SURE that this is not the case, delete the LOCK file "
+ << "in the data directory and try again.";
}
}
- assert(s.ok());
- }
-
- ~Store() {
- close();
+ return s;
}
bool insert(const std::string& key, const std::string& value) {
@@ -121,7 +124,7 @@ public:
return StoreStatus::NOT_FOUND;
}
- LOG(ERR) << "Error while fetching the key: " << key << " - status is: " << status.ToString();
+ LOG(ERROR) << "Error while fetching the key: " << key << " - status is: " << status.ToString();
return StoreStatus::ERROR;
}
@@ -174,12 +177,12 @@ public:
rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);
if(!status.ok()) {
- LOG(ERR) << "Error while fetching updates for replication: " << status.ToString();
+ LOG(ERROR) << "Error while fetching updates for replication: " << status.ToString();
std::ostringstream error;
error << "Unable to fetch updates. " << "Master's latest sequence number is " << local_latest_seq_num
<< " but requested sequence number is " << seq_number;
- LOG(ERR) << error.str();
+ LOG(ERROR) << error.str();
return Option<std::vector<std::string>*>(400, error.str());
}
@@ -189,7 +192,7 @@ public:
error << "Invalid iterator. Master's latest sequence number is " << local_latest_seq_num << " but "
<< "updates are requested from sequence number " << seq_number << ". "
<< "The master's WAL entries might have expired (they are kept only for 24 hours).";
- LOG(ERR) << error.str();
+ LOG(ERROR) << error.str();
return Option<std::vector<std::string>*>(400, error.str());
}
@@ -207,7 +210,7 @@ public:
error << "Invalid iterator. Requested sequence number is " << seq_number << " but "
<< "updates are available only from sequence number " << batch.sequence << ". "
<< "The master's WAL entries might have expired (they are kept only for 24 hours).";
- LOG(ERR) << error.str();
+ LOG(ERROR) << error.str();
return Option<std::vector<std::string>*>(400, error.str());
}
}
diff --git a/include/typesense_server_utils.h b/include/typesense_server_utils.h
index c9147348..ea114220 100644
--- a/include/typesense_server_utils.h
+++ b/include/typesense_server_utils.h
@@ -24,7 +24,7 @@ void stream_response(bool (*handler)(http_req* req, http_res* res, void* data),
void init_cmdline_options(cmdline::parser & options, int argc, char **argv);
-int init_logger(Config & config, const std::string & server_version, std::unique_ptr<g3::LogWorker> & log_worker);
+int init_logger(Config & config, const std::string & server_version);
int run_server(const Config & config, const std::string & version,
void (*master_server_routes)(), void (*replica_server_routes)());
diff --git a/src/collection.cpp b/src/collection.cpp
index 6fe665b1..d8150587 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -93,7 +93,7 @@ Option<uint32_t> Collection::to_doc(const std::string & json_str, nlohmann::json
try {
document = nlohmann::json::parse(json_str);
} catch(const std::exception& e) {
- LOG(ERR) << "JSON error: " << e.what();
+ LOG(ERROR) << "JSON error: " << e.what();
return Option<uint32_t>(400, "Bad JSON.");
}
@@ -717,7 +717,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
if(!document_op.ok()) {
- LOG(ERR) << "Document fetch error. " << document_op.error();
+ LOG(ERROR) << "Document fetch error. " << document_op.error();
continue;
}
@@ -819,7 +819,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
if(!document_op.ok()) {
- LOG(ERR) << "Facet fetch error. " << document_op.error();
+ LOG(ERROR) << "Facet fetch error. " << document_op.error();
continue;
}
@@ -1084,7 +1084,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) {
StoreStatus doc_status = store->get(get_seq_id_key(seq_id), parsed_document);
if(doc_status == StoreStatus::NOT_FOUND) {
- LOG(ERR) << "Sequence ID exists, but document is missing for id: " << id;
+ LOG(ERROR) << "Sequence ID exists, but document is missing for id: " << id;
return Option<nlohmann::json>(404, "Could not find a document with id: " + id);
}
@@ -1133,7 +1133,7 @@ Option<std::string> Collection::remove(const std::string & id, const bool remove
StoreStatus doc_status = store->get(get_seq_id_key(seq_id), parsed_document);
if(doc_status == StoreStatus::NOT_FOUND) {
- LOG(ERR) << "Sequence ID exists, but document is missing for id: " << id;
+ LOG(ERROR) << "Sequence ID exists, but document is missing for id: " << id;
return Option<std::string>(404, "Could not find a document with id: " + id);
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 11fba173..d4d8cd89 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -137,7 +137,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
try {
document = nlohmann::json::parse(iter->value().ToString());
} catch(const std::exception& e) {
- LOG(ERR) << "JSON error: " << e.what();
+ LOG(ERROR) << "JSON error: " << e.what();
return Option<bool>(false, "Bad JSON.");
}
@@ -203,6 +203,7 @@ void CollectionManager::dispose() {
}
collections.clear();
+ store->close();
}
bool CollectionManager::auth_key_matches(std::string auth_key_sent) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 6720f859..c88c90a7 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -58,7 +58,7 @@ bool post_create_collection(http_req & req, http_res & res) {
try {
req_json = nlohmann::json::parse(req.body);
} catch(const std::exception& e) {
- //LOG(ERR) << "JSON error: " << e.what();
+ //LOG(ERROR) << "JSON error: " << e.what();
res.send_400("Bad JSON.");
return false;
}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 2bb8d3e3..9a9bd90f 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -61,7 +61,7 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
int nid = NID_X9_62_prime256v1;
EC_KEY *key = EC_KEY_new_by_curve_name(nid);
if (key == NULL) {
- LOG(ERR) << "Failed to create DH/ECDH.";
+ LOG(ERROR) << "Failed to create DH/ECDH.";
return -1;
}
@@ -73,11 +73,11 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
SSL_CTX_set_options(accept_ctx->ssl_ctx, SSL_OP_SINGLE_ECDH_USE);
if (SSL_CTX_use_certificate_chain_file(accept_ctx->ssl_ctx, cert_file) != 1) {
- LOG(ERR) << "An error occurred while trying to load server certificate file:" << cert_file;
+ LOG(ERROR) << "An error occurred while trying to load server certificate file:" << cert_file;
return -1;
}
if (SSL_CTX_use_PrivateKey_file(accept_ctx->ssl_ctx, key_file, SSL_FILETYPE_PEM) != 1) {
- LOG(ERR) << "An error occurred while trying to load private key file: " << key_file;
+ LOG(ERROR) << "An error occurred while trying to load private key file: " << key_file;
return -1;
}
@@ -124,7 +124,7 @@ int HttpServer::run(ReplicationState* replication_state) {
this->replication_state = replication_state;
if (create_listener() != 0) {
- LOG(ERR) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);
+ LOG(ERROR) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);
return 1;
} else {
LOG(INFO) << "Typesense has started. Ready to accept requests on port " << listen_port;
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 7e92c7a8..28e23e00 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -67,8 +67,7 @@ int main(int argc, char **argv) {
exit(1);
}
- std::unique_ptr<g3::LogWorker> log_worker = g3::LogWorker::createLogWorker();
- int ret_code = init_logger(config, TYPESENSE_VERSION, log_worker);
+ int ret_code = init_logger(config, TYPESENSE_VERSION);
if(ret_code != 0) {
return ret_code;
}
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 385258b6..e3bac75b 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -65,8 +65,8 @@ int ReplicationState::start(int port, int election_timeout_ms, int snapshot_inte
LOG(INFO) << "Snapshot does not exist. We will remove db dir and init db fresh.";
reset_db();
- if (!butil::DeleteFile(butil::FilePath(db_path), true)) {
- LOG(WARNING) << "rm " << db_path << " failed";
+ if (!butil::DeleteFile(butil::FilePath(store->get_state_dir_path()), true)) {
+ LOG(WARNING) << "rm " << store->get_state_dir_path() << " failed";
return -1;
}
@@ -225,7 +225,7 @@ void* ReplicationState::save_snapshot(void* arg) {
void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done) {
// Start a new bthread to avoid blocking StateMachine since it could be slow to write data to disk
SnapshotArg* arg = new SnapshotArg;
- arg->db = db;
+ arg->db = store->_get_db_unsafe();
arg->writer = writer;
arg->done = done;
bthread_t tid;
@@ -233,15 +233,14 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
}
int ReplicationState::init_db() {
- if (!butil::CreateDirectory(butil::FilePath(db_path))) {
- LOG(WARNING) << "CreateDirectory " << db_path << " failed";
+ if (!butil::CreateDirectory(butil::FilePath(store->get_state_dir_path()))) {
+ LOG(WARNING) << "CreateDirectory " << store->get_state_dir_path() << " failed";
return -1;
}
- db_options.create_if_missing = true;
- rocksdb::Status status = rocksdb::DB::Open(db_options, db_path, &db);
+ const rocksdb::Status& status = store->init_db();
if (!status.ok()) {
- LOG(WARNING) << "Open DB " << db_path << " failed, msg: " << status.ToString();
+ LOG(WARNING) << "Open DB " << store->get_state_dir_path() << " failed, msg: " << status.ToString();
return -1;
}
@@ -273,23 +272,23 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
// Load snapshot from reader, replacing the running StateMachine
reset_db();
- if (!butil::DeleteFile(butil::FilePath(db_path), true)) {
- LOG(WARNING) << "rm " << db_path << " failed";
+ if (!butil::DeleteFile(butil::FilePath(store->get_state_dir_path()), true)) {
+ LOG(WARNING) << "rm " << store->get_state_dir_path() << " failed";
return -1;
}
- LOG(TRACE) << "rm " << db_path << " success";
+ LOG(TRACE) << "rm " << store->get_state_dir_path() << " success";
std::string snapshot_path = reader->get_path();
snapshot_path.append(std::string("/") + db_snapshot_name);
// tries to use link if possible, or else copies
- if (!copy_dir(snapshot_path, db_path)) {
- LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << db_path << " failed";
+ if (!copy_dir(snapshot_path, store->get_state_dir_path())) {
+ LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << store->get_state_dir_path() << " failed";
return -1;
}
- LOG(TRACE) << "copy snapshot " << snapshot_path << " to " << db_path << " success";
+ LOG(TRACE) << "copy snapshot " << snapshot_path << " to " << store->get_state_dir_path() << " success";
return init_db();
}
@@ -306,22 +305,15 @@ void ReplicationState::refresh_peers(const std::string & peers) {
}
}
-rocksdb::DB *ReplicationState::get_db() const {
- return db;
-}
-
ReplicationState::ReplicationState(Store *store, http_message_dispatcher *message_dispatcher,
std::promise<bool> *ready, bool create_init_db_snapshot):
- node(nullptr), leader_term(-1), message_dispatcher(message_dispatcher),
+ node(nullptr), leader_term(-1), store(store), message_dispatcher(message_dispatcher),
has_initialized(false), ready(ready), create_init_db_snapshot(create_init_db_snapshot) {
- db = store->_get_db_unsafe();
- db_path = store->get_state_dir_path();
- db_options = store->get_db_options();
+
}
void ReplicationState::reset_db() {
- delete db;
- db = nullptr;
+ store->close();
}
void InitSnapshotClosure::Run() {
diff --git a/src/replicator.cpp b/src/replicator.cpp
index 3b275cee..7c42fe17 100644
--- a/src/replicator.cpp
+++ b/src/replicator.cpp
@@ -120,11 +120,11 @@ void Replicator::start(http_message_dispatcher* message_dispatcher, const std::s
}
} else {
- LOG(ERR) << "Replication error while fetching records from master, status_code=" << status_code
+ LOG(ERROR) << "Replication error while fetching records from master, status_code=" << status_code
<< ", replica's latest sequence number: " << latest_seq_num;
if(status_code != 0) {
- LOG(ERR) << json_response;
+ LOG(ERROR) << json_response;
}
}
@@ -145,8 +145,8 @@ bool Replicator::on_replication_event(void *data) {
try {
collection_meta = nlohmann::json::parse(replication_event->value);
} catch(...) {
- LOG(ERR) << "Failed to parse collection meta JSON.";
- LOG(ERR) << "Replication event value: " << replication_event->value;
+ LOG(ERROR) << "Failed to parse collection meta JSON.";
+ LOG(ERROR) << "Replication event value: " << replication_event->value;
delete replication_event;
exit(1);
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 296f0c36..5a5c733d 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -7,19 +7,38 @@
#include <braft/raft.h>
#include <raft_server.h>
#include <fstream>
+#include <execinfo.h>
#include "core_api.h"
#include "typesense_server_utils.h"
#include "file_utils.h"
HttpServer* server;
+std::atomic<bool> quit_raft_service;
void catch_interrupt(int sig) {
LOG(INFO) << "Stopping Typesense server...";
signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantly
+ quit_raft_service = true;
server->stop();
}
+void catch_crash(int sig) {
+ void *bt[1024];
+ char **bt_syms;
+ size_t bt_size;
+
+ bt_size = backtrace(bt, 1024);
+ bt_syms = backtrace_symbols(bt, bt_size);
+
+ LOG(ERROR) << "Typesense crashed...";
+ for (size_t i = 0; i < bt_size; i++) {
+ LOG(ERROR) << bt_syms[i];
+ }
+
+ exit(-1);
+}
+
Option<std::string> fetch_file_contents(const std::string & file_path) {
if(!file_exists(file_path)) {
return Option<std::string>(404, "Error reading file containing raft peers.");
@@ -103,33 +122,41 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");
}
-int init_logger(Config & config, const std::string & server_version, std::unique_ptr<g3::LogWorker> & log_worker) {
+int init_logger(Config & config, const std::string & server_version) {
// remove SIGTERM since we handle it on our own
- g3::overrideSetupSignals({{SIGABRT, "SIGABRT"}, {SIGFPE, "SIGFPE"},{SIGILL, "SIGILL"}, {SIGSEGV, "SIGSEGV"},});
+ signal(SIGABRT, catch_crash);
+ signal(SIGFPE, catch_crash);
+ signal(SIGILL, catch_crash);
+ signal(SIGSEGV, catch_crash);
+
+ logging::LoggingSettings log_settings;
// we can install new signal handlers only after overriding above
signal(SIGINT, catch_interrupt);
signal(SIGTERM, catch_interrupt);
std::string log_dir = config.get_log_dir();
+ std::string log_path;
if(log_dir.empty()) {
// use console logger if log dir is not specified
- log_worker->addSink(std2::make_unique<ConsoleLoggingSink>(),
- &ConsoleLoggingSink::ReceiveLogMessage);
+ log_settings.logging_dest = logging::LOG_TO_SYSTEM_DEBUG_LOG;
} else {
if(!directory_exists(log_dir)) {
std::cerr << "Typesense failed to start. " << "Log directory " << log_dir << " does not exist.";
return 1;
}
- log_worker->addDefaultLogger("typesense", log_dir, "");
+ log_settings.logging_dest = logging::LOG_TO_FILE;
+ log_path = log_dir + "/" + "typesense.log";
+ log_settings.log_file = log_path.c_str();
- std::cout << "Starting Typesense " << server_version << ". Log directory is configured as: "
+ LOG(INFO) << "Starting Typesense " << server_version << ". Log directory is configured as: "
<< log_dir << std::endl;
}
- g3::initializeLogging(log_worker.get());
+ logging::InitLogging(log_settings);
+ logging::SetMinLogLevel(0);
return 0;
}
@@ -153,10 +180,10 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
const Option<std::string> & peers_op = fetch_file_contents(path_to_peers);
if(!peers_op.ok()) {
- LOG(ERR) << peers_op.error();
+ LOG(ERROR) << peers_op.error();
exit(-1);
} else if(peer_ips_string.empty()) {
- LOG(ERR) << "File containing raft peers is empty.";
+ LOG(ERROR) << "File containing raft peers is empty.";
exit(-1);
} else {
peer_ips_string = peers_op.get();
@@ -167,12 +194,12 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
brpc::Server raft_server;
if (braft::add_service(&raft_server, raft_port) != 0) {
- LOG(ERR) << "Failed to add raft service";
+ LOG(ERROR) << "Failed to add raft service";
exit(-1);
}
if (raft_server.Start(raft_port, nullptr) != 0) {
- LOG(ERR) << "Failed to start raft server";
+ LOG(ERROR) << "Failed to start raft server";
exit(-1);
}
@@ -181,7 +208,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
std::string peers = StringUtils::join(peer_ips, ":0,");
if (replication_state.start(raft_port, 1000, 600, state_dir, peers) != 0) {
- LOG(ERR) << "Failed to start raft state";
+ LOG(ERROR) << "Failed to start raft state";
exit(-1);
}
@@ -189,12 +216,12 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
// Wait until 'CTRL-C' is pressed. then Stop() and Join() the service
size_t raft_counter = 0;
- while (!brpc::IsAskedToQuit()) {
+ while (!brpc::IsAskedToQuit() && !quit_raft_service.load()) {
if(++raft_counter % 10 == 0 && !path_to_peers.empty()) {
// reset peer configuration periodically to identify change in cluster membership
const Option<std::string> & refreshed_peers_op = fetch_file_contents(path_to_peers);
if(!refreshed_peers_op.ok()) {
- LOG(ERR) << "Error while refreshing peer configuration: " << refreshed_peers_op.error();
+ LOG(ERROR) << "Error while refreshing peer configuration: " << refreshed_peers_op.error();
continue;
}
@@ -221,9 +248,10 @@ int run_server(const Config & config, const std::string & version,
void (*master_server_routes)(), void (*replica_server_routes)()) {
LOG(INFO) << "Starting Typesense " << version << std::flush;
+ quit_raft_service = false;
if(!directory_exists(config.get_data_dir())) {
- LOG(ERR) << "Typesense failed to start. " << "Data directory " << config.get_data_dir()
+ LOG(ERROR) << "Typesense failed to start. " << "Data directory " << config.get_data_dir()
<< " does not exist.";
return 1;
}
@@ -236,16 +264,16 @@ int run_server(const Config & config, const std::string & version,
if(!directory_exists(db_dir) && file_exists(data_dir+"/CURRENT") && file_exists(data_dir+"/IDENTITY")) {
if(!config.get_raft_peers().empty()) {
- LOG(ERR) << "Your data directory needs to be migrated to the new format.";
- LOG(ERR) << "To do that, please start the Typesense server without the --peers argument.";
+ LOG(ERROR) << "Your data directory needs to be migrated to the new format.";
+ LOG(ERROR) << "To do that, please start the Typesense server without the --peers argument.";
return 1;
}
LOG(INFO) << "Migrating contents of data directory in a `db` sub-directory, as per the new data layout.";
bool moved = mv_dir(data_dir, db_dir);
if(!moved) {
- LOG(ERR) << "CRITICAL ERROR! Failed to migrate all files in the data directory into a `db` sub-directory.";
- LOG(ERR) << "NOTE: Please move remaining files manually. Failure to do so **WILL** lead to **DATA LOSS**.";
+ LOG(ERROR) << "CRITICAL ERROR! Failed to migrate all files in the data directory into a `db` sub-directory.";
+ LOG(ERROR) << "NOTE: Please move remaining files manually. Failure to do so **WILL** lead to **DATA LOSS**.";
return 1;
}
@@ -285,8 +313,6 @@ int run_server(const Config & config, const std::string & version,
start_raft_server(replication_state, state_dir, path_to_peers, config.get_raft_port());
});
- raft_thread.detach();
-
// wait for raft service to be ready before starting http
ready_future.get();
@@ -299,7 +325,7 @@ int run_server(const Config & config, const std::string & version,
std::vector<std::string> parts;
StringUtils::split(master_host_port, parts, ":");
if(parts.size() != 3) {
- LOG(ERR) << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>";
+ LOG(ERROR) << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>";
return 1;
}
@@ -317,6 +343,8 @@ int run_server(const Config & config, const std::string & version,
// we are out of the event loop here
+ raft_thread.join();
+
curl_global_cleanup();
delete server;