Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Don't trigger snapshots for small unapplied logs.

kishorenc9091df57c08b
diff --git a/.circleci/config.yml b/.circleci/config.yml
index a55fe2d4..7c4880df 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2
jobs:
build:
docker:
- - image: typesense/typesense-development:23-JUNE-2020-1
+ - image: typesense/typesense-development:10-SEPTEMBER-2020-1
environment:
- PROJECT_DIR: /typesense
- TYPESENSE_VERSION: $CIRCLE_BRANCH-$CIRCLE_SHA1
diff --git a/docker-build.sh b/docker-build.sh
index 89dcaeb6..93d8b98c 100755
--- a/docker-build.sh
+++ b/docker-build.sh
@@ -22,13 +22,13 @@ if [[ "$@" == *"--depclean"* ]]; then
fi
#echo "Creating development image..."
-#docker build --file $PROJECT_DIR/docker/development.Dockerfile --tag typesense/typesense-development:23-JUNE-2020-1 $PROJECT_DIR/docker
+#docker build --file $PROJECT_DIR/docker/development.Dockerfile --tag typesense/typesense-development:10-SEPTEMBER-2020-1 $PROJECT_DIR/docker
echo "Building Typesense $TYPESENSE_VERSION..."
-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:23-JUNE-2020-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:10-SEPTEMBER-2020-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
-DCMAKE_BUILD_TYPE=Release -H/typesense -B/typesense/$BUILD_DIR
-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:23-JUNE-2020-1 make typesense-server -C/typesense/$BUILD_DIR
+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:10-SEPTEMBER-2020-1 make typesense-server -C/typesense/$BUILD_DIR
if [[ "$@" == *"--build-deploy-image"* ]]; then
echo "Creating deployment image for Typesense $TYPESENSE_VERSION server ..."
@@ -48,7 +48,7 @@ fi
#
#if [[ "$@" == *"--create-deb-upload"* ]]; then
-# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:23-JUNE-2020-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
+# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:10-SEPTEMBER-2020-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
# -DCMAKE_BUILD_TYPE=Debug -H/typesense -B/typesense/$BUILD_DIR
#fi
diff --git a/docker/development.Dockerfile b/docker/development.Dockerfile
index ad2acb33..b10d1385 100644
--- a/docker/development.Dockerfile
+++ b/docker/development.Dockerfile
@@ -76,13 +76,15 @@ RUN mkdir -p /opt/incubator-brpc-0.9.7-rc03/bld && cd /opt/incubator-brpc-0.9.7-
make -j8 && make install && rm -rf /usr/local/lib/*.so* && \
rm -rf /opt/incubator-brpc-0.9.7-rc03/bld/output/bin
-ADD https://github.com/baidu/braft/archive/v1.1.1.tar.gz /opt/braft-v1.1.1.tar.gz
-RUN tar -C /opt -xf /opt/braft-v1.1.1.tar.gz
-COPY patches/braft_cmakelists.txt /opt/braft-1.1.1/src/CMakeLists.txt
-RUN chown root:root /opt/braft-1.1.1/src/CMakeLists.txt
-RUN mkdir -p /opt/braft-1.1.1/bld && cd /opt/braft-1.1.1/bld && \
- cmake -DWITH_DEBUG_SYMBOLS=ON -DBRPC_WITH_GLOG=ON .. && make -j8 && make install && rm -rf /usr/local/lib/*.so* && \
- rm -rf /opt/braft-1.1.1/bld/output/bin
+ADD https://github.com/baidu/braft/archive/22b0d4e.tar.gz /opt/braft-22b0d4e.tar.gz
+RUN tar -C /opt -xf /opt/braft-22b0d4e.tar.gz
+COPY patches/braft_cmakelists.txt /opt/braft-22b0d4ec45a2abdaceeb2216e668130a839ed931/src/CMakeLists.txt
+RUN chown root:root /opt/braft-22b0d4ec45a2abdaceeb2216e668130a839ed931/src/CMakeLists.txt
+RUN mkdir -p /opt/braft-22b0d4ec45a2abdaceeb2216e668130a839ed931/bld && \
+ cd /opt/braft-22b0d4ec45a2abdaceeb2216e668130a839ed931/bld && \
+ cmake -DWITH_DEBUG_SYMBOLS=ON -DBRPC_WITH_GLOG=ON .. && make -j8 && \
+ make install && rm -rf /usr/local/lib/*.so* && \
+ rm -rf /opt/braft-22b0d4ec45a2abdaceeb2216e668130a839ed931/bld/output/bin
ENV CC /usr/local/gcc-6.4.0/bin/gcc
ENV CXX /usr/local/gcc-6.4.0/bin/g++
diff --git a/src/auth_manager.cpp b/src/auth_manager.cpp
index e4e6fbd7..af860ed3 100644
--- a/src/auth_manager.cpp
+++ b/src/auth_manager.cpp
@@ -130,8 +130,7 @@ uint32_t AuthManager::get_next_api_key_id() {
bool AuthManager::authenticate(const std::string& req_api_key, const std::string& action,
const std::string& collection, std::map<std::string, std::string>& params) {
- // FIXME:
- LOG(INFO) << "AuthManager::authenticate()";
+ //LOG(INFO) << "AuthManager::authenticate()";
if(req_api_key.size() > KEY_LEN) {
// scoped API key: validate and if valid, extract params
@@ -170,8 +169,7 @@ bool AuthManager::authenticate(const std::string& req_api_key, const std::string
return true;
}
- // FIXME:
- LOG(INFO) << "api_keys.size() = " << api_keys.size();
+ //LOG(INFO) << "api_keys.size() = " << api_keys.size();
if(api_keys.count(req_api_key) == 0) {
return false;
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 24964a26..f0fa153f 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -273,7 +273,7 @@ uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, con
}
void HttpServer::on_res_generator_dispose(void *self) {
- LOG(INFO) << "on_res_generator_dispose fires";
+ //LOG(INFO) << "on_res_generator_dispose fires";
h2o_custom_generator_t* res_generator = static_cast<h2o_custom_generator_t*>(self);
destroy_request_response(res_generator->request, res_generator->response);
}
@@ -554,7 +554,7 @@ int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string
}
void HttpServer::send_response(http_req* request, http_res* response) {
- LOG(INFO) << "send_response, request->_req=" << request->_req;
+ //LOG(INFO) << "send_response, request->_req=" << request->_req;
if(request->_req == nullptr) {
// indicates serialized request and response -- lifecycle must be managed here
@@ -661,17 +661,17 @@ void HttpServer::destroy_request_response(http_req* request, http_res* response)
delete deferred_req_res;
}
- LOG(INFO) << "destroy_request_response, response->proxied_stream=" << response->proxied_stream
- << ", request->_req=" << request->_req << ", response->await=" << &response->await;
+ /*LOG(INFO) << "destroy_request_response, response->proxied_stream=" << response->proxied_stream
+ << ", request->_req=" << request->_req << ", response->await=" << &response->await;*/
if(response->auto_dispose) {
- LOG(INFO) << "destroy_request_response: deleting req/res";
+ //LOG(INFO) << "destroy_request_response: deleting req/res";
delete request;
delete response;
} else {
// lifecycle of proxied/replicated resources are managed externally
// we will just nullify _req to indicate that original request is dead
- LOG(INFO) << "Ignoring request/response cleanup since auto_dispose is false.";
+ //LOG(INFO) << "Ignoring request/response cleanup since auto_dispose is false.";
response->final = true;
request->_req = nullptr;
response->await.notify();
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 2ccfa6d8..b501dcf4 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -9,6 +9,9 @@
#include <http_client.h>
#include "rocksdb/utilities/checkpoint.h"
+namespace braft {
+ DECLARE_int32(raft_do_snapshot_min_index_gap);
+}
void ReplicationClosure::Run() {
// nothing much to do here since responding to client is handled upstream
@@ -30,6 +33,9 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
return -1;
}
+ // do snapshot only when the gap between applied index and last snapshot index is >= this number
+ braft::FLAGS_raft_do_snapshot_min_index_gap = 20;
+
node_options.election_timeout_ms = election_timeout_ms;
node_options.fsm = this;
node_options.node_owns_fsm = false;
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 0ade9d35..69f510ee 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -220,7 +220,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
// NOTE: braft uses `election_timeout_ms / 2` as the brpc channel `timeout_ms` configuration,
// which in turn is the upper bound for brpc `connect_timeout_ms` value.
// Reference: https://github.com/apache/incubator-brpc/blob/122770d/docs/en/client.md#timeout
- size_t election_timeout_ms = 4000;
+ size_t election_timeout_ms = 5000;
if (replication_state.start(peering_endpoint, api_port, election_timeout_ms, 600, state_dir,
nodes_config_op.get()) != 0) {