Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

moving gzip handling to leader node only

krunal131346a2aa140b8b
diff --git a/include/raft_server.h b/include/raft_server.h
index 0a9c8368..2ea9c8d0 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -142,6 +142,8 @@ private:
butil::EndPoint peering_endpoint;
+ void handle_gzip(const std::shared_ptr<http_req>& request);
+
public:
static constexpr const char* log_dir_name = "log";
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 24f3df05..2055e6a3 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -646,63 +646,11 @@ int HttpServer::async_req_cb(void *ctx, int is_end_stream) {
return 0;
}
-void HttpServer::handle_gzip(const std::shared_ptr<http_req>& request) {
- if(!request->zstream_initialized) {
- request->zs.zalloc = Z_NULL;
- request->zs.zfree = Z_NULL;
- request->zs.opaque = Z_NULL;
- request->zs.avail_in = 0;
- request->zs.next_in = Z_NULL;
-
- if (inflateInit2(&request->zs, 16 + MAX_WBITS) != Z_OK)
- throw (std::runtime_error("inflateInit failed while decompressing."));
- request->zstream_initialized = true;
- }
-
- if(request->zstream_initialized) {
- std::string outbuffer;
- outbuffer.resize(10 * request->body.size());
-
- request->zs.next_in = (Bytef *) request->body.c_str();
- request->zs.avail_in = request->body.size();
- std::size_t size_uncompressed = 0;
- int ret = 0;
- do {
- request->zs.avail_out = static_cast<unsigned int>(outbuffer.size());
- request->zs.next_out = reinterpret_cast<Bytef *>(&outbuffer[0] + size_uncompressed);
- ret = inflate(&request->zs, Z_FINISH);
- if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
- std::string error_msg = request->zs.msg;
- inflateEnd(&request->zs);
- throw std::runtime_error(error_msg);
- }
-
- size_uncompressed += (outbuffer.size() - request->zs.avail_out);
- } while (request->zs.avail_out == 0);
-
- if (ret == Z_STREAM_END) {
- request->zstream_initialized = false;
- inflateEnd(&request->zs);
- }
-
- outbuffer.resize(size_uncompressed);
-
- request->body = outbuffer;
- request->chunk_len = outbuffer.size();
- }
-}
-
int HttpServer::process_request(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response,
route_path *rpath, const h2o_custom_req_handler_t *handler,
const bool use_meta_thread_pool) {
//LOG(INFO) << "process_request called";
- //check if it's first gzip chunk or is gzip stream initialized
- if(((request->body.size() > 2) &&
- (31 == (int)request->body[0] && -117 == (int)request->body[1])) || request->zstream_initialized) {
- handle_gzip(request);
- }
-
const std::string& root_resource = (rpath->path_parts.empty()) ? "" : rpath->path_parts[0];
if(root_resource == "multi_search") {
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index e882244a..51027bf1 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -180,6 +180,52 @@ string ReplicationState::resolve_node_hosts(const string& nodes_config) {
return final_nodes_config;
}
+void ReplicationState::handle_gzip(const std::shared_ptr<http_req>& request) {
+ if(!request->zstream_initialized) {
+ request->zs.zalloc = Z_NULL;
+ request->zs.zfree = Z_NULL;
+ request->zs.opaque = Z_NULL;
+ request->zs.avail_in = 0;
+ request->zs.next_in = Z_NULL;
+
+ if (inflateInit2(&request->zs, 16 + MAX_WBITS) != Z_OK)
+ throw (std::runtime_error("inflateInit failed while decompressing."));
+ request->zstream_initialized = true;
+ }
+
+ if(request->zstream_initialized) {
+ std::string outbuffer;
+ outbuffer.resize(10 * request->body.size());
+
+ request->zs.next_in = (Bytef *) request->body.c_str();
+ request->zs.avail_in = request->body.size();
+ std::size_t size_uncompressed = 0;
+ int ret = 0;
+ do {
+ request->zs.avail_out = static_cast<unsigned int>(outbuffer.size());
+ request->zs.next_out = reinterpret_cast<Bytef *>(&outbuffer[0] + size_uncompressed);
+ ret = inflate(&request->zs, Z_FINISH);
+ if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {
+ std::string error_msg = request->zs.msg;
+ inflateEnd(&request->zs);
+ throw std::runtime_error(error_msg);
+ }
+
+ size_uncompressed += (outbuffer.size() - request->zs.avail_out);
+ } while (request->zs.avail_out == 0);
+
+ if (ret == Z_STREAM_END) {
+ request->zstream_initialized = false;
+ inflateEnd(&request->zs);
+ }
+
+ outbuffer.resize(size_uncompressed);
+
+ request->body = outbuffer;
+ request->chunk_len = outbuffer.size();
+ }
+}
+
void ReplicationState::write(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response) {
if(shutting_down) {
//LOG(INFO) << "write(), force shutdown";
@@ -220,6 +266,12 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
return write_to_leader(request, response);
}
+ //check if it's first gzip chunk or is gzip stream initialized
+ if(((request->body.size() > 2) &&
+ (31 == (int)request->body[0] && -117 == (int)request->body[1])) || request->zstream_initialized) {
+ handle_gzip(request);
+ }
+
// Serialize request to replicated WAL so that all the nodes in the group receive it as well.
// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.