Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
moving gzip handling to leader node only
krunal131346a2aa140b8b
diff --git a/include/raft_server.h b/include/raft_server.hindex 0a9c8368..2ea9c8d0 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -142,6 +142,8 @@ private:butil::EndPoint peering_endpoint;+ void handle_gzip(const std::shared_ptr<http_req>& request);+public:static constexpr const char* log_dir_name = "log";diff --git a/src/http_server.cpp b/src/http_server.cppindex 24f3df05..2055e6a3 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -646,63 +646,11 @@ int HttpServer::async_req_cb(void *ctx, int is_end_stream) {return 0;}-void HttpServer::handle_gzip(const std::shared_ptr<http_req>& request) {- if(!request->zstream_initialized) {- request->zs.zalloc = Z_NULL;- request->zs.zfree = Z_NULL;- request->zs.opaque = Z_NULL;- request->zs.avail_in = 0;- request->zs.next_in = Z_NULL;-- if (inflateInit2(&request->zs, 16 + MAX_WBITS) != Z_OK)- throw (std::runtime_error("inflateInit failed while decompressing."));- request->zstream_initialized = true;- }-- if(request->zstream_initialized) {- std::string outbuffer;- outbuffer.resize(10 * request->body.size());-- request->zs.next_in = (Bytef *) request->body.c_str();- request->zs.avail_in = request->body.size();- std::size_t size_uncompressed = 0;- int ret = 0;- do {- request->zs.avail_out = static_cast<unsigned int>(outbuffer.size());- request->zs.next_out = reinterpret_cast<Bytef *>(&outbuffer[0] + size_uncompressed);- ret = inflate(&request->zs, Z_FINISH);- if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {- std::string error_msg = request->zs.msg;- inflateEnd(&request->zs);- throw std::runtime_error(error_msg);- }-- size_uncompressed += (outbuffer.size() - request->zs.avail_out);- } while (request->zs.avail_out == 0);-- if (ret == Z_STREAM_END) {- request->zstream_initialized = false;- inflateEnd(&request->zs);- }-- outbuffer.resize(size_uncompressed);-- request->body = outbuffer;- request->chunk_len = outbuffer.size();- }-}-int HttpServer::process_request(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response,route_path *rpath, const h2o_custom_req_handler_t *handler,const bool use_meta_thread_pool) {//LOG(INFO) << "process_request called";- //check if it's first gzip chunk or is gzip stream initialized- if(((request->body.size() > 2) &&- (31 == (int)request->body[0] && -117 == (int)request->body[1])) || request->zstream_initialized) {- handle_gzip(request);- }-const std::string& root_resource = (rpath->path_parts.empty()) ? "" : rpath->path_parts[0];if(root_resource == "multi_search") {diff --git a/src/raft_server.cpp b/src/raft_server.cppindex e882244a..51027bf1 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -180,6 +180,52 @@ string ReplicationState::resolve_node_hosts(const string& nodes_config) {return final_nodes_config;}+void ReplicationState::handle_gzip(const std::shared_ptr<http_req>& request) {+ if(!request->zstream_initialized) {+ request->zs.zalloc = Z_NULL;+ request->zs.zfree = Z_NULL;+ request->zs.opaque = Z_NULL;+ request->zs.avail_in = 0;+ request->zs.next_in = Z_NULL;++ if (inflateInit2(&request->zs, 16 + MAX_WBITS) != Z_OK)+ throw (std::runtime_error("inflateInit failed while decompressing."));+ request->zstream_initialized = true;+ }++ if(request->zstream_initialized) {+ std::string outbuffer;+ outbuffer.resize(10 * request->body.size());++ request->zs.next_in = (Bytef *) request->body.c_str();+ request->zs.avail_in = request->body.size();+ std::size_t size_uncompressed = 0;+ int ret = 0;+ do {+ request->zs.avail_out = static_cast<unsigned int>(outbuffer.size());+ request->zs.next_out = reinterpret_cast<Bytef *>(&outbuffer[0] + size_uncompressed);+ ret = inflate(&request->zs, Z_FINISH);+ if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) {+ std::string error_msg = request->zs.msg;+ inflateEnd(&request->zs);+ throw std::runtime_error(error_msg);+ }++ size_uncompressed += (outbuffer.size() - request->zs.avail_out);+ } while (request->zs.avail_out == 0);++ if (ret == Z_STREAM_END) {+ request->zstream_initialized = false;+ inflateEnd(&request->zs);+ }++ outbuffer.resize(size_uncompressed);++ request->body = outbuffer;+ request->chunk_len = outbuffer.size();+ }+}+void ReplicationState::write(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response) {if(shutting_down) {//LOG(INFO) << "write(), force shutdown";@@ -220,6 +266,12 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const stdreturn write_to_leader(request, response);}+ //check if it's first gzip chunk or is gzip stream initialized+ if(((request->body.size() > 2) &&+ (31 == (int)request->body[0] && -117 == (int)request->body[1])) || request->zstream_initialized) {+ handle_gzip(request);+ }+// Serialize request to replicated WAL so that all the nodes in the group receive it as well.// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.