A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
perf(gateway): shrink per-connection zstd stream contexts
Hampuse78707aef87f
diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl b/fluxer_gateway/src/gateway/fluxer_gateway_config.erlindex a997c5f31..71295320e 100644--- a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl+++ b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl@@ -74,6 +74,7 @@ env_services_config() ->-spec env_gateway_config() -> map().env_gateway_config() ->maps:merge(env_gateway_base_config(), #{+ <<"zstd_window_log">> => env_int("FLUXER_GATEWAY_ZSTD_WINDOW_LOG", 17),<<"guild_visitor_sessions_max">> => env_int("FLUXER_GATEWAY_GUILD_VISITOR_SESSIONS_MAX",visitor_limit_default(guild_visitor_sessions_max)@@ -348,6 +349,7 @@ build_misc_config(Service) ->#{voice_state_counts_sync_interval_ms =>get_int(Service, <<"voice_state_counts_sync_interval_ms">>, 30000),+ zstd_window_log => get_int(Service, <<"zstd_window_log">>, 17),logger_level => get_log_level(Service, <<"logger_level">>, info)}.diff --git a/fluxer_gateway/src/gateway/gateway_compress.erl b/fluxer_gateway/src/gateway/gateway_compress.erlindex 7fa6cbee6..3ad0716f6 100644--- a/fluxer_gateway/src/gateway/gateway_compress.erl+++ b/fluxer_gateway/src/gateway/gateway_compress.erl@@ -20,7 +20,11 @@-define(MAX_DECOMPRESSED_SIZE, 10 * 1024 * 1024).-define(ZSTD_STREAM_BUFFER_SIZE, 64 * 1024).-define(ZSTD_COMPRESSION_LEVEL, 3).--define(ZSTD_DECOMPRESS_WINDOW_LOG_MAX, 23).+-define(ZSTD_DECOMPRESS_WINDOW_LOG_MAX, 21).+-define(ZSTD_DEFAULT_WINDOW_LOG, 17).+-define(ZSTD_MIN_WINDOW_LOG, 10).+-define(ZSTD_MAX_WINDOW_LOG, 21).+-define(ZSTD_MAX_TABLE_LOG, 15).-define(ZSTD_STREAM_MAX_CHUNKS, 1000).-define(ZSTD_AVAILABLE_KEY, {?MODULE, zstd_available}).-define(ZSTD_STREAM_AVAILABLE_KEY, {?MODULE, zstd_stream_available}).@@ -176,25 +180,47 @@ ensure_zstd_stream_context(#{type := zstd_stream, stream_ctx := StreamCtx} = Ctxensure_zstd_stream_context(#{type := zstd_stream} = Ctx) ->case erlang:apply(ezstd, create_compression_context, [?ZSTD_STREAM_BUFFER_SIZE]) ofStreamCtx when is_reference(StreamCtx) ->- set_zstd_stream_compression_level(StreamCtx, Ctx);+ set_zstd_stream_compression_parameters(StreamCtx, Ctx);{error, Reason} ->{error, {compress_failed, Reason}};Other ->{error, {compress_failed, {case_clause, Other}}}end.--spec set_zstd_stream_compression_level(reference(), zstd_stream_ctx()) ->+-spec set_zstd_stream_compression_parameters(reference(), zstd_stream_ctx()) ->{ok, reference(), zstd_stream_ctx()} | {error, term()}.-set_zstd_stream_compression_level(StreamCtx, Ctx) ->- Result = erlang:apply(ezstd, set_compression_parameter, [- StreamCtx, zstd_c_compression_level, ?ZSTD_COMPRESSION_LEVEL- ]),- case Result of+set_zstd_stream_compression_parameters(StreamCtx, Ctx) ->+ WindowLog = zstd_window_log(),+ TableLog = min(?ZSTD_MAX_TABLE_LOG, WindowLog - 1),+ Parameters = [+ {zstd_c_compression_level, ?ZSTD_COMPRESSION_LEVEL},+ {zstd_c_window_log, WindowLog},+ {zstd_c_hash_log, TableLog},+ {zstd_c_chain_log, TableLog}+ ],+ case apply_zstd_compression_parameters(StreamCtx, Parameters, ok) ofok -> {ok, StreamCtx, Ctx#{stream_ctx => StreamCtx}};{error, Reason} -> {error, {compress_failed, Reason}};Other -> {error, {compress_failed, {case_clause, Other}}}end.+-spec apply_zstd_compression_parameters(reference(), [{atom(), integer()}], term()) ->+ term().+apply_zstd_compression_parameters(StreamCtx, [{Name, Value} | Rest], ok) ->+ Result = erlang:apply(ezstd, set_compression_parameter, [StreamCtx, Name, Value]),+ apply_zstd_compression_parameters(StreamCtx, Rest, Result);+apply_zstd_compression_parameters(_StreamCtx, _Parameters, Result) ->+ Result.++-spec zstd_window_log() -> pos_integer().+zstd_window_log() ->+ case fluxer_gateway_env:get(zstd_window_log) of+ WindowLog when is_integer(WindowLog) ->+ max(?ZSTD_MIN_WINDOW_LOG, min(?ZSTD_MAX_WINDOW_LOG, WindowLog));+ _ ->+ ?ZSTD_DEFAULT_WINDOW_LOG+ end.+-spec handle_zstd_stream_compress_result(term(), compress_ctx()) ->{ok, binary(), compress_ctx()} | {error, term()}.handle_zstd_stream_compress_result(Compressed, Ctx) when is_binary(Compressed) ->diff --git a/fluxer_rexctl/internal/stack/files/docker-compose.yml b/fluxer_rexctl/internal/stack/files/docker-compose.ymlindex d9c298347..608cb45b3 100644--- a/fluxer_rexctl/internal/stack/files/docker-compose.yml+++ b/fluxer_rexctl/internal/stack/files/docker-compose.yml@@ -575,6 +575,7 @@ services:FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES: ${FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES:-}FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY: ${FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY:-}FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS: ${FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS:-}+ FLUXER_GATEWAY_ZSTD_WINDOW_LOG: ${FLUXER_GATEWAY_ZSTD_WINDOW_LOG:-}FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD: ${FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD:-}FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS: ${FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS:-}FLUXER_GATEWAY_GUILD_VISITOR_SESSIONS_MAX: ${FLUXER_GATEWAY_GUILD_VISITOR_SESSIONS_MAX:-}