Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(gateway): shrink per-connection zstd stream contexts

Hampuse78707aef87f
diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl
index a997c5f31..71295320e 100644
--- a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl
+++ b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl
@@ -74,6 +74,7 @@ env_services_config() ->
-spec env_gateway_config() -> map().
env_gateway_config() ->
maps:merge(env_gateway_base_config(), #{
+ <<"zstd_window_log">> => env_int("FLUXER_GATEWAY_ZSTD_WINDOW_LOG", 17),
<<"guild_visitor_sessions_max">> => env_int(
"FLUXER_GATEWAY_GUILD_VISITOR_SESSIONS_MAX",
visitor_limit_default(guild_visitor_sessions_max)
@@ -348,6 +349,7 @@ build_misc_config(Service) ->
#{
voice_state_counts_sync_interval_ms =>
get_int(Service, <<"voice_state_counts_sync_interval_ms">>, 30000),
+ zstd_window_log => get_int(Service, <<"zstd_window_log">>, 17),
logger_level => get_log_level(Service, <<"logger_level">>, info)
}.
diff --git a/fluxer_gateway/src/gateway/gateway_compress.erl b/fluxer_gateway/src/gateway/gateway_compress.erl
index 7fa6cbee6..3ad0716f6 100644
--- a/fluxer_gateway/src/gateway/gateway_compress.erl
+++ b/fluxer_gateway/src/gateway/gateway_compress.erl
@@ -20,7 +20,11 @@
-define(MAX_DECOMPRESSED_SIZE, 10 * 1024 * 1024).
-define(ZSTD_STREAM_BUFFER_SIZE, 64 * 1024).
-define(ZSTD_COMPRESSION_LEVEL, 3).
--define(ZSTD_DECOMPRESS_WINDOW_LOG_MAX, 23).
+-define(ZSTD_DECOMPRESS_WINDOW_LOG_MAX, 21).
+-define(ZSTD_DEFAULT_WINDOW_LOG, 17).
+-define(ZSTD_MIN_WINDOW_LOG, 10).
+-define(ZSTD_MAX_WINDOW_LOG, 21).
+-define(ZSTD_MAX_TABLE_LOG, 15).
-define(ZSTD_STREAM_MAX_CHUNKS, 1000).
-define(ZSTD_AVAILABLE_KEY, {?MODULE, zstd_available}).
-define(ZSTD_STREAM_AVAILABLE_KEY, {?MODULE, zstd_stream_available}).
@@ -176,25 +180,47 @@ ensure_zstd_stream_context(#{type := zstd_stream, stream_ctx := StreamCtx} = Ctx
ensure_zstd_stream_context(#{type := zstd_stream} = Ctx) ->
case erlang:apply(ezstd, create_compression_context, [?ZSTD_STREAM_BUFFER_SIZE]) of
StreamCtx when is_reference(StreamCtx) ->
- set_zstd_stream_compression_level(StreamCtx, Ctx);
+ set_zstd_stream_compression_parameters(StreamCtx, Ctx);
{error, Reason} ->
{error, {compress_failed, Reason}};
Other ->
{error, {compress_failed, {case_clause, Other}}}
end.
--spec set_zstd_stream_compression_level(reference(), zstd_stream_ctx()) ->
+-spec set_zstd_stream_compression_parameters(reference(), zstd_stream_ctx()) ->
{ok, reference(), zstd_stream_ctx()} | {error, term()}.
-set_zstd_stream_compression_level(StreamCtx, Ctx) ->
- Result = erlang:apply(ezstd, set_compression_parameter, [
- StreamCtx, zstd_c_compression_level, ?ZSTD_COMPRESSION_LEVEL
- ]),
- case Result of
+set_zstd_stream_compression_parameters(StreamCtx, Ctx) ->
+ WindowLog = zstd_window_log(),
+ TableLog = min(?ZSTD_MAX_TABLE_LOG, WindowLog - 1),
+ Parameters = [
+ {zstd_c_compression_level, ?ZSTD_COMPRESSION_LEVEL},
+ {zstd_c_window_log, WindowLog},
+ {zstd_c_hash_log, TableLog},
+ {zstd_c_chain_log, TableLog}
+ ],
+ case apply_zstd_compression_parameters(StreamCtx, Parameters, ok) of
ok -> {ok, StreamCtx, Ctx#{stream_ctx => StreamCtx}};
{error, Reason} -> {error, {compress_failed, Reason}};
Other -> {error, {compress_failed, {case_clause, Other}}}
end.
+-spec apply_zstd_compression_parameters(reference(), [{atom(), integer()}], term()) ->
+ term().
+apply_zstd_compression_parameters(StreamCtx, [{Name, Value} | Rest], ok) ->
+ Result = erlang:apply(ezstd, set_compression_parameter, [StreamCtx, Name, Value]),
+ apply_zstd_compression_parameters(StreamCtx, Rest, Result);
+apply_zstd_compression_parameters(_StreamCtx, _Parameters, Result) ->
+ Result.
+
+-spec zstd_window_log() -> pos_integer().
+zstd_window_log() ->
+ case fluxer_gateway_env:get(zstd_window_log) of
+ WindowLog when is_integer(WindowLog) ->
+ max(?ZSTD_MIN_WINDOW_LOG, min(?ZSTD_MAX_WINDOW_LOG, WindowLog));
+ _ ->
+ ?ZSTD_DEFAULT_WINDOW_LOG
+ end.
+
-spec handle_zstd_stream_compress_result(term(), compress_ctx()) ->
{ok, binary(), compress_ctx()} | {error, term()}.
handle_zstd_stream_compress_result(Compressed, Ctx) when is_binary(Compressed) ->
diff --git a/fluxer_rexctl/internal/stack/files/docker-compose.yml b/fluxer_rexctl/internal/stack/files/docker-compose.yml
index d9c298347..608cb45b3 100644
--- a/fluxer_rexctl/internal/stack/files/docker-compose.yml
+++ b/fluxer_rexctl/internal/stack/files/docker-compose.yml
@@ -575,6 +575,7 @@ services:
FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES: ${FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES:-}
FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY: ${FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY:-}
FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS: ${FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS:-}
+ FLUXER_GATEWAY_ZSTD_WINDOW_LOG: ${FLUXER_GATEWAY_ZSTD_WINDOW_LOG:-}
FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD: ${FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD:-}
FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS: ${FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS:-}
FLUXER_GATEWAY_GUILD_VISITOR_SESSIONS_MAX: ${FLUXER_GATEWAY_GUILD_VISITOR_SESSIONS_MAX:-}