Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(gateway): wait for session starts without blocking sockets

Hampuscdb0ca64d03c
diff --git a/fluxer_gateway/src/gateway/gateway_handler.erl b/fluxer_gateway/src/gateway/gateway_handler.erl
index ad2f6954f..3993ec814 100644
--- a/fluxer_gateway/src/gateway/gateway_handler.erl
+++ b/fluxer_gateway/src/gateway/gateway_handler.erl
@@ -22,17 +22,25 @@
voice_queue_timer := reference() | undefined,
pending_identify := {map(), pid()} | undefined,
pending_identify_retry_timer := {reference(), reference()} | undefined,
+ session_start := session_start() | undefined,
+ deferred_payloads := [map()],
connection_acquired => boolean()
}.
+-type session_start() :: {
+ session_manager:start_request(), map(), pid(), {reference(), reference()}
+}.
-type ws_frame() :: {text, binary()} | {binary, binary()}.
--type ws_result() :: {ok, state()} | {[ws_frame() | {close, integer(), binary()}], state()}.
+-type ws_command() :: ws_frame() | {close, integer(), binary()} | {active, boolean()}.
+-type ws_result() :: {ok, state()} | {[ws_command()], state()}.
-type ws_info_result() ::
ws_result()
| {ok, state(), hibernate}
- | {[ws_frame() | {close, integer(), binary()}], state(), hibernate}.
+ | {[ws_command()], state(), hibernate}.
+
+-export_type([state/0, ws_frame/0, ws_result/0, session_start/0]).
--export_type([state/0, ws_frame/0, ws_result/0]).
+-define(MAX_DEFERRED_PAYLOADS, 16).
-spec new_state() -> state().
new_state() ->
@@ -54,7 +62,9 @@ new_state() ->
request_workers => #{},
voice_queue_timer => undefined,
pending_identify => undefined,
- pending_identify_retry_timer => undefined
+ pending_identify_retry_timer => undefined,
+ session_start => undefined,
+ deferred_payloads => []
}.
-spec init(cowboy_req:req(), term()) ->
@@ -102,64 +112,124 @@ websocket_init(State) ->
-spec websocket_handle(
{text, binary()} | {binary, binary()} | term(), state()
-) -> ws_result().
-websocket_handle({text, Text}, State) when is_binary(Text) ->
+) -> ws_info_result().
+websocket_handle(Frame, State) ->
+ after_callback(handle_frame(Frame, State)).
+
+-spec handle_frame({text, binary()} | {binary, binary()} | term(), state()) -> ws_result().
+handle_frame({text, Text}, State) when is_binary(Text) ->
handle_incoming_data(Text, State);
-websocket_handle({binary, Binary}, State) when is_binary(Binary) ->
+handle_frame({binary, Binary}, State) when is_binary(Binary) ->
handle_incoming_data(Binary, State);
-websocket_handle(_, State) ->
+handle_frame(_, State) ->
{ok, State}.
-spec websocket_info(term(), state()) -> ws_info_result().
-websocket_info({heartbeat_check, Token}, State) when is_reference(Token) ->
+websocket_info(Msg, State) ->
+ after_callback(handle_message(Msg, State)).
+
+-spec handle_message(term(), state()) -> ws_info_result().
+handle_message({heartbeat_check, Token}, State) when is_reference(Token) ->
hibernate(gateway_handler_heartbeat:handle_heartbeat_check(Token, State));
-websocket_info({dispatch, ready, Data, Seq}, State) when is_integer(Seq), is_map(Data) ->
+handle_message({dispatch, ready, Data, Seq}, State) when is_integer(Seq), is_map(Data) ->
hibernate(gateway_handler_dispatch:handle_dispatch(ready, Data, Seq, State));
-websocket_info({dispatch, Event, Data, Seq}, State) when
+handle_message({dispatch, Event, Data, Seq}, State) when
is_integer(Seq), is_atom(Event), is_map(Data);
is_integer(Seq), is_binary(Event), is_map(Data);
is_integer(Seq), is_atom(Event), is_list(Data);
is_integer(Seq), is_binary(Event), is_list(Data)
->
gateway_handler_dispatch:handle_dispatch(Event, Data, Seq, State);
-websocket_info({dispatch, Event, null, Seq}, State) when
+handle_message({dispatch, Event, null, Seq}, State) when
is_integer(Seq), is_atom(Event); is_integer(Seq), is_binary(Event)
->
gateway_handler_dispatch:handle_dispatch(Event, null, Seq, State);
-websocket_info({dispatch, Event, {pre_encoded, Bin} = Data, Seq}, State) when
+handle_message({dispatch, Event, {pre_encoded, Bin} = Data, Seq}, State) when
is_integer(Seq), is_atom(Event), is_binary(Bin);
is_integer(Seq), is_binary(Event), is_binary(Bin)
->
gateway_handler_dispatch:handle_dispatch(Event, Data, Seq, State);
-websocket_info({gateway_error, ErrorAtom}, State) when is_atom(ErrorAtom) ->
+handle_message({gateway_error, ErrorAtom}, State) when is_atom(ErrorAtom) ->
gateway_handler_dispatch:handle_gateway_error(ErrorAtom, State);
-websocket_info(rollout_config_changed, State) ->
+handle_message(rollout_config_changed, State) ->
gateway_handler_identify:handle_rollout_config_changed(State);
-websocket_info({retry_pending_identify, Token}, State) when is_reference(Token) ->
+handle_message({retry_pending_identify, Token}, State) when is_reference(Token) ->
gateway_handler_identify:handle_pending_identify_retry(Token, State);
-websocket_info(session_reconnect, State) ->
+handle_message(session_reconnect, State) ->
gateway_handler_dispatch:handle_session_reconnect(State);
-websocket_info({'DOWN', _, process, Pid, _}, #{session_pid := Pid} = State) ->
+handle_message({'DOWN', _, process, Pid, _}, #{session_pid := Pid} = State) ->
gateway_handler_dispatch:handle_session_down(State);
-websocket_info({'DOWN', Ref, process, Pid, Reason}, State) when
+handle_message({'DOWN', Ref, process, Pid, Reason}, State) when
is_reference(Ref), is_pid(Pid)
->
gateway_handler_dispatch:handle_request_worker_down(Ref, Pid, Reason, State);
-websocket_info({gateway_request_worker_timeout, Ref, Type}, State) when is_reference(Ref) ->
+handle_message({gateway_request_worker_timeout, Ref, Type}, State) when is_reference(Ref) ->
gateway_handler_dispatch:handle_request_worker_timeout(Ref, Type, State);
-websocket_info(forum_unreads_unknown_opcode, State) ->
+handle_message(forum_unreads_unknown_opcode, State) ->
gateway_handler_encode:close_with_reason(unknown_opcode, <<"Unknown opcode">>, State);
-websocket_info({process_voice_queue}, State) ->
+handle_message({process_voice_queue}, State) ->
QueueState = State#{voice_queue_timer => undefined},
NewState = gateway_handler_voice:process_queued_voice_updates(QueueState),
{ok, NewState};
-websocket_info(_, State) ->
+handle_message(_, State) ->
{ok, State}.
-spec hibernate(ws_result()) -> ws_info_result().
hibernate({Frames, State}) ->
{Frames, State, hibernate}.
+-spec after_callback(ws_info_result()) -> ws_info_result().
+after_callback({Commands, State, hibernate}) ->
+ {Replayed, NewState} = replay_deferred({Commands, State}),
+ {Replayed, NewState, hibernate};
+after_callback({Commands, State}) ->
+ {Replayed, NewState} = replay_deferred({Commands, State}),
+ case awaiting_session(NewState) of
+ true -> {Replayed, NewState, hibernate};
+ false -> {Replayed, NewState}
+ end.
+
+-spec awaiting_session(state()) -> boolean().
+awaiting_session(#{session_pid := undefined, session_start := SessionStart}) when
+ SessionStart =/= undefined
+->
+ true;
+awaiting_session(#{session_pid := undefined, pending_identify := Pending}) ->
+ Pending =/= undefined;
+awaiting_session(_State) ->
+ false.
+
+-spec replay_deferred(ws_result()) -> {[ws_command()] | ok, state()}.
+replay_deferred(
+ {Commands, #{session_start := undefined, deferred_payloads := [_ | _]} = State}
+) ->
+ Payloads = maps:get(deferred_payloads, State),
+ replay_payloads(Payloads, commands(Commands), State#{deferred_payloads => []});
+replay_deferred(Result) ->
+ Result.
+
+-spec replay_payloads([map()], [ws_command()], state()) -> {[ws_command()], state()}.
+replay_payloads([], Commands, #{session_start := undefined} = State) ->
+ {Commands ++ [{active, true}], State};
+replay_payloads([], Commands, State) ->
+ {Commands, State};
+replay_payloads([Payload | Rest], Commands, State) ->
+ case closes(Commands) of
+ true ->
+ {Commands, State};
+ false ->
+ {More, NewState} = handle_decode({ok, Payload}, State),
+ replay_payloads(Rest, Commands ++ commands(More), NewState)
+ end.
+
+-spec commands([ws_command()] | ok) -> [ws_command()].
+commands(ok) -> [];
+commands(Commands) -> Commands.
+
+-spec closes([ws_command()]) -> boolean().
+closes(Commands) ->
+ lists:any(fun(Command) -> element(1, Command) =:= close end, Commands).
+
-spec terminate(term(), cowboy_req:req(), state() | term()) -> ok.
terminate(Reason, _Req, State) when is_map(State) ->
terminate_with_state(eqwalizer:dynamic_cast(State)),
@@ -294,13 +364,32 @@ handle_decompressed_incoming_data(Data, Encoding, CompressCtx, State) ->
-spec handle_decode({ok, map()} | {error, term()}, state()) -> ws_result().
handle_decode({ok, #{<<"op">> := Op} = Payload}, State) ->
OpAtom = constants:gateway_opcode(Op),
- RateLimitResult = gateway_handler_rate_limit:check_rate_limit(State, OpAtom),
- dispatch_after_rate_limit(RateLimitResult, OpAtom, Payload);
+ case defer_payload(OpAtom, State) of
+ true ->
+ defer(Payload, State);
+ false ->
+ RateLimitResult = gateway_handler_rate_limit:check_rate_limit(State, OpAtom),
+ dispatch_after_rate_limit(RateLimitResult, OpAtom, Payload)
+ end;
handle_decode({ok, _}, State) ->
gateway_handler_encode:close_with_reason(decode_error, <<"Invalid payload">>, State);
handle_decode({error, _Reason}, State) ->
gateway_handler_encode:close_with_reason(decode_error, <<"Decode failed">>, State).
+-spec defer_payload(atom(), state()) -> boolean().
+defer_payload(heartbeat, _State) ->
+ false;
+defer_payload(_OpAtom, _State) ->
+ true.
+
+-spec defer(map(), state()) -> ws_result().
+defer(Payload, #{deferred_payloads := Deferred} = State) ->
+ NewState = State#{deferred_payloads => Deferred ++ [Payload]},
+ case length(Deferred) + 1 >= ?MAX_DEFERRED_PAYLOADS of
+ true -> {[{active, false}], NewState};
+ false -> {ok, NewState}
+ end.
+
-spec dispatch_after_rate_limit(
{ok, state()} | {rate_limited, state()} | {opcode_rate_limited, state()},
atom(),
diff --git a/fluxer_gateway/src/gateway/gateway_handler_identify.erl b/fluxer_gateway/src/gateway/gateway_handler_identify.erl
index 4ee150e27..b5ed2de29 100644
--- a/fluxer_gateway/src/gateway/gateway_handler_identify.erl
+++ b/fluxer_gateway/src/gateway/gateway_handler_identify.erl
@@ -7,7 +7,8 @@
handle_identify/3,
handle_resume/2,
handle_rollout_config_changed/1,
- handle_pending_identify_retry/2
+ handle_pending_identify_retry/2,
+ handle_session_start_message/2
]).
-type state() :: gateway_handler:state().
@@ -282,8 +283,47 @@ start_session(Request, SocketPid, State) ->
start_session_with_drain_guard(true, Request, SocketPid, State) ->
hold_session_start(Request, SocketPid, <<"Gateway draining">>, State);
start_session_with_drain_guard(false, Request, SocketPid, State) ->
- StartResult = session_manager:start(Request, SocketPid),
- handle_session_start_result(StartResult, Request, SocketPid, State).
+ case session_manager:send_start(Request, SocketPid) of
+ {ok, StartRequest} ->
+ {ok, await_session_start(StartRequest, Request, SocketPid, State)};
+ {error, Reason} ->
+ handle_session_start_result({error, Reason}, Request, SocketPid, State)
+ end.
+
+-spec await_session_start(session_manager:start_request(), map(), pid(), state()) -> state().
+await_session_start(StartRequest, Request, SocketPid, State) ->
+ Token = make_ref(),
+ TimerRef = erlang:send_after(
+ session_manager:start_timeout(StartRequest), self(), {session_start_timeout, Token}
+ ),
+ State#{session_start => {StartRequest, Request, SocketPid, {TimerRef, Token}}}.
+
+-spec handle_session_start_message(term(), state()) -> no_reply | ws_result().
+handle_session_start_message(
+ {session_start_timeout, Token},
+ #{session_start := {StartRequest, Request, SocketPid, {_TimerRef, Token}}} = State
+) ->
+ Reply = session_manager:abandon_start(StartRequest),
+ complete_session_start(Reply, Request, SocketPid, State);
+handle_session_start_message(
+ Msg, #{session_start := {StartRequest, Request, SocketPid, Timer}} = State
+) ->
+ case session_manager:check_start_response(Msg, StartRequest, Request, SocketPid) of
+ no_reply ->
+ no_reply;
+ {reply, Reply} ->
+ cancel_timer_value(Timer),
+ complete_session_start(Reply, Request, SocketPid, State);
+ {pending, NextRequest} ->
+ cancel_timer_value(Timer),
+ {ok, await_session_start(NextRequest, Request, SocketPid, State)}
+ end;
+handle_session_start_message(_Msg, _State) ->
+ no_reply.
+
+-spec complete_session_start(term(), map(), pid(), state()) -> ws_result().
+complete_session_start(Reply, Request, SocketPid, State) ->
+ handle_session_start_result(Reply, Request, SocketPid, State#{session_start => undefined}).
-spec handle_session_start_result(term(), map() | undefined, pid() | undefined, state()) ->
ws_result().
@@ -360,7 +400,7 @@ handle_rollout_config_changed(State) ->
handle_pending_identify_retry(Token, State) ->
case maps:get(pending_identify_retry_timer, State, undefined) of
{TimerRef, Token} ->
- cancel_pending_identify_retry_value({TimerRef, Token}),
+ cancel_timer_value({TimerRef, Token}),
retry_pending_identify(State#{pending_identify_retry_timer => undefined});
_ ->
{ok, State}
@@ -429,10 +469,10 @@ clear_pending_identify(State) ->
maybe_cancel_pending_identify_retry(undefined) ->
ok;
maybe_cancel_pending_identify_retry({TimerRef, Token}) when is_reference(TimerRef) ->
- cancel_pending_identify_retry_value({TimerRef, Token}).
+ cancel_timer_value({TimerRef, Token}).
--spec cancel_pending_identify_retry_value({reference(), reference()}) -> ok.
-cancel_pending_identify_retry_value({TimerRef, _Token}) ->
+-spec cancel_timer_value({reference(), reference()}) -> ok.
+cancel_timer_value({TimerRef, _Token}) ->
_ = erlang:cancel_timer(TimerRef, [{async, true}, {info, false}]),
ok.
diff --git a/fluxer_gateway/src/session/session_manager.erl b/fluxer_gateway/src/session/session_manager.erl
index 80798773d..3d06cecae 100644
--- a/fluxer_gateway/src/session/session_manager.erl
+++ b/fluxer_gateway/src/session/session_manager.erl
@@ -8,7 +8,10 @@
-export([
start_link/0,
- start/2,
+ send_start/2,
+ check_start_response/4,
+ abandon_start/1,
+ start_timeout/1,
lookup_or_rehydrate/3,
reconnect_drain/0,
handoff_to_topology/1,
@@ -20,12 +23,16 @@
ensure_shard/2, restart_shard/2
]).
+-export_type([start_request/0]).
+
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
-define(START_TIMEOUT, 135000).
-define(LOOKUP_TIMEOUT, 5000).
-type session_id() :: binary().
+-type start_request() ::
+ {shard | proxy, gen_server:request_id()} | {remote, erpc:request_id()}.
-type shard() :: #{pid := pid(), ref := reference()}.
-type state() :: #{shards := #{non_neg_integer() => shard()}, shard_count := pos_integer()}.
-type handoff_result() :: #{
@@ -37,12 +44,82 @@
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
--spec start(map(), pid()) -> term().
-start(Request, SocketPid) ->
+-spec send_start(map(), pid()) -> {ok, start_request()} | {error, unavailable}.
+send_start(Request, SocketPid) ->
SessionId = maps:get(session_id, Request),
- session_manager_routing:call_owner_manager(
- SessionId, {start, Request, SocketPid}, ?START_TIMEOUT
- ).
+ StartCall = {start, Request, SocketPid},
+ case session_manager_routing:owner_scope(SessionId) of
+ local ->
+ send_local_start(SessionId, StartCall);
+ {remote, OwnerNode} ->
+ Args = [SessionId, StartCall, ?START_TIMEOUT],
+ {ok, {remote, erpc:send_request(OwnerNode, ?MODULE, call_shard, Args)}};
+ unavailable ->
+ {error, unavailable}
+ end.
+
+-spec send_local_start(session_id(), term()) -> {ok, start_request()} | {error, unavailable}.
+send_local_start(SessionId, StartCall) ->
+ case session_manager_routing:shard_pid_from_table(SessionId) of
+ {ok, Pid} -> {ok, {shard, gen_server:send_request(Pid, StartCall)}};
+ error -> send_proxy_start(SessionId, StartCall)
+ end.
+
+-spec send_proxy_start(session_id(), term()) -> {ok, start_request()} | {error, unavailable}.
+send_proxy_start(SessionId, StartCall) ->
+ case fluxer_gateway_sup:role_enabled(sessions) of
+ true ->
+ ProxyCall = {proxy_call, SessionId, StartCall, ?START_TIMEOUT},
+ {ok, {proxy, gen_server:send_request(?MODULE, ProxyCall)}};
+ false ->
+ {error, unavailable}
+ end.
+
+-spec check_start_response(term(), start_request(), map(), pid()) ->
+ no_reply | {reply, term()} | {pending, start_request()}.
+check_start_response(Msg, {remote, ReqId}, _Request, _SocketPid) ->
+ try erpc:check_response(Msg, ReqId) of
+ no_response -> no_reply;
+ {response, Reply} -> {reply, Reply}
+ catch
+ _:_ -> {reply, {error, unavailable}}
+ end;
+check_start_response(Msg, {Kind, ReqId}, Request, SocketPid) ->
+ case gen_server:check_response(Msg, ReqId) of
+ no_reply -> no_reply;
+ {reply, Reply} -> {reply, Reply};
+ {error, _} -> start_request_down(Kind, Request, SocketPid)
+ end.
+
+-spec start_request_down(shard | proxy, map(), pid()) ->
+ {reply, {error, unavailable}} | {pending, start_request()}.
+start_request_down(shard, Request, SocketPid) ->
+ SessionId = maps:get(session_id, Request),
+ case send_proxy_start(SessionId, {start, Request, SocketPid}) of
+ {ok, StartRequest} -> {pending, StartRequest};
+ {error, unavailable} -> {reply, {error, unavailable}}
+ end;
+start_request_down(proxy, _Request, _SocketPid) ->
+ {reply, {error, unavailable}}.
+
+-spec abandon_start(start_request()) -> term().
+abandon_start({remote, ReqId}) ->
+ try erpc:receive_response(ReqId, 0) of
+ Reply -> Reply
+ catch
+ error:{erpc, timeout} -> {error, timeout};
+ _:_ -> {error, unavailable}
+ end;
+abandon_start({_Kind, ReqId}) ->
+ case gen_server:receive_response(ReqId, 0) of
+ {reply, Reply} -> Reply;
+ timeout -> {error, timeout};
+ {error, _} -> {error, unavailable}
+ end.
+
+-spec start_timeout(start_request()) -> pos_integer().
+start_timeout({shard, _ReqId}) -> ?START_TIMEOUT;
+start_timeout({_Kind, _ReqId}) -> ?START_TIMEOUT + 1000.
-spec lookup_or_rehydrate(session_id(), binary(), pid()) ->
{ok, pid()} | {error, not_found} | {error, invalid_token}.