A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
perf(gateway): wait for session starts without blocking sockets
Hampuscdb0ca64d03c
diff --git a/fluxer_gateway/src/gateway/gateway_handler.erl b/fluxer_gateway/src/gateway/gateway_handler.erlindex ad2f6954f..3993ec814 100644--- a/fluxer_gateway/src/gateway/gateway_handler.erl+++ b/fluxer_gateway/src/gateway/gateway_handler.erl@@ -22,17 +22,25 @@voice_queue_timer := reference() | undefined,pending_identify := {map(), pid()} | undefined,pending_identify_retry_timer := {reference(), reference()} | undefined,+ session_start := session_start() | undefined,+ deferred_payloads := [map()],connection_acquired => boolean()}.+-type session_start() :: {+ session_manager:start_request(), map(), pid(), {reference(), reference()}+}.-type ws_frame() :: {text, binary()} | {binary, binary()}.--type ws_result() :: {ok, state()} | {[ws_frame() | {close, integer(), binary()}], state()}.+-type ws_command() :: ws_frame() | {close, integer(), binary()} | {active, boolean()}.+-type ws_result() :: {ok, state()} | {[ws_command()], state()}.-type ws_info_result() ::ws_result()| {ok, state(), hibernate}- | {[ws_frame() | {close, integer(), binary()}], state(), hibernate}.+ | {[ws_command()], state(), hibernate}.++-export_type([state/0, ws_frame/0, ws_result/0, session_start/0]).--export_type([state/0, ws_frame/0, ws_result/0]).+-define(MAX_DEFERRED_PAYLOADS, 16).-spec new_state() -> state().new_state() ->@@ -54,7 +62,9 @@ new_state() ->request_workers => #{},voice_queue_timer => undefined,pending_identify => undefined,- pending_identify_retry_timer => undefined+ pending_identify_retry_timer => undefined,+ session_start => undefined,+ deferred_payloads => []}.-spec init(cowboy_req:req(), term()) ->@@ -102,64 +112,124 @@ websocket_init(State) ->-spec websocket_handle({text, binary()} | {binary, binary()} | term(), state()-) -> ws_result().-websocket_handle({text, Text}, State) when is_binary(Text) ->+) -> ws_info_result().+websocket_handle(Frame, State) ->+ after_callback(handle_frame(Frame, State)).++-spec handle_frame({text, binary()} | {binary, binary()} | term(), state()) -> ws_result().+handle_frame({text, Text}, State) when is_binary(Text) ->handle_incoming_data(Text, State);-websocket_handle({binary, Binary}, State) when is_binary(Binary) ->+handle_frame({binary, Binary}, State) when is_binary(Binary) ->handle_incoming_data(Binary, State);-websocket_handle(_, State) ->+handle_frame(_, State) ->{ok, State}.-spec websocket_info(term(), state()) -> ws_info_result().-websocket_info({heartbeat_check, Token}, State) when is_reference(Token) ->+websocket_info(Msg, State) ->+ after_callback(handle_message(Msg, State)).++-spec handle_message(term(), state()) -> ws_info_result().+handle_message({heartbeat_check, Token}, State) when is_reference(Token) ->hibernate(gateway_handler_heartbeat:handle_heartbeat_check(Token, State));-websocket_info({dispatch, ready, Data, Seq}, State) when is_integer(Seq), is_map(Data) ->+handle_message({dispatch, ready, Data, Seq}, State) when is_integer(Seq), is_map(Data) ->hibernate(gateway_handler_dispatch:handle_dispatch(ready, Data, Seq, State));-websocket_info({dispatch, Event, Data, Seq}, State) when+handle_message({dispatch, Event, Data, Seq}, State) whenis_integer(Seq), is_atom(Event), is_map(Data);is_integer(Seq), is_binary(Event), is_map(Data);is_integer(Seq), is_atom(Event), is_list(Data);is_integer(Seq), is_binary(Event), is_list(Data)->gateway_handler_dispatch:handle_dispatch(Event, Data, Seq, State);-websocket_info({dispatch, Event, null, Seq}, State) when+handle_message({dispatch, Event, null, Seq}, State) whenis_integer(Seq), is_atom(Event); is_integer(Seq), is_binary(Event)->gateway_handler_dispatch:handle_dispatch(Event, null, Seq, State);-websocket_info({dispatch, Event, {pre_encoded, Bin} = Data, Seq}, State) when+handle_message({dispatch, Event, {pre_encoded, Bin} = Data, Seq}, State) whenis_integer(Seq), is_atom(Event), is_binary(Bin);is_integer(Seq), is_binary(Event), is_binary(Bin)->gateway_handler_dispatch:handle_dispatch(Event, Data, Seq, State);-websocket_info({gateway_error, ErrorAtom}, State) when is_atom(ErrorAtom) ->+handle_message({gateway_error, ErrorAtom}, State) when is_atom(ErrorAtom) ->gateway_handler_dispatch:handle_gateway_error(ErrorAtom, State);-websocket_info(rollout_config_changed, State) ->+handle_message(rollout_config_changed, State) ->gateway_handler_identify:handle_rollout_config_changed(State);-websocket_info({retry_pending_identify, Token}, State) when is_reference(Token) ->+handle_message({retry_pending_identify, Token}, State) when is_reference(Token) ->gateway_handler_identify:handle_pending_identify_retry(Token, State);-websocket_info(session_reconnect, State) ->+handle_message(session_reconnect, State) ->gateway_handler_dispatch:handle_session_reconnect(State);-websocket_info({'DOWN', _, process, Pid, _}, #{session_pid := Pid} = State) ->+handle_message({'DOWN', _, process, Pid, _}, #{session_pid := Pid} = State) ->gateway_handler_dispatch:handle_session_down(State);-websocket_info({'DOWN', Ref, process, Pid, Reason}, State) when+handle_message({'DOWN', Ref, process, Pid, Reason}, State) whenis_reference(Ref), is_pid(Pid)->gateway_handler_dispatch:handle_request_worker_down(Ref, Pid, Reason, State);-websocket_info({gateway_request_worker_timeout, Ref, Type}, State) when is_reference(Ref) ->+handle_message({gateway_request_worker_timeout, Ref, Type}, State) when is_reference(Ref) ->gateway_handler_dispatch:handle_request_worker_timeout(Ref, Type, State);-websocket_info(forum_unreads_unknown_opcode, State) ->+handle_message(forum_unreads_unknown_opcode, State) ->gateway_handler_encode:close_with_reason(unknown_opcode, <<"Unknown opcode">>, State);-websocket_info({process_voice_queue}, State) ->+handle_message({process_voice_queue}, State) ->QueueState = State#{voice_queue_timer => undefined},NewState = gateway_handler_voice:process_queued_voice_updates(QueueState),{ok, NewState};-websocket_info(_, State) ->+handle_message(_, State) ->{ok, State}.-spec hibernate(ws_result()) -> ws_info_result().hibernate({Frames, State}) ->{Frames, State, hibernate}.+-spec after_callback(ws_info_result()) -> ws_info_result().+after_callback({Commands, State, hibernate}) ->+ {Replayed, NewState} = replay_deferred({Commands, State}),+ {Replayed, NewState, hibernate};+after_callback({Commands, State}) ->+ {Replayed, NewState} = replay_deferred({Commands, State}),+ case awaiting_session(NewState) of+ true -> {Replayed, NewState, hibernate};+ false -> {Replayed, NewState}+ end.++-spec awaiting_session(state()) -> boolean().+awaiting_session(#{session_pid := undefined, session_start := SessionStart}) when+ SessionStart =/= undefined+->+ true;+awaiting_session(#{session_pid := undefined, pending_identify := Pending}) ->+ Pending =/= undefined;+awaiting_session(_State) ->+ false.++-spec replay_deferred(ws_result()) -> {[ws_command()] | ok, state()}.+replay_deferred(+ {Commands, #{session_start := undefined, deferred_payloads := [_ | _]} = State}+) ->+ Payloads = maps:get(deferred_payloads, State),+ replay_payloads(Payloads, commands(Commands), State#{deferred_payloads => []});+replay_deferred(Result) ->+ Result.++-spec replay_payloads([map()], [ws_command()], state()) -> {[ws_command()], state()}.+replay_payloads([], Commands, #{session_start := undefined} = State) ->+ {Commands ++ [{active, true}], State};+replay_payloads([], Commands, State) ->+ {Commands, State};+replay_payloads([Payload | Rest], Commands, State) ->+ case closes(Commands) of+ true ->+ {Commands, State};+ false ->+ {More, NewState} = handle_decode({ok, Payload}, State),+ replay_payloads(Rest, Commands ++ commands(More), NewState)+ end.++-spec commands([ws_command()] | ok) -> [ws_command()].+commands(ok) -> [];+commands(Commands) -> Commands.++-spec closes([ws_command()]) -> boolean().+closes(Commands) ->+ lists:any(fun(Command) -> element(1, Command) =:= close end, Commands).+-spec terminate(term(), cowboy_req:req(), state() | term()) -> ok.terminate(Reason, _Req, State) when is_map(State) ->terminate_with_state(eqwalizer:dynamic_cast(State)),@@ -294,13 +364,32 @@ handle_decompressed_incoming_data(Data, Encoding, CompressCtx, State) ->-spec handle_decode({ok, map()} | {error, term()}, state()) -> ws_result().handle_decode({ok, #{<<"op">> := Op} = Payload}, State) ->OpAtom = constants:gateway_opcode(Op),- RateLimitResult = gateway_handler_rate_limit:check_rate_limit(State, OpAtom),- dispatch_after_rate_limit(RateLimitResult, OpAtom, Payload);+ case defer_payload(OpAtom, State) of+ true ->+ defer(Payload, State);+ false ->+ RateLimitResult = gateway_handler_rate_limit:check_rate_limit(State, OpAtom),+ dispatch_after_rate_limit(RateLimitResult, OpAtom, Payload)+ end;handle_decode({ok, _}, State) ->gateway_handler_encode:close_with_reason(decode_error, <<"Invalid payload">>, State);handle_decode({error, _Reason}, State) ->gateway_handler_encode:close_with_reason(decode_error, <<"Decode failed">>, State).+-spec defer_payload(atom(), state()) -> boolean().+defer_payload(heartbeat, _State) ->+ false;+defer_payload(_OpAtom, _State) ->+ true.++-spec defer(map(), state()) -> ws_result().+defer(Payload, #{deferred_payloads := Deferred} = State) ->+ NewState = State#{deferred_payloads => Deferred ++ [Payload]},+ case length(Deferred) + 1 >= ?MAX_DEFERRED_PAYLOADS of+ true -> {[{active, false}], NewState};+ false -> {ok, NewState}+ end.+-spec dispatch_after_rate_limit({ok, state()} | {rate_limited, state()} | {opcode_rate_limited, state()},atom(),diff --git a/fluxer_gateway/src/gateway/gateway_handler_identify.erl b/fluxer_gateway/src/gateway/gateway_handler_identify.erlindex 4ee150e27..b5ed2de29 100644--- a/fluxer_gateway/src/gateway/gateway_handler_identify.erl+++ b/fluxer_gateway/src/gateway/gateway_handler_identify.erl@@ -7,7 +7,8 @@handle_identify/3,handle_resume/2,handle_rollout_config_changed/1,- handle_pending_identify_retry/2+ handle_pending_identify_retry/2,+ handle_session_start_message/2]).-type state() :: gateway_handler:state().@@ -282,8 +283,47 @@ start_session(Request, SocketPid, State) ->start_session_with_drain_guard(true, Request, SocketPid, State) ->hold_session_start(Request, SocketPid, <<"Gateway draining">>, State);start_session_with_drain_guard(false, Request, SocketPid, State) ->- StartResult = session_manager:start(Request, SocketPid),- handle_session_start_result(StartResult, Request, SocketPid, State).+ case session_manager:send_start(Request, SocketPid) of+ {ok, StartRequest} ->+ {ok, await_session_start(StartRequest, Request, SocketPid, State)};+ {error, Reason} ->+ handle_session_start_result({error, Reason}, Request, SocketPid, State)+ end.++-spec await_session_start(session_manager:start_request(), map(), pid(), state()) -> state().+await_session_start(StartRequest, Request, SocketPid, State) ->+ Token = make_ref(),+ TimerRef = erlang:send_after(+ session_manager:start_timeout(StartRequest), self(), {session_start_timeout, Token}+ ),+ State#{session_start => {StartRequest, Request, SocketPid, {TimerRef, Token}}}.++-spec handle_session_start_message(term(), state()) -> no_reply | ws_result().+handle_session_start_message(+ {session_start_timeout, Token},+ #{session_start := {StartRequest, Request, SocketPid, {_TimerRef, Token}}} = State+) ->+ Reply = session_manager:abandon_start(StartRequest),+ complete_session_start(Reply, Request, SocketPid, State);+handle_session_start_message(+ Msg, #{session_start := {StartRequest, Request, SocketPid, Timer}} = State+) ->+ case session_manager:check_start_response(Msg, StartRequest, Request, SocketPid) of+ no_reply ->+ no_reply;+ {reply, Reply} ->+ cancel_timer_value(Timer),+ complete_session_start(Reply, Request, SocketPid, State);+ {pending, NextRequest} ->+ cancel_timer_value(Timer),+ {ok, await_session_start(NextRequest, Request, SocketPid, State)}+ end;+handle_session_start_message(_Msg, _State) ->+ no_reply.++-spec complete_session_start(term(), map(), pid(), state()) -> ws_result().+complete_session_start(Reply, Request, SocketPid, State) ->+ handle_session_start_result(Reply, Request, SocketPid, State#{session_start => undefined}).-spec handle_session_start_result(term(), map() | undefined, pid() | undefined, state()) ->ws_result().@@ -360,7 +400,7 @@ handle_rollout_config_changed(State) ->handle_pending_identify_retry(Token, State) ->case maps:get(pending_identify_retry_timer, State, undefined) of{TimerRef, Token} ->- cancel_pending_identify_retry_value({TimerRef, Token}),+ cancel_timer_value({TimerRef, Token}),retry_pending_identify(State#{pending_identify_retry_timer => undefined});_ ->{ok, State}@@ -429,10 +469,10 @@ clear_pending_identify(State) ->maybe_cancel_pending_identify_retry(undefined) ->ok;maybe_cancel_pending_identify_retry({TimerRef, Token}) when is_reference(TimerRef) ->- cancel_pending_identify_retry_value({TimerRef, Token}).+ cancel_timer_value({TimerRef, Token}).--spec cancel_pending_identify_retry_value({reference(), reference()}) -> ok.-cancel_pending_identify_retry_value({TimerRef, _Token}) ->+-spec cancel_timer_value({reference(), reference()}) -> ok.+cancel_timer_value({TimerRef, _Token}) ->_ = erlang:cancel_timer(TimerRef, [{async, true}, {info, false}]),ok.diff --git a/fluxer_gateway/src/session/session_manager.erl b/fluxer_gateway/src/session/session_manager.erlindex 80798773d..3d06cecae 100644--- a/fluxer_gateway/src/session/session_manager.erl+++ b/fluxer_gateway/src/session/session_manager.erl@@ -8,7 +8,10 @@-export([start_link/0,- start/2,+ send_start/2,+ check_start_response/4,+ abandon_start/1,+ start_timeout/1,lookup_or_rehydrate/3,reconnect_drain/0,handoff_to_topology/1,@@ -20,12 +23,16 @@ensure_shard/2, restart_shard/2]).+-export_type([start_request/0]).+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).-define(START_TIMEOUT, 135000).-define(LOOKUP_TIMEOUT, 5000).-type session_id() :: binary().+-type start_request() ::+ {shard | proxy, gen_server:request_id()} | {remote, erpc:request_id()}.-type shard() :: #{pid := pid(), ref := reference()}.-type state() :: #{shards := #{non_neg_integer() => shard()}, shard_count := pos_integer()}.-type handoff_result() :: #{@@ -37,12 +44,82 @@start_link() ->gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).--spec start(map(), pid()) -> term().-start(Request, SocketPid) ->+-spec send_start(map(), pid()) -> {ok, start_request()} | {error, unavailable}.+send_start(Request, SocketPid) ->SessionId = maps:get(session_id, Request),- session_manager_routing:call_owner_manager(- SessionId, {start, Request, SocketPid}, ?START_TIMEOUT- ).+ StartCall = {start, Request, SocketPid},+ case session_manager_routing:owner_scope(SessionId) of+ local ->+ send_local_start(SessionId, StartCall);+ {remote, OwnerNode} ->+ Args = [SessionId, StartCall, ?START_TIMEOUT],+ {ok, {remote, erpc:send_request(OwnerNode, ?MODULE, call_shard, Args)}};+ unavailable ->+ {error, unavailable}+ end.++-spec send_local_start(session_id(), term()) -> {ok, start_request()} | {error, unavailable}.+send_local_start(SessionId, StartCall) ->+ case session_manager_routing:shard_pid_from_table(SessionId) of+ {ok, Pid} -> {ok, {shard, gen_server:send_request(Pid, StartCall)}};+ error -> send_proxy_start(SessionId, StartCall)+ end.++-spec send_proxy_start(session_id(), term()) -> {ok, start_request()} | {error, unavailable}.+send_proxy_start(SessionId, StartCall) ->+ case fluxer_gateway_sup:role_enabled(sessions) of+ true ->+ ProxyCall = {proxy_call, SessionId, StartCall, ?START_TIMEOUT},+ {ok, {proxy, gen_server:send_request(?MODULE, ProxyCall)}};+ false ->+ {error, unavailable}+ end.++-spec check_start_response(term(), start_request(), map(), pid()) ->+ no_reply | {reply, term()} | {pending, start_request()}.+check_start_response(Msg, {remote, ReqId}, _Request, _SocketPid) ->+ try erpc:check_response(Msg, ReqId) of+ no_response -> no_reply;+ {response, Reply} -> {reply, Reply}+ catch+ _:_ -> {reply, {error, unavailable}}+ end;+check_start_response(Msg, {Kind, ReqId}, Request, SocketPid) ->+ case gen_server:check_response(Msg, ReqId) of+ no_reply -> no_reply;+ {reply, Reply} -> {reply, Reply};+ {error, _} -> start_request_down(Kind, Request, SocketPid)+ end.++-spec start_request_down(shard | proxy, map(), pid()) ->+ {reply, {error, unavailable}} | {pending, start_request()}.+start_request_down(shard, Request, SocketPid) ->+ SessionId = maps:get(session_id, Request),+ case send_proxy_start(SessionId, {start, Request, SocketPid}) of+ {ok, StartRequest} -> {pending, StartRequest};+ {error, unavailable} -> {reply, {error, unavailable}}+ end;+start_request_down(proxy, _Request, _SocketPid) ->+ {reply, {error, unavailable}}.++-spec abandon_start(start_request()) -> term().+abandon_start({remote, ReqId}) ->+ try erpc:receive_response(ReqId, 0) of+ Reply -> Reply+ catch+ error:{erpc, timeout} -> {error, timeout};+ _:_ -> {error, unavailable}+ end;+abandon_start({_Kind, ReqId}) ->+ case gen_server:receive_response(ReqId, 0) of+ {reply, Reply} -> Reply;+ timeout -> {error, timeout};+ {error, _} -> {error, unavailable}+ end.++-spec start_timeout(start_request()) -> pos_integer().+start_timeout({shard, _ReqId}) -> ?START_TIMEOUT;+start_timeout({_Kind, _ReqId}) -> ?START_TIMEOUT + 1000.-spec lookup_or_rehydrate(session_id(), binary(), pid()) ->{ok, pid()} | {error, not_found} | {error, invalid_token}.