A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
perf(gateway): admit session starts from one ordered queue
Hampus04fd65c733bc
diff --git a/fluxer_gateway/src/gateway/gateway_concurrency.erl b/fluxer_gateway/src/gateway/gateway_concurrency.erlindex fa1a9a194..ee0fbf145 100644--- a/fluxer_gateway/src/gateway/gateway_concurrency.erl+++ b/fluxer_gateway/src/gateway/gateway_concurrency.erl@@ -6,7 +6,7 @@-export([start_link/0,- try_acquire_session_start/0,+ request_session_start/1,release_session_start/0,try_acquire_guild_start/0,release_guild_start/0,@@ -19,7 +19,10 @@-define(GUILD_START_IDX, 2).-type state() :: #{- counters := counters:counters_ref()+ counters := counters:counters_ref(),+ session_queue := queue:queue({pid(), term()}),+ session_holders := #{pid() => non_neg_integer()},+ monitors := #{pid() => reference()}}.-spec start_link() -> {ok, pid()} | {error, term()}.@@ -30,13 +33,13 @@ start_link() ->ignore -> {error, ignore}end.--spec try_acquire_session_start() -> ok | {error, at_capacity}.-try_acquire_session_start() ->- try_acquire(?SESSION_START_IDX, fun gateway_rollout_config:max_concurrent_session_starts/0).+-spec request_session_start(term()) -> ok.+request_session_start(Tag) ->+ gen_server:cast(?MODULE, {request_session_start, self(), Tag}).-spec release_session_start() -> ok.release_session_start() ->- release_counter(?SESSION_START_IDX).+ gen_server:cast(?MODULE, {release_session_start, self()}).-spec try_acquire_guild_start() -> ok | {error, at_capacity}.try_acquire_guild_start() ->@@ -96,7 +99,12 @@ init([]) ->erlang:process_flag(fullsweep_after, 50),Ref = counters:new(2, [write_concurrency]),persistent_term:put(gateway_concurrency_counters, Ref),- {ok, #{counters => Ref}}.+ {ok, #{+ counters => Ref,+ session_queue => queue:new(),+ session_holders => #{},+ monitors => #{}+ }}.-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}.handle_call({acquire, Idx, Limit}, _From, #{counters := Ref} = State) when@@ -107,13 +115,78 @@ handle_call(_Request, _From, State) ->{reply, ok, State}.-spec handle_cast(term(), state()) -> {noreply, state()}.+handle_cast({request_session_start, Pid, Tag}, #{session_queue := Queue} = State) when+ is_pid(Pid)+->+ Monitored = ensure_monitor(Pid, State),+ {noreply, admit_session_starts(Monitored#{session_queue := queue:in({Pid, Tag}, Queue)})};+handle_cast({release_session_start, Pid}, #{session_holders := Holders} = State) when+ is_pid(Pid)+->+ case maps:get(Pid, Holders, 0) of+ Held when Held > 0 ->+ Released = release_held(Pid, 1, State),+ {noreply, admit_session_starts(Released)};+ _ ->+ {noreply, State}+ end;handle_cast(_Msg, State) ->{noreply, State}.-spec handle_info(term(), state()) -> {noreply, state()}.+handle_info({'DOWN', MonitorRef, process, Pid, _Reason}, State) when is_pid(Pid) ->+ #{session_holders := Holders, session_queue := Queue, monitors := Monitors} = State,+ case maps:get(Pid, Monitors, undefined) of+ MonitorRef ->+ Released = release_held(Pid, maps:get(Pid, Holders, 0), State),+ Remaining = queue:filter(fun({QueuedPid, _Tag}) -> QueuedPid =/= Pid end, Queue),+ {noreply,+ admit_session_starts(Released#{+ session_queue := Remaining, monitors := maps:remove(Pid, Monitors)+ })};+ _ ->+ {noreply, State}+ end;handle_info(_Info, State) ->{noreply, State}.+-spec ensure_monitor(pid(), state()) -> state().+ensure_monitor(Pid, #{monitors := Monitors} = State) ->+ case maps:is_key(Pid, Monitors) of+ true -> State;+ false -> State#{monitors := Monitors#{Pid => erlang:monitor(process, Pid)}}+ end.++-spec release_held(pid(), non_neg_integer(), state()) -> state().+release_held(_Pid, 0, State) ->+ State;+release_held(Pid, Count, #{counters := Ref, session_holders := Holders} = State) ->+ counters:sub(Ref, ?SESSION_START_IDX, Count),+ case maps:get(Pid, Holders, 0) - Count of+ Left when Left > 0 -> State#{session_holders := Holders#{Pid => Left}};+ _ -> State#{session_holders := maps:remove(Pid, Holders)}+ end.++-spec admit_session_starts(state()) -> state().+admit_session_starts(State) ->+ admit_session_starts(gateway_rollout_config:max_concurrent_session_starts(), State).++-spec admit_session_starts(integer(), state()) -> state().+admit_session_starts(Limit, #{counters := Ref, session_queue := Queue} = State) ->+ case counters:get(Ref, ?SESSION_START_IDX) < Limit andalso queue:out(Queue) of+ {{value, {Pid, Tag}}, Rest} ->+ Pid ! {session_start_admitted, Tag},+ counters:add(Ref, ?SESSION_START_IDX, 1),+ Holders = maps:get(session_holders, State),+ Admitted = State#{+ session_queue := Rest,+ session_holders := Holders#{Pid => maps:get(Pid, Holders, 0) + 1}+ },+ admit_session_starts(Limit, Admitted);+ _ ->+ State+ end.+-spec terminate(term(), state()) -> ok.terminate(_Reason, _State) ->ok.diff --git a/fluxer_gateway/src/session/session_manager_shard.erl b/fluxer_gateway/src/session/session_manager_shard.erlindex bf1c743fc..9bdfd4b63 100644--- a/fluxer_gateway/src/session/session_manager_shard.erl+++ b/fluxer_gateway/src/session/session_manager_shard.erl@@ -50,9 +50,10 @@request := identify_request(),socket_pid := pid(),froms := [gen_server:from()],- worker_ref := reference(),+ worker_ref := reference() | undefined,worker_token := reference(),timeout_ref := reference(),+ queued_at := integer(),slot_held => boolean()}.@@ -171,12 +172,10 @@ handle_cast(_, State) ->-spec handle_info(Info, State) -> {noreply, state()} whenInfo :: {'DOWN', reference(), process, pid(), term()} | term(),State :: state().-handle_info({identify_slot_acquired, SessionId, WorkerToken}, State) when+handle_info({session_start_admitted, {SessionId, WorkerToken}}, State) whenis_binary(SessionId), is_reference(WorkerToken)->- session_manager_shard_lifecycle:note_identify_slot_acquired(- SessionId, WorkerToken, State- );+ session_manager_shard_lifecycle:admit_identify_fetch(SessionId, WorkerToken, State);handle_info({identify_fetch_result, SessionId, WorkerToken, FetchResult}, State) whenis_binary(SessionId), is_reference(WorkerToken)->@@ -185,10 +184,12 @@ handle_info({identify_fetch_result, SessionId, WorkerToken, FetchResult}, State));handle_info({identify_fetch_result, SessionId, FetchResult}, State) when is_binary(SessionId) ->session_manager_shard_lifecycle:complete_identify_fetch(SessionId, FetchResult, State);-handle_info({identify_fetch_timeout, SessionId, WorkerRef}, State) when- is_binary(SessionId), is_reference(WorkerRef)+handle_info({identify_fetch_timeout, SessionId, WorkerToken}, State) when+ is_binary(SessionId), is_reference(WorkerToken)->- session_manager_shard_lifecycle:handle_identify_fetch_timeout(SessionId, WorkerRef, State);+ session_manager_shard_lifecycle:handle_identify_fetch_timeout(+ SessionId, WorkerToken, State+ );handle_info({'DOWN', Ref, process, Pid, Reason}, State) when is_reference(Ref), is_pid(Pid) ->IdentifyWorkers = maps:get(identify_workers, State),case maps:take(Ref, IdentifyWorkers) ofdiff --git a/fluxer_gateway/src/session/session_manager_shard_lifecycle.erl b/fluxer_gateway/src/session/session_manager_shard_lifecycle.erlindex 617f2cc67..fc6e9e383 100644--- a/fluxer_gateway/src/session/session_manager_shard_lifecycle.erl+++ b/fluxer_gateway/src/session/session_manager_shard_lifecycle.erl@@ -7,7 +7,7 @@handle_start_call/5,start_session_process/4,start_identify_fetch/5,- note_identify_slot_acquired/3,+ admit_identify_fetch/3,complete_identify_fetch/3, complete_identify_fetch/4,handle_identify_fetch_timeout/3,maybe_fail_pending_identify/4,@@ -33,9 +33,10 @@request := map(),socket_pid := pid(),froms := [gen_server:from()],- worker_ref := reference(),+ worker_ref := reference() | undefined,worker_token := reference(),timeout_ref := reference(),+ queued_at := integer(),slot_held => boolean()}.-type start_reply() ::@@ -262,15 +263,10 @@ supersede_identify_fetch(Request, SocketPid, From, State, SessionId, PendingIdenstate()) -> {noreply, state()}.start_identify_fetch_with_waiters(Request, SocketPid, SessionId, Froms, State) ->- ManagerPid = self(),- PeerIP = maps:get(peer_ip, Request),WorkerToken = make_ref(),- {WorkerPid, WorkerRef} = spawn_monitor(fun() ->- erlang:process_flag(fullsweep_after, 0),- run_identify_fetch(ManagerPid, SessionId, WorkerToken, PeerIP, Request)- end),+ ok = gateway_concurrency:request_session_start({SessionId, WorkerToken}),TimeoutRef = erlang:send_after(- ?IDENTIFY_FETCH_TIMEOUT_MS, self(), {identify_fetch_timeout, SessionId, WorkerRef}+ ?IDENTIFY_FETCH_TIMEOUT_MS, self(), {identify_fetch_timeout, SessionId, WorkerToken}),PendingIdentifies = maps:get(pending_identifies, State),NewPending = PendingIdentifies#{@@ -278,75 +274,77 @@ start_identify_fetch_with_waiters(Request, SocketPid, SessionId, Froms, State) -request => Request,socket_pid => SocketPid,froms => Froms,- worker_ref => WorkerRef,+ worker_ref => undefined,worker_token => WorkerToken,timeout_ref => TimeoutRef,+ queued_at => gateway_timings:start(),slot_held => false}},- IdentifyWorkers = maps:get(identify_workers, State),- NewWorkers = IdentifyWorkers#{WorkerRef => {SessionId, WorkerPid}},- {noreply, State#{pending_identifies := NewPending, identify_workers := NewWorkers}}.+ {noreply, State#{pending_identifies := NewPending}}.--spec note_identify_slot_acquired(session_id(), reference(), state()) -> {noreply, state()}.-note_identify_slot_acquired(SessionId, WorkerToken, State) ->+-spec admit_identify_fetch(session_id(), reference(), state()) -> {noreply, state()}.+admit_identify_fetch(SessionId, WorkerToken, State) ->PendingIdentifies = maps:get(pending_identifies, State),case maps:get(SessionId, PendingIdentifies, undefined) of- #{worker_token := WorkerToken} = PendingIdentify ->- Updated = PendingIdentify#{slot_held => true},- NewPending = PendingIdentifies#{SessionId := Updated},- {noreply, State#{pending_identifies := NewPending}};+ #{worker_token := WorkerToken, worker_ref := undefined} = PendingIdentify ->+ admit_pending_identify(+ socket_alive(maps:get(socket_pid, PendingIdentify)),+ SessionId,+ PendingIdentify,+ State+ );_ ->gateway_concurrency:release_session_start(),{noreply, State}end.--spec run_identify_fetch(pid(), session_id(), reference(), term(), map()) -> ok.-run_identify_fetch(ManagerPid, SessionId, WorkerToken, PeerIP, Request) ->- Result = acquire_and_fetch(ManagerPid, SessionId, WorkerToken, Request, PeerIP),+-spec admit_pending_identify(boolean(), session_id(), pending_identify(), state()) ->+ {noreply, state()}.+admit_pending_identify(false, SessionId, PendingIdentify, State) ->+ gateway_concurrency:release_session_start(),+ cancel_identify_timeout(PendingIdentify),+ ManagerPid = self(),+ Request = maps:get(request, PendingIdentify),+ WorkerToken = maps:get(worker_token, PendingIdentify),+ QueuedAt = maps:get(queued_at, PendingIdentify),+ {WorkerPid, WorkerRef} = spawn_monitor(fun() ->+ erlang:process_flag(fullsweep_after, 0),+ run_identify_fetch(ManagerPid, SessionId, WorkerToken, Request, QueuedAt)+ end),+ PendingIdentifies = maps:get(pending_identifies, State),+ Admitted = PendingIdentify#{worker_ref => WorkerRef, slot_held => true},+ IdentifyWorkers = maps:get(identify_workers, State),+ {noreply, State#{+ pending_identifies := PendingIdentifies#{SessionId := Admitted},+ identify_workers := IdentifyWorkers#{WorkerRef => {SessionId, WorkerPid}}+ }}.++-spec run_identify_fetch(pid(), session_id(), reference(), map(), integer()) -> ok.+run_identify_fetch(ManagerPid, SessionId, WorkerToken, Request, QueuedAt) ->+ Result = fetch_identify_data(Request, QueuedAt),ManagerPid ! {identify_fetch_result, SessionId, WorkerToken, Result},ok.--spec acquire_and_fetch(pid(), session_id(), reference(), map(), term()) -> term().-acquire_and_fetch(ManagerPid, SessionId, WorkerToken, Request, PeerIP) ->+-spec fetch_identify_data(map(), integer()) -> {term(), gateway_timings:recorder()}.+fetch_identify_data(Request, QueuedAt) ->GwTimings0 = maps:get(gw_timings, Request, gateway_timings:new()),- AcquireFetchStartedAt = gateway_timings:start(),- AcquireStartedAt = gateway_timings:start(),- AcquireResult = gateway_concurrency:try_acquire_session_start(),- AcquireSpan = gateway_timings:span(- <<"gateway_concurrency:try_acquire_session_start/0">>, AcquireStartedAt+ WaitSpan = gateway_timings:span(+ <<"gateway_concurrency:request_session_start/1">>, QueuedAt),- case AcquireResult of- {error, at_capacity} ->- GwTimings = gateway_timings:record_function(- acquire_and_fetch,- <<"session_manager_shard_lifecycle:acquire_and_fetch/2">>,- AcquireFetchStartedAt,- #{children => [AcquireSpan]},- GwTimings0- ),- {{error, at_capacity}, GwTimings};- ok ->- ManagerPid ! {identify_slot_acquired, SessionId, WorkerToken},- do_fetch_rpc(Request, PeerIP, GwTimings0, AcquireFetchStartedAt, [AcquireSpan])- end.---spec do_fetch_rpc(map(), term(), gateway_timings:recorder(), integer(), [map()]) ->- {term(), gateway_timings:recorder()}.-do_fetch_rpc(Request, PeerIP, GwTimings0, AcquireFetchStartedAt, ParentChildren) ->- RpcStartedAt = gateway_timings:start(),- Result = session_manager_shard_drain:fetch_rpc_data(Request, PeerIP),+ FetchStartedAt = gateway_timings:start(),+ Result = session_manager_shard_drain:fetch_rpc_data(Request, maps:get(peer_ip, Request)),Remote = gateway_timings:api_remote_from_session_result(Result),RpcSpan = gateway_timings:span(<<"session_manager_shard_drain:fetch_rpc_data/2">>,- RpcStartedAt,+ FetchStartedAt,#{remote => Remote}),GwTimings = gateway_timings:record_function(- acquire_and_fetch,- <<"session_manager_shard_lifecycle:acquire_and_fetch/2">>,- AcquireFetchStartedAt,- #{children => ParentChildren ++ [RpcSpan]},+ fetch_identify_data,+ <<"session_manager_shard_lifecycle:fetch_identify_data/2">>,+ QueuedAt,+ #{children => [WaitSpan, RpcSpan]},GwTimings0),{Result, GwTimings}.@@ -405,12 +403,14 @@ release_identify_slot(PendingIdentify) ->end.-spec handle_identify_fetch_timeout(session_id(), reference(), state()) -> {noreply, state()}.-handle_identify_fetch_timeout(SessionId, WorkerRef, State) ->+handle_identify_fetch_timeout(SessionId, WorkerToken, State) ->PendingIdentifies = maps:get(pending_identifies, State),case maps:get(SessionId, PendingIdentifies, undefined) of- #{worker_ref := WorkerRef} = PendingIdentify ->+ #{worker_token := WorkerToken} = PendingIdentify ->reply_to_waiters(maps:get(froms, PendingIdentify, []), {error, timeout}),- State1 = kill_identify_worker(WorkerRef, PendingIdentify, State),+ State1 = kill_identify_worker(+ maps:get(worker_ref, PendingIdentify), PendingIdentify, State+ ),RemainingPending = maps:remove(SessionId, maps:get(pending_identifies, State1)),{noreply, State1#{pending_identifies := RemainingPending}};_Other ->@@ -524,7 +524,9 @@ split_fetch_result({Result, GwTimings} = FetchResult, Request) ->split_fetch_result(Result, Request) ->{Result, maps:get(gw_timings, Request, gateway_timings:new())}.--spec kill_identify_worker(reference(), pending_identify(), state()) -> state().+-spec kill_identify_worker(reference() | undefined, pending_identify(), state()) -> state().+kill_identify_worker(undefined, _PendingIdentify, State) ->+ State;kill_identify_worker(WorkerRef, PendingIdentify, State) ->IdentifyWorkers = maps:get(identify_workers, State),case maps:take(WorkerRef, IdentifyWorkers) of