Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(gateway): reconcile hotpatched member-list and push fixes (#2425)

Hampusdd1d554cc618
diff --git a/fluxer_gateway/src/gateway/gateway_http_client_response.erl b/fluxer_gateway/src/gateway/gateway_http_client_response.erl
index 6b8ec7591..dc69b48e4 100644
--- a/fluxer_gateway/src/gateway/gateway_http_client_response.erl
+++ b/fluxer_gateway/src/gateway/gateway_http_client_response.erl
@@ -38,7 +38,7 @@ allow_circuit_request(CircuitKey, RecoveryTimeoutMs) ->
-spec update_circuit_state_direct({atom(), binary()}, response(), pos_integer()) -> ok.
update_circuit_state_direct(CircuitKey, Result, FailureThreshold) ->
Now = erlang:system_time(millisecond),
- IsFailure = is_countable_failure(Result),
+ IsFailure = is_countable_circuit_failure(CircuitKey, Result),
record_result(CircuitKey, IsFailure, Now, FailureThreshold).
-spec acquire_inflight_slot(atom(), pos_integer()) -> ok | {error, overloaded}.
@@ -185,14 +185,29 @@ retry_update_counter(Table, Key, Op) ->
error:badarg -> {error, badarg}
end.
--spec is_countable_failure(response()) -> boolean().
-is_countable_failure({error, nxdomain}) -> false;
-is_countable_failure({error, {failed_connect, _}}) -> false;
-is_countable_failure({error, timeout}) -> false;
-is_countable_failure({error, {timeout, _}}) -> false;
-is_countable_failure({error, _}) -> true;
-is_countable_failure({ok, StatusCode, _, _}) when StatusCode >= 500 -> true;
-is_countable_failure(_) -> false.
+-spec is_countable_circuit_failure({atom(), binary()}, response()) -> boolean().
+is_countable_circuit_failure({rpc, _Host}, Result) ->
+ is_countable_failure_with_transport(Result);
+is_countable_circuit_failure({_Workload, _Host}, Result) ->
+ is_countable_failure_without_transport(Result).
+
+-spec is_countable_failure_without_transport(response()) -> boolean().
+is_countable_failure_without_transport({error, nxdomain}) -> false;
+is_countable_failure_without_transport({error, {failed_connect, _}}) -> false;
+is_countable_failure_without_transport({error, timeout}) -> false;
+is_countable_failure_without_transport({error, {timeout, _}}) -> false;
+is_countable_failure_without_transport({error, _}) -> true;
+is_countable_failure_without_transport({ok, StatusCode, _, _}) when StatusCode >= 500 -> true;
+is_countable_failure_without_transport(_) -> false.
+
+-spec is_countable_failure_with_transport(response()) -> boolean().
+is_countable_failure_with_transport({error, nxdomain}) -> true;
+is_countable_failure_with_transport({error, {failed_connect, _}}) -> true;
+is_countable_failure_with_transport({error, timeout}) -> true;
+is_countable_failure_with_transport({error, {timeout, _}}) -> true;
+is_countable_failure_with_transport({error, _}) -> true;
+is_countable_failure_with_transport({ok, StatusCode, _, _}) when StatusCode >= 500 -> true;
+is_countable_failure_with_transport(_) -> false.
-spec record_result({atom(), binary()}, boolean(), integer(), pos_integer()) -> ok.
record_result(CircuitKey, IsFailure, Now, FailureThreshold) ->
@@ -314,3 +329,34 @@ safe_delete(Table, Key) ->
catch
error:badarg -> ok
end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+circuit_ignores_transport_failures_for_non_rpc_workloads_test() ->
+ ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {error, nxdomain})),
+ ?assertEqual(
+ false, is_countable_circuit_failure({push, <<"h">>}, {error, {failed_connect, []}})
+ ),
+ ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {error, timeout})),
+ ?assertEqual(
+ false, is_countable_circuit_failure({push, <<"h">>}, {error, {timeout, connect}})
+ ),
+ ?assertEqual(true, is_countable_circuit_failure({push, <<"h">>}, {error, closed})),
+ ?assertEqual(true, is_countable_circuit_failure({push, <<"h">>}, {ok, 500, [], <<>>})),
+ ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {ok, 200, [], <<>>})).
+
+circuit_counts_transport_failures_for_rpc_test() ->
+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, nxdomain})),
+ ?assertEqual(
+ true, is_countable_circuit_failure({rpc, <<"h">>}, {error, {failed_connect, []}})
+ ),
+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, timeout})),
+ ?assertEqual(
+ true, is_countable_circuit_failure({rpc, <<"h">>}, {error, {timeout, connect}})
+ ),
+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, closed})),
+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {ok, 500, [], <<>>})),
+ ?assertEqual(false, is_countable_circuit_failure({rpc, <<"h">>}, {ok, 200, [], <<>>})).
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild.erl b/fluxer_gateway/src/guild/guild.erl
index 00d190746..b44133945 100644
--- a/fluxer_gateway/src/guild/guild.erl
+++ b/fluxer_gateway/src/guild/guild.erl
@@ -9,6 +9,7 @@
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
-define(HIBERNATE_TIMEOUT, 60000).
+-define(VOICE_MEMBERS_TABLE_WARNED, {?MODULE, voice_members_table_unavailable}).
-type guild_state() :: map().
-type call_reply() ::
@@ -46,6 +47,10 @@ handle_call(export_handoff_state, _From, State) ->
{reply, {ok, guild_handoff:export_handoff_state(State)}, State};
handle_call({get_cached_voice_state_by_connection, ConnectionId}, _From, State) ->
handle_cached_voice_state_call(ConnectionId, State);
+handle_call({get_guild_id}, _From, State) ->
+ {reply, maps:get(id, State, undefined), State};
+handle_call({get_voice_guild_state}, _From, State) ->
+ {reply, voice_guild_state(State), State};
handle_call({dispatch, Request}, _From, State) ->
handle_dispatch_call(Request, State);
handle_call({reload, NewData}, _From, State) ->
@@ -603,6 +608,112 @@ require_map(Value) when is_map(Value) ->
require_map(Value) ->
erlang:error({badmap, Value}).
+-spec voice_guild_state(guild_state()) -> map().
+voice_guild_state(State) ->
+ case maps:get(data, State, #{}) of
+ #{members_ets := Tab} = Data ->
+ case voice_members_table_healthy(Tab) of
+ true ->
+ ok = clear_voice_members_table_warning(),
+ project_voice_guild_state(Data, State);
+ false ->
+ voice_members_table_unavailable(State)
+ end;
+ _ ->
+ voice_members_table_unavailable(State)
+ end.
+
+-spec voice_members_table_healthy(term()) -> boolean().
+voice_members_table_healthy(Tab) when is_reference(Tab) ->
+ try ets:info(eqwalizer:dynamic_cast(Tab), owner) of
+ Owner when is_pid(Owner) -> Owner =:= self();
+ _ -> false
+ catch
+ error:badarg -> false
+ end;
+voice_members_table_healthy(_Tab) ->
+ false.
+
+-spec project_voice_guild_state(map(), guild_state()) -> map().
+project_voice_guild_state(Data, State) ->
+ Projected = maps:with(voice_guild_state_keys(), State),
+ project_voice_sessions(Projected#{data => maps:with(voice_guild_data_keys(), Data)}).
+
+%% The sessions map keeps every key it had, so fold order over it is unchanged and the
+%% dispatch order of every voice broadcast is unchanged. Only session ENTRIES are narrowed.
+-spec project_voice_sessions(map()) -> map().
+project_voice_sessions(#{sessions := Sessions} = Projected) when is_map(Sessions) ->
+ Projected#{sessions => maps:map(fun project_voice_session/2, Sessions)};
+project_voice_sessions(Projected) ->
+ Projected.
+
+-spec project_voice_session(term(), term()) -> term().
+project_voice_session(_SessionId, Session) when is_map(Session) ->
+ maps:without(voice_session_drop_keys(), Session);
+project_voice_session(_SessionId, Session) ->
+ Session.
+
+%% Every reader reachable from the voice guild state takes only pending_connect, user_id,
+%% viewable_channels and pid out of a session entry: guild_sessions:filter_sessions_for_channel,
+%% guild_voice_broadcast and guild_virtual_channel_access. These five are read nowhere on that path.
+-spec voice_session_drop_keys() -> [atom()].
+voice_session_drop_keys() ->
+ [active_guilds, user_roles, mref, bot, is_staff].
+
+-spec voice_guild_state_keys() -> [atom() | binary()].
+voice_guild_state_keys() ->
+ [
+ id,
+ <<"id">>,
+ sessions,
+ guild_pid,
+ voice_states,
+ pending_voice_connections,
+ recently_disconnected_voice_states,
+ e2ee_room_keys,
+ virtual_channel_access,
+ virtual_channel_access_pending,
+ virtual_channel_access_preserve,
+ virtual_channel_access_move_pending,
+ test_perm_fun,
+ test_force_disconnect_fun,
+ test_livekit_fun,
+ test_permission_sync_fun
+ ].
+
+-spec voice_guild_data_keys() -> [atom() | binary()].
+voice_guild_data_keys() ->
+ [
+ <<"id">>,
+ <<"guild">>,
+ <<"roles">>,
+ <<"role_index">>,
+ role_perms_cache,
+ <<"channels">>,
+ <<"channel_index">>,
+ overwrite_perms_cache,
+ members_ets
+ ].
+
+-spec voice_members_table_unavailable(guild_state()) -> map().
+voice_members_table_unavailable(State) ->
+ ok = log_voice_members_table_unavailable(maps:get(id, State, undefined)),
+ State.
+
+-spec log_voice_members_table_unavailable(term()) -> ok.
+log_voice_members_table_unavailable(GuildId) ->
+ case erlang:put(?VOICE_MEMBERS_TABLE_WARNED, true) of
+ true ->
+ ok;
+ _ ->
+ logger:warning("guild_voice_members_table_unavailable: guild_id=~p", [GuildId])
+ end.
+
+-spec clear_voice_members_table_warning() -> ok.
+clear_voice_members_table_warning() ->
+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED),
+ ok.
+
-spec maybe_report_crash(term(), term()) -> ok.
maybe_report_crash(normal, _) ->
ok;
@@ -630,4 +741,349 @@ handle_non_voice_exit_other_linked_stops_guild_test() ->
handle_non_voice_exit(OtherPid, boom, State)
).
+voice_guild_state_pins_projected_key_set_test() ->
+ with_voice_projection_state(fun(Full) ->
+ Projected = voice_guild_state(Full),
+ ?assertEqual(
+ lists:sort([
+ id,
+ <<"id">>,
+ data,
+ sessions,
+ guild_pid,
+ voice_states,
+ pending_voice_connections,
+ recently_disconnected_voice_states,
+ e2ee_room_keys,
+ virtual_channel_access,
+ virtual_channel_access_pending,
+ virtual_channel_access_preserve,
+ virtual_channel_access_move_pending
+ ]),
+ lists:sort(maps:keys(Projected))
+ ),
+ ?assertEqual(
+ lists:sort([
+ <<"id">>,
+ <<"guild">>,
+ <<"roles">>,
+ <<"role_index">>,
+ role_perms_cache,
+ <<"channels">>,
+ <<"channel_index">>,
+ overwrite_perms_cache,
+ members_ets
+ ]),
+ lists:sort(maps:keys(maps:get(data, Projected)))
+ )
+ end).
+
+voice_guild_state_drops_non_voice_payload_test() ->
+ with_voice_projection_state(fun(Full) ->
+ Projected = voice_guild_state(Full),
+ ?assertNot(maps:is_key(member_list_subscriptions, Projected)),
+ ?assertNot(maps:is_key(presence_subscriptions, Projected)),
+ ?assertNot(maps:is_key(connected_user_ids, Projected)),
+ Data = maps:get(data, Projected),
+ ?assertNot(maps:is_key(<<"members">>, Data)),
+ ?assertNot(maps:is_key(members_normalized, Data)),
+ ?assertNot(maps:is_key(<<"member_role_index">>, Data)),
+ ?assertNot(maps:is_key(<<"emojis">>, Data))
+ end).
+
+voice_guild_state_preserves_member_permissions_test() ->
+ with_voice_projection_state(fun(Full) ->
+ Projected = voice_guild_state(Full),
+ Expected = guild_permissions:get_member_permissions(10, 500, Full),
+ ?assert(Expected > 0),
+ ?assert(permission_bits:has(Expected, constants:speak_permission())),
+ ?assertEqual(Expected, guild_permissions:get_member_permissions(10, 500, Projected)),
+ ?assertEqual(
+ voice_utils:compute_voice_permissions(10, 500, Full),
+ voice_utils:compute_voice_permissions(10, 500, Projected)
+ ),
+ ?assertEqual(
+ guild_permissions:find_member_by_user_id(10, Full),
+ guild_permissions:find_member_by_user_id(10, Projected)
+ ),
+ ?assertEqual(
+ guild_permissions:can_view_channel(10, 500, undefined, Full),
+ guild_permissions:can_view_channel(10, 500, undefined, Projected)
+ )
+ end).
+
+voice_guild_state_absent_member_still_resolves_to_zero_test() ->
+ with_voice_projection_state(fun(Full) ->
+ Projected = voice_guild_state(Full),
+ ?assertEqual(undefined, guild_permissions:find_member_by_user_id(11, Projected)),
+ ?assertEqual(0, guild_permissions:get_member_permissions(11, 500, Projected)),
+ ?assertEqual(
+ guild_permissions:get_member_permissions(11, 500, Full),
+ guild_permissions:get_member_permissions(11, 500, Projected)
+ )
+ end).
+
+voice_guild_state_put_member_keeps_permissions_resolvable_test() ->
+ with_voice_projection_state(fun(Full) ->
+ Projected = voice_guild_state(Full),
+ Data = maps:get(data, Projected),
+ Member = guild_permissions:find_member_by_user_id(10, Projected),
+ ?assert(is_map(Member)),
+ Rebuilt = guild_data_index:put_member(Member#{<<"mute">> => true}, Data),
+ RebuiltState = Projected#{data => Rebuilt},
+ ?assertEqual(1, map_size(maps:get(<<"members">>, Rebuilt))),
+ ?assertEqual(1, map_size(maps:get(members_normalized, Rebuilt))),
+ ?assertEqual(maps:get(members_ets, Data), maps:get(members_ets, Rebuilt)),
+ ?assertEqual(
+ guild_permissions:get_member_permissions(10, 500, Full),
+ guild_permissions:get_member_permissions(10, 500, RebuiltState)
+ ),
+ ?assertMatch(
+ #{<<"mute">> := true},
+ guild_permissions:find_member_by_user_id(10, RebuiltState)
+ ),
+ ?assertEqual(
+ guild_permissions:can_view_channel(10, 500, undefined, Full),
+ guild_permissions:can_view_channel(10, 500, undefined, RebuiltState)
+ )
+ end).
+
+voice_guild_state_without_members_table_returns_full_state_test() ->
+ with_voice_projection_state(fun(Full) ->
+ Data = maps:get(data, Full),
+ Broken = Full#{data => maps:remove(members_ets, Data)},
+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED),
+ ?assertEqual(Broken, voice_guild_state(Broken)),
+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),
+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED)
+ end).
+
+voice_members_table_warning_rearms_after_recovery_test() ->
+ with_voice_projection_state(fun(Full) ->
+ Data = maps:get(data, Full),
+ Broken = Full#{data => maps:remove(members_ets, Data)},
+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED),
+ _ = voice_guild_state(Broken),
+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),
+ _ = voice_guild_state(Broken),
+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),
+ _ = voice_guild_state(Full),
+ ?assertEqual(undefined, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),
+ _ = voice_guild_state(Broken),
+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),
+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED)
+ end).
+
+voice_guild_state_session_entries_keep_only_reader_keys_test() ->
+ with_voice_projection_state(fun(Base) ->
+ Full = Base#{sessions => voice_projection_sessions()},
+ Projected = voice_guild_state(Full),
+ Sessions = maps:get(sessions, Projected),
+ Session = maps:get(<<"s-view">>, Sessions),
+ ?assertEqual(
+ lists:sort([session_id, user_id, pid, pending_connect, viewable_channels]),
+ lists:sort(maps:keys(Session))
+ ),
+ ?assertNot(maps:is_key(active_guilds, Session)),
+ ?assertNot(maps:is_key(user_roles, Session)),
+ ?assertNot(maps:is_key(mref, Session)),
+ ?assertNot(maps:is_key(bot, Session)),
+ ?assertNot(maps:is_key(is_staff, Session))
+ end).
+
+voice_guild_state_session_projection_matches_reference_test() ->
+ with_voice_projection_state(fun(Base) ->
+ Full = Base#{sessions => voice_projection_sessions()},
+ Reference = reference_voice_guild_state(Full),
+ Projected = voice_guild_state(Full),
+ ?assertEqual(maps:remove(sessions, Reference), maps:remove(sessions, Projected)),
+ RefSessions = maps:get(sessions, Reference),
+ NewSessions = maps:get(sessions, Projected),
+ ?assertEqual(maps:keys(RefSessions), maps:keys(NewSessions)),
+ ?assertEqual(
+ maps:map(fun project_voice_session/2, RefSessions),
+ NewSessions
+ )
+ end).
+
+voice_guild_state_session_readers_see_identical_results_test() ->
+ with_voice_projection_state(fun(Base) ->
+ Full = Base#{sessions => voice_projection_sessions()},
+ Reference = reference_voice_guild_state(Full),
+ Projected = voice_guild_state(Full),
+ Expected = voice_session_reader_result(500, Reference),
+ ?assertEqual(Expected, voice_session_reader_result(500, Projected)),
+ {SessionIds, Pids} = Expected,
+ ?assertEqual([<<"s-cached">>, <<"s-perm">>, <<"s-view">>], lists:sort(SessionIds)),
+ ?assertEqual([self(), self(), self()], Pids),
+ Hidden = voice_session_reader_result(501, Reference),
+ ?assertEqual(Hidden, voice_session_reader_result(501, Projected)),
+ ?assertEqual({[<<"s-cached">>], [self()]}, Hidden)
+ end).
+
+voice_guild_state_session_projection_boundaries_test() ->
+ with_voice_projection_state(fun(Base) ->
+ Bare = #{user_id => 10, pid => self()},
+ Full = Base#{sessions => #{<<"s-bare">> => Bare, <<"s-odd">> => not_a_map}},
+ Sessions = maps:get(sessions, voice_guild_state(Full)),
+ ?assertEqual(Bare, maps:get(<<"s-bare">>, Sessions)),
+ ?assertEqual(not_a_map, maps:get(<<"s-odd">>, Sessions)),
+ NoSessions = maps:remove(sessions, Base),
+ ?assertNot(maps:is_key(sessions, voice_guild_state(NoSessions))),
+ BadSessions = Base#{sessions => not_a_map},
+ ?assertEqual(not_a_map, maps:get(sessions, voice_guild_state(BadSessions)))
+ end).
+
+voice_guild_state_preserves_session_fold_order_at_scale_test() ->
+ with_voice_projection_state(fun voice_session_fold_order_scenario/1).
+
+%% 200 entries forces the hashmap representation, where iteration order is driven by key
+%% hashes. maps:map/2 rewrites values only, so the key set and therefore the order is identical.
+voice_session_fold_order_scenario(Base) ->
+ Sessions = maps:from_list(lists:map(fun voice_scale_session/1, lists:seq(1, 200))),
+ Full = Base#{sessions => Sessions},
+ Projected = voice_guild_state(Full),
+ ?assertEqual(maps:keys(Sessions), maps:keys(maps:get(sessions, Projected))),
+ ?assertEqual(
+ voice_session_reader_result(500, reference_voice_guild_state(Full)),
+ voice_session_reader_result(500, Projected)
+ ).
+
+voice_scale_session(N) ->
+ SessionId = integer_to_binary(N),
+ {SessionId, voice_projection_session(SessionId, 10, #{500 => true}, false)}.
+
+voice_session_drop_keys_never_drops_a_reader_key_test() ->
+ Dropped = voice_session_drop_keys(),
+ Readers = [pending_connect, user_id, viewable_channels, pid],
+ ?assertEqual([], [K || K <- Readers, lists:member(K, Dropped)]).
+
+%% Reproduces the projection exactly as it was before session entries were narrowed, so the
+%% tests above compare the new path against a live oracle rather than a hand-written literal.
+reference_voice_guild_state(State) ->
+ Data = maps:get(data, State, #{}),
+ Projected = maps:with(voice_guild_state_keys(), State),
+ Projected#{data => maps:with(voice_guild_data_keys(), Data)}.
+
+%% The reader tuple is everything the voice path can observe about the sessions map: which
+%% sessions match, in which fold order, and which pids the broadcast dispatches to.
+voice_session_reader_result(ChannelId, State) ->
+ Sessions = maps:get(sessions, State, #{}),
+ Pairs = guild_sessions:filter_sessions_for_channel(Sessions, ChannelId, undefined, State),
+ {[Sid || {Sid, _S} <- Pairs], [maps:get(pid, S) || {_Sid, S} <- Pairs]}.
+
+voice_projection_sessions() ->
+ #{
+ <<"s-view">> => voice_projection_session(<<"s-view">>, 10, #{500 => true}, false),
+ <<"s-perm">> => voice_projection_session(<<"s-perm">>, 10, #{}, false),
+ <<"s-pending">> => voice_projection_session(<<"s-pending">>, 10, #{500 => true}, true),
+ <<"s-stranger">> => voice_projection_session(<<"s-stranger">>, 11, #{}, false),
+ <<"s-cached">> => voice_projection_session(<<"s-cached">>, 10, #{501 => true}, false)
+ }.
+
+voice_projection_session(SessionId, UserId, ViewableChannels, Pending) ->
+ #{
+ session_id => SessionId,
+ user_id => UserId,
+ pid => self(),
+ mref => make_ref(),
+ active_guilds => sets:from_list([42, 43, 44]),
+ user_roles => [77],
+ bot => false,
+ is_staff => false,
+ pending_connect => Pending,
+ viewable_channels => ViewableChannels
+ }.
+
+voice_members_table_warning_key_is_module_scoped_test() ->
+ ?assertEqual({?MODULE, voice_members_table_unavailable}, ?VOICE_MEMBERS_TABLE_WARNED),
+ ?assertNot(is_atom(?VOICE_MEMBERS_TABLE_WARNED)).
+
+with_voice_projection_state(Fun) ->
+ Tab = ets:new(guild_members_data, [set, public, {read_concurrency, true}]),
+ try
+ Fun(voice_projection_state(Tab))
+ after
+ ets:delete(Tab)
+ end.
+
+voice_projection_state(Tab) ->
+ Data = voice_projection_data(),
+ maps:foreach(
+ fun(UserId, Member) -> ets:insert(Tab, {UserId, Member}) end,
+ maps:get(members_normalized, Data)
+ ),
+ #{
+ id => 42,
+ <<"id">> => <<"42">>,
+ data => Data#{members_ets => Tab, <<"emojis">> => [], channels_stale => true},
+ sessions => #{},
+ guild_pid => self(),
+ voice_states => #{},
+ pending_voice_connections => #{},
+ recently_disconnected_voice_states => #{},
+ e2ee_room_keys => #{},
+ virtual_channel_access => #{},
+ virtual_channel_access_pending => #{},
+ virtual_channel_access_preserve => #{},
+ virtual_channel_access_move_pending => #{},
+ presence_subscriptions => #{},
+ member_list_subscriptions => #{},
+ connected_user_ids => sets:new(),
+ counts => #{member_count => 1}
+ }.
+
+voice_projection_data() ->
+ Everyone = #{
+ <<"id">> => <<"42">>,
+ <<"name">> => <<"@everyone">>,
+ <<"permissions">> => integer_to_binary(constants:view_channel_permission())
+ },
+ Speaker = #{
+ <<"id">> => <<"77">>,
+ <<"name">> => <<"speaker">>,
+ <<"permissions">> => integer_to_binary(
+ constants:connect_permission() bor constants:speak_permission()
+ )
+ },
+ Channel = #{
+ <<"id">> => <<"500">>,
+ <<"type">> => 2,
+ <<"name">> => <<"General">>,
+ <<"permission_overwrites">> => [
+ #{
+ <<"id">> => <<"77">>,
+ <<"type">> => 0,
+ <<"allow">> => integer_to_binary(constants:stream_permission()),
+ <<"deny">> => <<"0">>
+ }
+ ]
+ },
+ HiddenChannel = #{
+ <<"id">> => <<"501">>,
+ <<"type">> => 2,
+ <<"name">> => <<"Hidden">>,
+ <<"permission_overwrites">> => [
+ #{
+ <<"id">> => <<"42">>,
+ <<"type">> => 0,
+ <<"allow">> => <<"0">>,
+ <<"deny">> => integer_to_binary(constants:view_channel_permission())
+ }
+ ]
+ },
+ Member = #{
+ <<"user">> => #{<<"id">> => <<"10">>, <<"username">> => <<"speaker">>},
+ <<"roles">> => [<<"77">>],
+ <<"mute">> => false,
+ <<"deaf">> => false
+ },
+ guild_data_index:normalize_data(#{
+ <<"id">> => <<"42">>,
+ <<"guild">> => #{<<"id">> => <<"42">>, <<"owner_id">> => <<"9999">>},
+ <<"roles">> => [Everyone, Speaker],
+ <<"channels">> => [Channel, HiddenChannel],
+ <<"members">> => [Member]
+ }).
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_connect_async.erl b/fluxer_gateway/src/guild/guild_connect_async.erl
index a0d81a736..b75bbe960 100644
--- a/fluxer_gateway/src/guild/guild_connect_async.erl
+++ b/fluxer_gateway/src/guild/guild_connect_async.erl
@@ -644,3 +644,61 @@ send_result(GuildId, Attempt, Result0, SessionPid) ->
end,
SessionPid ! {guild_connect_result, GuildId, Attempt, Reply},
ok.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+pending_connect_state(Sessions) ->
+ #{
+ id => 42,
+ sessions => Sessions,
+ session_connect_pending => #{},
+ session_connect_queue => queue:new(),
+ session_connect_inflight => ?SESSION_CONNECT_MAX_WORKERS,
+ session_connect_max_queue => ?SESSION_CONNECT_DEFAULT_MAX_QUEUE
+ }.
+
+pending_connect_request(SessionId, UserId) ->
+ #{
+ session_id => SessionId,
+ user_id => UserId,
+ session_pid => self(),
+ bot => false,
+ is_staff => false,
+ active_guilds => sets:new()
+ }.
+
+enqueued_session_entry(SessionId, UserId, State0) ->
+ State1 = enqueue_session_connect_async(
+ 42, 1, pending_connect_request(SessionId, UserId), #{}, State0
+ ),
+ maps:get(SessionId, maps:get(sessions, State1)).
+
+enqueue_marks_existing_session_pending_connect_test() ->
+ SessionId = <<"s-existing">>,
+ UserId = 21,
+ MRef = make_ref(),
+ Existing = #{
+ session_id => SessionId,
+ user_id => UserId,
+ pid => self(),
+ mref => MRef,
+ pending_connect => false,
+ active_guilds => sets:new()
+ },
+ Entry = enqueued_session_entry(
+ SessionId, UserId, pending_connect_state(#{SessionId => Existing})
+ ),
+ ?assertEqual(true, maps:get(pending_connect, Entry)),
+ ?assertEqual(MRef, maps:get(mref, Entry)),
+ ?assertEqual(UserId, maps:get(user_id, Entry)).
+
+enqueue_creates_pending_session_entry_test() ->
+ SessionId = <<"s-fresh">>,
+ UserId = 22,
+ Entry = enqueued_session_entry(SessionId, UserId, pending_connect_state(#{})),
+ demonitor(maps:get(mref, Entry), [flush]),
+ ?assertEqual(true, maps:get(pending_connect, Entry)),
+ ?assertEqual(UserId, maps:get(user_id, Entry)).
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_data.erl b/fluxer_gateway/src/guild/guild_data.erl
index 6b2b69ccc..0854cc23e 100644
--- a/fluxer_gateway/src/guild/guild_data.erl
+++ b/fluxer_gateway/src/guild/guild_data.erl
@@ -25,6 +25,7 @@
-define(CONNECT_SNAPSHOT_HEAVY_MEMBER_KEYS, [
<<"members">>, members_normalized, <<"member_role_index">>, members_sorted_ids
]).
+-define(CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, [active_guilds, user_roles, viewable_channels]).
-export_type([guild_state/0, guild_reply/1, user_id/0]).
@@ -167,15 +168,56 @@ build_connect_snapshot(Item, State) ->
],
State
),
- maybe_trim_connect_snapshot(Item, Base, State).
+ project_snapshot_sessions(maybe_trim_connect_snapshot(Item, Base, State)).
+
+-spec project_snapshot_sessions(map()) -> map().
+project_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) ->
+ Projected = maps:map(
+ fun(_SessionId, SessionData) -> project_snapshot_session(SessionData) end,
+ Sessions
+ ),
+ Snapshot#{sessions => Projected};
+project_snapshot_sessions(Snapshot) ->
+ Snapshot.
+
+-spec project_snapshot_session(term()) -> term().
+project_snapshot_session(SessionData) when is_map(SessionData) ->
+ maps:without(?CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, SessionData);
+project_snapshot_session(SessionData) ->
+ SessionData.
-spec maybe_trim_connect_snapshot(map(), map(), guild_state()) -> map().
maybe_trim_connect_snapshot(Item, Base, State) ->
- case has_members_ets(State) of
+ case should_trim_connect_snapshot(State) of
true -> trim_connect_snapshot(Item, Base);
false -> Base
end.
+-spec should_trim_connect_snapshot(guild_state()) -> boolean().
+should_trim_connect_snapshot(State) ->
+ case connect_snapshot_trim_member_threshold() of
+ undefined ->
+ false;
+ Threshold ->
+ member_count_at_least(Threshold, State) andalso has_members_ets(State)
+ end.
+
+-spec member_count_at_least(pos_integer(), guild_state()) -> boolean().
+member_count_at_least(Threshold, State) ->
+ case maps:get(member_count, State, undefined) of
+ Count when is_integer(Count) -> Count >= Threshold;
+ _ -> false
+ end.
+
+-spec connect_snapshot_trim_member_threshold() -> pos_integer() | undefined.
+connect_snapshot_trim_member_threshold() ->
+ case
+ application:get_env(fluxer_gateway, connect_snapshot_trim_member_threshold, undefined)
+ of
+ N when is_integer(N), N > 0 -> N;
+ _ -> undefined
+ end.
+
-spec has_members_ets(guild_state()) -> boolean().
has_members_ets(#{data := #{members_ets := Tab}}) -> is_reference(Tab);
has_members_ets(_) -> false.
@@ -448,3 +490,107 @@ guild_id_wire_value(undefined) ->
null;
guild_id_wire_value(GuildId) ->
integer_to_binary(GuildId).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+projection_snapshot() ->
+ #{
+ sessions => #{
+ <<"s1">> => #{
+ session_id => <<"s1">>,
+ user_id => 7,
+ pid => self(),
+ bot => false,
+ is_staff => true,
+ pending_connect => false,
+ active_guilds => sets:new(),
+ user_roles => [1, 2],
+ viewable_channels => #{10 => true}
+ }
+ }
+ }.
+
+projection_entry(Snapshot) ->
+ maps:get(<<"s1">>, maps:get(sessions, Snapshot)).
+
+project_snapshot_sessions_drops_only_heavy_keys_test() ->
+ Entry = projection_entry(project_snapshot_sessions(projection_snapshot())),
+ ?assertEqual(7, maps:get(user_id, Entry)),
+ ?assertEqual(true, maps:get(is_staff, Entry)),
+ ?assertEqual(false, maps:get(pending_connect, Entry)),
+ ?assertEqual(<<"s1">>, maps:get(session_id, Entry)),
+ ?assertNot(maps:is_key(active_guilds, Entry)),
+ ?assertNot(maps:is_key(user_roles, Entry)),
+ ?assertNot(maps:is_key(viewable_channels, Entry)).
+
+projection_keeps_session_staff_lookup_test() ->
+ Projected = project_snapshot_sessions(projection_snapshot()),
+ ?assertEqual(true, guild_availability_check:is_user_staff(7, Projected)).
+
+projection_leaves_snapshot_data_untouched_test() ->
+ Data = #{<<"members">> => #{7 => #{}}, members_ets => make_ref()},
+ Snapshot = maps:put(data, Data, projection_snapshot()),
+ ?assertEqual(Data, maps:get(data, project_snapshot_sessions(Snapshot))).
+
+projection_keeps_non_map_session_entries_test() ->
+ Snapshot = #{sessions => #{<<"broken">> => not_a_map}},
+ ?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)).
+
+projection_keeps_snapshot_without_sessions_test() ->
+ Snapshot = #{id => 42, member_count => 3},
+ ?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)).
+
+trim_state(MemberCount) ->
+ #{id => 42, member_count => MemberCount, data => #{members_ets => make_ref()}}.
+
+with_trim_threshold(Threshold, Fun) ->
+ application:set_env(fluxer_gateway, connect_snapshot_trim_member_threshold, Threshold),
+ try
+ Fun()
+ after
+ application:unset_env(fluxer_gateway, connect_snapshot_trim_member_threshold)
+ end.
+
+trim_is_off_without_threshold_test() ->
+ application:unset_env(fluxer_gateway, connect_snapshot_trim_member_threshold),
+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435))).
+
+trim_needs_member_count_at_threshold_test() ->
+ with_trim_threshold(20000, fun() ->
+ ?assertEqual(true, should_trim_connect_snapshot(trim_state(49435))),
+ ?assertEqual(true, should_trim_connect_snapshot(trim_state(20000))),
+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(19999))),
+ NoCount = maps:remove(member_count, trim_state(1)),
+ ?assertEqual(false, should_trim_connect_snapshot(NoCount))
+ end).
+
+trim_needs_members_ets_test() ->
+ with_trim_threshold(1, fun() ->
+ ?assertEqual(false, should_trim_connect_snapshot(#{member_count => 10, data => #{}})),
+ ?assertEqual(
+ false,
+ should_trim_connect_snapshot(#{member_count => 10, data => #{members_ets => 7}})
+ ),
+ ?assertEqual(false, should_trim_connect_snapshot(#{member_count => 10}))
+ end).
+
+trim_ignores_invalid_threshold_test() ->
+ with_trim_threshold(0, fun() ->
+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435)))
+ end),
+ with_trim_threshold(not_an_integer, fun() ->
+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435)))
+ end).
+
+trim_and_projection_target_disjoint_keys_test() ->
+ ?assertEqual(
+ [],
+ [
+ Key
+ || Key <- ?CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS,
+ lists:member(Key, ?CONNECT_SNAPSHOT_HEAVY_MEMBER_KEYS)
+ ]
+ ).
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_data_index_members.erl b/fluxer_gateway/src/guild/guild_data_index_members.erl
index f333c1ad1..e43d02739 100644
--- a/fluxer_gateway/src/guild/guild_data_index_members.erl
+++ b/fluxer_gateway/src/guild/guild_data_index_members.erl
@@ -9,6 +9,7 @@
member_list/1,
member_count/1,
member_ids/1,
+ sorted_member_ids/2,
member_role_index/1,
get_member/2,
get_member_ets/2,
@@ -84,6 +85,8 @@ member_list(Data) ->
SortedIds = sorted_member_ids(Data, MemberMap),
[Member || UserId <- SortedIds, {ok, Member} <- [maps:find(UserId, MemberMap)]].
+%% Every member mutation drops members_sorted_ids, so a cache that is present always
+%% equals lists:sort(maps:keys(member_map(Data))) and callers may page it as-is.
-spec sorted_member_ids(term(), #{user_id() => member()}) -> [user_id()].
sorted_member_ids(Data, MemberMap) when is_map(Data) ->
case maps:find(members_sorted_ids, Data) of
@@ -369,3 +372,50 @@ members_ets_table(#{members_ets := Tab}) ->
Tab;
members_ets_table(_) ->
undefined.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+sorted_member_ids_equals_sorted_keys_test() ->
+ lists:foreach(fun assert_sorted_ids_match_keys/1, sorted_ids_fixtures()).
+
+assert_sorted_ids_match_keys(Data) ->
+ MemberMap = member_map(Data),
+ ?assertEqual(lists:sort(maps:keys(MemberMap)), sorted_member_ids(Data, MemberMap)).
+
+sorted_member_ids_cache_dropped_by_mutations_test() ->
+ Base = put_member_map(fixture_member_map([12, 3, 7]), #{}),
+ ?assertEqual([3, 7, 12], maps:get(members_sorted_ids, Base)),
+ ?assertNot(maps:is_key(members_sorted_ids, put_member(fixture_member(5), Base))),
+ ?assertNot(maps:is_key(members_sorted_ids, remove_member(3, Base))).
+
+sorted_ids_fixtures() ->
+ Base = put_member_map(fixture_member_map([12, 3, 7]), #{}),
+ Large = put_member_map(fixture_member_map(fixture_user_ids(40)), #{}),
+ [
+ #{},
+ #{<<"members">> => #{}},
+ #{<<"members">> => fixture_member_list([9, 2, 5])},
+ #{<<"members">> => #{<<"9">> => fixture_member(9), <<"2">> => fixture_member(2)}},
+ #{<<"members">> => fixture_member_list(fixture_user_ids(40))},
+ Base,
+ Large,
+ put_member(fixture_member(5), Base),
+ put_member(fixture_member(500), Large),
+ remove_member(3, Base),
+ put_member_list(fixture_member_list([4, 1]), Base)
+ ].
+
+fixture_user_ids(Count) ->
+ lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)).
+
+fixture_member_map(UserIds) ->
+ maps:from_list([{UserId, fixture_member(UserId)} || UserId <- UserIds]).
+
+fixture_member_list(UserIds) ->
+ [fixture_member(UserId) || UserId <- UserIds].
+
+fixture_member(UserId) ->
+ #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}.
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_data_members.erl b/fluxer_gateway/src/guild/guild_data_members.erl
index e91c51331..35ff5caba 100644
--- a/fluxer_gateway/src/guild/guild_data_members.erl
+++ b/fluxer_gateway/src/guild/guild_data_members.erl
@@ -56,10 +56,15 @@ has_member(#{user_id := UserId}, State) ->
-spec list_guild_members(map(), guild_state()) -> {reply, map(), guild_state()}.
list_guild_members(#{limit := Limit, offset := Offset}, State) ->
Data = guild_data_index:ensure_data_map(State),
- AllMembers = guild_data_index:member_list(Data),
- TotalCount = length(AllMembers),
- PaginatedMembers = paginate_members(AllMembers, Limit, Offset),
- {reply, #{members => PaginatedMembers, total => TotalCount}, State}.
+ MemberMap = guild_data_index:member_map(Data),
+ SortedIds = guild_data_index_members:sorted_member_ids(Data, MemberMap),
+ PageIds = paginate_members(SortedIds, Limit, Offset),
+ PaginatedMembers = lookup_members(PageIds, MemberMap),
+ {reply, #{members => PaginatedMembers, total => map_size(MemberMap)}, State}.
+
+-spec lookup_members([term()], map()) -> [guild_member()].
+lookup_members(UserIds, MemberMap) ->
+ [Member || UserId <- UserIds, {ok, Member} <- [maps:find(UserId, MemberMap)]].
-spec search_guild_members(map(), guild_state()) -> {reply, map(), guild_state()}.
search_guild_members(#{query := Query, limit := Limit}, State) when
@@ -113,23 +118,69 @@ collect_query_match(Member, NormalizedQuery, Remaining, Acc, NextIterator) ->
find_member_by_user_id(UserId, State) ->
guild_permissions:find_member_by_user_id(UserId, State).
--spec paginate_members([guild_member()], non_neg_integer(), non_neg_integer()) ->
- [guild_member()].
-paginate_members(Members, Limit, Offset) ->
- take_page(drop_members(Members, Offset), Limit, []).
+-spec paginate_members([term()], non_neg_integer(), non_neg_integer()) -> [term()].
+paginate_members(Items, Limit, Offset) ->
+ take_page(drop_items(Items, Offset), Limit, []).
--spec drop_members([guild_member()], non_neg_integer()) -> [guild_member()].
-drop_members(Members, 0) ->
- Members;
-drop_members([], _Offset) ->
+-spec drop_items([term()], non_neg_integer()) -> [term()].
+drop_items(Items, 0) ->
+ Items;
+drop_items([], _Offset) ->
[];
-drop_members([_ | Rest], Offset) ->
- drop_members(Rest, Offset - 1).
+drop_items([_ | Rest], Offset) ->
+ drop_items(Rest, Offset - 1).
--spec take_page([guild_member()], non_neg_integer(), [guild_member()]) -> [guild_member()].
+-spec take_page([term()], non_neg_integer(), [term()]) -> [term()].
take_page(_Remaining, 0, Acc) ->
lists:reverse(Acc);
take_page([], _Limit, Acc) ->
lists:reverse(Acc);
-take_page([Member | Rest], Limit, Acc) ->
- take_page(Rest, Limit - 1, [Member | Acc]).
+take_page([Item | Rest], Limit, Acc) ->
+ take_page(Rest, Limit - 1, [Item | Acc]).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+list_guild_members_matches_full_list_paging_test() ->
+ lists:foreach(fun assert_list_page_matches/1, list_paging_cases()).
+
+assert_list_page_matches({State, Limit, Offset}) ->
+ {reply, Reply, _} = list_guild_members(#{limit => Limit, offset => Offset}, State),
+ ?assertEqual(legacy_list_guild_members(Limit, Offset, State), Reply).
+
+legacy_list_guild_members(Limit, Offset, State) ->
+ Data = guild_data_index:ensure_data_map(State),
+ AllMembers = guild_data_index:member_list(Data),
+ #{members => paginate_members(AllMembers, Limit, Offset), total => length(AllMembers)}.
+
+list_paging_cases() ->
+ [
+ {State, Limit, Offset}
+ || State <- list_paging_states(),
+ Limit <- [0, 1, 2, 5, 99],
+ Offset <- [0, 1, 3, 5, 6, 20, 39, 40, 41, 99]
+ ].
+
+list_paging_states() ->
+ Members = paging_members(paging_user_ids(40)),
+ Cached = guild_data_index:normalize_data(#{<<"members">> => Members}),
+ [
+ #{data => Cached},
+ #{data => guild_data_index:put_member(paging_member(500), Cached)},
+ #{data => guild_data_index:remove_member(7, Cached)},
+ #{data => #{<<"members">> => Members}},
+ #{data => guild_data_index:normalize_data(#{<<"members">> => []})},
+ #{data => #{<<"members">> => #{}}},
+ #{}
+ ].
+
+paging_members(UserIds) ->
+ [paging_member(UserId) || UserId <- UserIds].
+
+paging_user_ids(Count) ->
+ lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)).
+
+paging_member(UserId) ->
+ #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}.
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_data_wire.erl b/fluxer_gateway/src/guild/guild_data_wire.erl
index dd7aee96b..cb972d164 100644
--- a/fluxer_gateway/src/guild/guild_data_wire.erl
+++ b/fluxer_gateway/src/guild/guild_data_wire.erl
@@ -5,219 +5,523 @@
-export([payload/1]).
--type field_kind() :: snowflake | permission | snowflake_list | id_string | generic.
--type path() :: [binary()].
+-type value_kind() :: scalar | scalar_list | maybe_scalar_list | id | restrict | generic.
+-type key_kind() :: drop | value_kind().
-spec payload(term()) -> term().
payload(Value) ->
- payload(Value, []).
+ fast_payload(Value, false).
--spec payload(term(), path()) -> term().
-payload(Value, Path) when is_map(Value) ->
+-spec fast_payload(term(), boolean()) -> term().
+fast_payload(Value, Restricted) when is_map(Value) ->
maps:fold(
fun(Key, FieldValue, Acc) ->
- payload_map_field(Path, Key, FieldValue, Acc)
+ fast_map_field(Restricted, Key, FieldValue, Acc)
end,
#{},
Value
);
-payload(Value, Path) when is_list(Value) ->
- [payload(Item, Path) || Item <- Value];
-payload(Value, _Path) ->
+fast_payload(Value, Restricted) when is_list(Value) ->
+ [fast_payload(Item, Restricted) || Item <- Value];
+fast_payload(Value, _Restricted) ->
Value.
--spec payload_map_field(path(), term(), term(), map()) -> map().
-payload_map_field(Path, Key, FieldValue, Acc) ->
- KeyBinary = key_binary(Key),
- case keep_payload_field(KeyBinary) of
- true -> Acc#{payload_key(Key) => payload_field(Path, Key, KeyBinary, FieldValue)};
- false -> Acc
+-spec fast_map_field(boolean(), term(), term(), map()) -> map().
+fast_map_field(Restricted, Key, Value, Acc) when is_binary(Key) ->
+ fast_put(binary_key_kind(Key), Restricted, Key, Value, Acc);
+fast_map_field(Restricted, Key, Value, Acc) when is_atom(Key) ->
+ Name = atom_to_binary(Key, utf8),
+ fast_put(named_or_suffix_kind(Name), Restricted, Name, Value, Acc);
+fast_map_field(Restricted, Key, Value, Acc) when is_integer(Key), Key > 0 ->
+ fast_put(maybe_scalar_list, Restricted, integer_to_binary(Key), Value, Acc);
+fast_map_field(Restricted, Key, Value, Acc) when is_integer(Key) ->
+ fast_put(generic, Restricted, integer_to_binary(Key), Value, Acc);
+fast_map_field(Restricted, Key, Value, Acc) ->
+ fast_put(generic, Restricted, Key, Value, Acc).
+
+-spec fast_put(key_kind(), boolean(), term(), term(), map()) -> map().
+fast_put(drop, _Restricted, _Key, _Value, Acc) ->
+ Acc;
+fast_put(Kind, Restricted, Key, Value, Acc) ->
+ Acc#{Key => fast_field(Kind, Restricted, Value)}.
+
+-spec fast_field(key_kind(), boolean(), term()) -> term().
+fast_field(generic, Restricted, Value) ->
+ fast_payload(Value, Restricted);
+fast_field(scalar, Restricted, Value) ->
+ fast_scalar(Value, Restricted);
+fast_field(scalar_list, Restricted, Value) ->
+ fast_scalar_list(Value, Restricted);
+fast_field(maybe_scalar_list, Restricted, Value) ->
+ fast_maybe_scalar_list(Value, Restricted);
+fast_field(id, false, Value) ->
+ fast_scalar(Value, false);
+fast_field(id, true, Value) ->
+ fast_payload(Value, true);
+fast_field(restrict, _Restricted, Value) ->
+ fast_payload(Value, true).
+
+-spec fast_scalar(term(), boolean()) -> term().
+fast_scalar(Value, _Restricted) when is_integer(Value) ->
+ integer_to_binary(Value);
+fast_scalar(Value, Restricted) ->
+ fast_payload(Value, Restricted).
+
+-spec fast_scalar_list(term(), boolean()) -> term().
+fast_scalar_list(Values, Restricted) when is_list(Values) ->
+ [fast_scalar(Item, Restricted) || Item <- Values];
+fast_scalar_list(Value, Restricted) ->
+ fast_payload(Value, Restricted).
+
+-spec fast_maybe_scalar_list(term(), boolean()) -> term().
+fast_maybe_scalar_list(Value, Restricted) ->
+ case is_fast_scalar_snowflake_list(Value) of
+ true -> fast_scalar_list(Value, Restricted);
+ false -> fast_payload(Value, Restricted)
end.
--spec keep_payload_field(binary() | undefined) -> boolean().
-keep_payload_field(<<"recipient_ids">>) -> false;
-keep_payload_field(<<"role_index">>) -> false;
-keep_payload_field(<<"channel_index">>) -> false;
-keep_payload_field(<<"member_role_index">>) -> false;
-keep_payload_field(<<"role_perms_cache">>) -> false;
-keep_payload_field(<<"overwrite_perms_cache">>) -> false;
-keep_payload_field(_KeyBinary) -> true.
-
--spec payload_key(term()) -> term().
-payload_key(Key) when is_integer(Key) ->
+-spec is_fast_scalar_snowflake_list(term()) -> boolean().
+is_fast_scalar_snowflake_list([]) ->
+ true;
+is_fast_scalar_snowflake_list(Values) when is_list(Values) ->
+ lists:all(fun is_fast_snowflake_scalar/1, Values);
+is_fast_scalar_snowflake_list(_) ->
+ false.
+
+-spec is_fast_snowflake_scalar(term()) -> boolean().
+is_fast_snowflake_scalar(Value) when is_integer(Value), Value > 0 ->
+ true;
+is_fast_snowflake_scalar(<<First, _/binary>> = Value) when First >= $1, First =< $9 ->
+ snowflake_id:is_valid(Value);
+is_fast_snowflake_scalar(_) ->
+ false.
+
+-spec binary_key_kind(binary()) -> key_kind().
+binary_key_kind(<<First, _/binary>> = Key) when First >= $1, First =< $9 ->
+ numeric_binary_key_kind(Key);
+binary_key_kind(Key) ->
+ named_or_suffix_kind(Key).
+
+-spec named_or_suffix_kind(binary()) -> key_kind().
+named_or_suffix_kind(Key) ->
+ case named_key_kind(Key) of
+ unknown -> suffix_key_kind(Key);
+ Kind -> Kind
+ end.
+
+-spec numeric_binary_key_kind(binary()) -> value_kind().
+numeric_binary_key_kind(Key) ->
+ case snowflake_id:is_valid(Key) of
+ true -> maybe_scalar_list;
+ false -> suffix_key_kind(Key)
+ end.
+
+-spec suffix_key_kind(binary()) -> value_kind().
+suffix_key_kind(Key) ->
+ case has_suffix(Key, <<"_ids">>) of
+ true -> scalar_list;
+ false -> trailing_id_key_kind(Key)
+ end.
+
+-spec trailing_id_key_kind(binary()) -> value_kind().
+trailing_id_key_kind(Key) ->
+ case has_suffix(Key, <<"_id">>) of
+ true -> scalar;
+ false -> generic
+ end.
+
+-spec named_key_kind(binary()) -> key_kind() | unknown.
+named_key_kind(<<"id">>) -> id;
+named_key_kind(<<"permissions">>) -> scalar;
+named_key_kind(<<"allow">>) -> scalar;
+named_key_kind(<<"deny">>) -> scalar;
+named_key_kind(<<"session_id">>) -> scalar;
+named_key_kind(<<"connection_id">>) -> scalar;
+named_key_kind(<<"subscription_id">>) -> scalar;
+named_key_kind(<<"app_id">>) -> scalar;
+named_key_kind(<<"device_id">>) -> scalar;
+named_key_kind(<<"region_id">>) -> scalar;
+named_key_kind(<<"server_id">>) -> scalar;
+named_key_kind(<<"target_id">>) -> scalar;
+named_key_kind(<<"mention_roles">>) -> scalar_list;
+named_key_kind(<<"participants">>) -> scalar_list;
+named_key_kind(<<"ringing">>) -> scalar_list;
+named_key_kind(<<"nsfw_emojis">>) -> scalar_list;
+named_key_kind(<<"pinned_dms">>) -> scalar_list;
+named_key_kind(<<"restricted_guilds">>) -> scalar_list;
+named_key_kind(<<"bot_restricted_guilds">>) -> scalar_list;
+named_key_kind(<<"roles">>) -> maybe_scalar_list;
+named_key_kind(<<"mentions">>) -> maybe_scalar_list;
+named_key_kind(<<"recipients">>) -> maybe_scalar_list;
+named_key_kind(<<"guild_folders">>) -> restrict;
+named_key_kind(<<"rtc_regions">>) -> restrict;
+named_key_kind(<<"recipient_ids">>) -> drop;
+named_key_kind(<<"role_index">>) -> drop;
+named_key_kind(<<"channel_index">>) -> drop;
+named_key_kind(<<"member_role_index">>) -> drop;
+named_key_kind(<<"role_perms_cache">>) -> drop;
+named_key_kind(<<"overwrite_perms_cache">>) -> drop;
+named_key_kind(_) -> unknown.
+
+-spec has_suffix(binary(), binary()) -> boolean().
+has_suffix(Value, Suffix) ->
+ Size = byte_size(Value),
+ SuffixSize = byte_size(Suffix),
+ case Size >= SuffixSize of
+ true ->
+ has_suffix(Value, Suffix, Size - SuffixSize, SuffixSize);
+ false ->
+ false
+ end.
+
+-spec has_suffix(binary(), binary(), non_neg_integer(), non_neg_integer()) -> boolean().
+has_suffix(Value, Suffix, PrefixSize, SuffixSize) ->
+ case Value of
+ <<_:PrefixSize/binary, Suffix:SuffixSize/binary>> -> true;
+ _ -> false
+ end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+-type field_kind() :: snowflake | permission | snowflake_list | id_string | generic.
+-type path() :: [binary()].
+
+fast_payload_matches_original_test() ->
+ lists:foreach(fun assert_fast_matches_original/1, corpus()).
+
+fast_payload_matches_original_for_mixed_keys_test() ->
+ lists:foreach(fun assert_fast_matches_original/1, mixed_key_maps()).
+
+fast_payload_matches_original_under_restricted_paths_test() ->
+ lists:foreach(fun assert_fast_matches_original/1, restricted_corpus()).
+
+fast_payload_keeps_restricted_ids_opaque_test() ->
+ Data = #{<<"rtc_regions">> => [#{<<"id">> => <<"us-east">>, <<"guild_id">> => 7}]},
+ ?assertEqual(
+ #{<<"rtc_regions">> => [#{<<"id">> => <<"us-east">>, <<"guild_id">> => <<"7">>}]},
+ fast_payload(Data, false)
+ ).
+
+fast_payload_drops_internal_keys_test() ->
+ Data = #{<<"id">> => 1, <<"role_index">> => #{}, role_perms_cache => #{}},
+ ?assertEqual(#{<<"id">> => <<"1">>}, fast_payload(Data, false)).
+
+pre_encoded_payload_passes_through_unchanged_test() ->
+ Data = {pre_encoded, <<"{}">>},
+ ?assertEqual(Data, payload(Data)).
+
+assert_fast_matches_original(Value) ->
+ ?assertEqual({Value, reference_payload(Value, [])}, {Value, fast_payload(Value, false)}).
+
+corpus() ->
+ [#{Key => Value} || Key <- sample_keys(), Value <- sample_values()].
+
+restricted_corpus() ->
+ lists:flatmap(fun restricted_variants/1, corpus()).
+
+restricted_variants(Map) ->
+ [
+ #{<<"guild_folders">> => [Map]},
+ #{<<"rtc_regions">> => Map},
+ #{guild_folders => #{<<"nested">> => Map}},
+ #{<<"data">> => [#{<<"rtc_regions">> => [Map]}]},
+ #{<<"data">> => #{<<"more">> => Map}}
+ ].
+
+mixed_key_maps() ->
+ [
+ #{<<"id">> => 1, id => 2},
+ #{<<"100">> => [1], 100 => [2]},
+ #{<<"permissions">> => [1], permissions => 2},
+ #{<<"recipient_ids">> => [1], recipient_ids => [2], <<"id">> => 3},
+ #{1 => [2], -1 => [3], 0 => [4]},
+ #{<<"guild_folders">> => [#{<<"id">> => 1, <<"guild_ids">> => [2], id => 3}]},
+ #{<<"roles">> => [1, 2], <<"mentions">> => [#{<<"id">> => 3}], <<"deny">> => 4}
+ ].
+
+sample_keys() ->
+ [
+ <<"id">>,
+ <<"guild_id">>,
+ <<"session_id">>,
+ <<"connection_id">>,
+ <<"subscription_id">>,
+ <<"app_id">>,
+ <<"device_id">>,
+ <<"region_id">>,
+ <<"server_id">>,
+ <<"target_id">>,
+ <<"permissions">>,
+ <<"allow">>,
+ <<"deny">>,
+ <<"roles">>,
+ <<"mentions">>,
+ <<"recipients">>,
+ <<"ringing">>,
+ <<"mention_roles">>,
+ <<"participants">>,
+ <<"nsfw_emojis">>,
+ <<"pinned_dms">>,
+ <<"restricted_guilds">>,
+ <<"bot_restricted_guilds">>,
+ <<"recipient_ids">>,
+ <<"role_index">>,
+ <<"channel_index">>,
+ <<"member_role_index">>,
+ <<"role_perms_cache">>,
+ <<"overwrite_perms_cache">>,
+ <<"guild_folders">>,
+ <<"rtc_regions">>,
+ <<"guild_ids">>,
+ <<"username">>,
+ <<"_id">>,
+ <<"_ids">>,
+ <<"id_">>,
+ <<"9_ids">>,
+ <<"100">>,
+ <<"0100">>,
+ <<"1x">>,
+ <<>>,
+ id,
+ guild_id,
+ session_id,
+ roles,
+ permissions,
+ role_perms_cache,
+ guild_folders,
+ '100',
+ 100,
+ 0,
+ -1,
+ 1.5,
+ {tuple, key}
+ ].
+
+sample_values() ->
+ [
+ 42,
+ -7,
+ 0,
+ <<"text">>,
+ <<"42">>,
+ null,
+ true,
+ undefined,
+ [],
+ [1, 2],
+ [<<"3">>, 4],
+ [<<"abc">>],
+ [[1, 2]],
+ [#{<<"id">> => 5}],
+ #{<<"id">> => 6, <<"session_id">> => <<"s">>, <<"user_id">> => 7},
+ #{<<"id">> => 8, <<"roles">> => [9], <<"role_index">> => #{}},
+ #{<<"nested">> => #{<<"id">> => 10, <<"guild_ids">> => [11]}}
+ ].
+
+-spec reference_payload(term(), path()) -> term().
+reference_payload(Value, Path) when is_map(Value) ->
+ maps:fold(
+ fun(Key, FieldValue, Acc) ->
+ reference_payload_map_field(Path, Key, FieldValue, Acc)
+ end,
+ #{},
+ Value
+ );
+reference_payload(Value, Path) when is_list(Value) ->
+ [reference_payload(Item, Path) || Item <- Value];
+reference_payload(Value, _Path) ->
+ Value.
+
+-spec reference_payload_map_field(path(), term(), term(), map()) -> map().
+reference_payload_map_field(Path, Key, FieldValue, Acc) ->
+ case reference_keep_payload_field(Key) of
+ true ->
+ Acc#{reference_payload_key(Key) => reference_payload_field(Path, Key, FieldValue)};
+ false ->
+ Acc
+ end.
+
+-spec reference_keep_payload_field(term()) -> boolean().
+reference_keep_payload_field(Key) ->
+ not lists:member(reference_key_binary(Key), [
+ <<"recipient_ids">>,
+ <<"role_index">>,
+ <<"channel_index">>,
+ <<"member_role_index">>,
+ <<"role_perms_cache">>,
+ <<"overwrite_perms_cache">>
+ ]).
+
+-spec reference_payload_key(term()) -> term().
+reference_payload_key(Key) when is_integer(Key) ->
integer_to_binary(Key);
-payload_key(Key) when is_atom(Key) ->
+reference_payload_key(Key) when is_atom(Key) ->
atom_to_binary(Key, utf8);
-payload_key(Key) ->
+reference_payload_key(Key) ->
Key.
--spec payload_field(path(), term(), binary() | undefined, term()) -> term().
-payload_field(Path, Key, KeyBinary, Value) ->
- FieldPath = path_push(KeyBinary, Path),
- case field_kind(Path, Key, KeyBinary, Value) of
- snowflake -> payload_snowflake(Value, FieldPath);
- permission -> payload_permission(Value, FieldPath);
- snowflake_list -> payload_snowflake_list(Value, FieldPath);
- id_string -> payload_id_string(Value, FieldPath);
- generic -> payload(Value, FieldPath)
+-spec reference_payload_field(path(), term(), term()) -> term().
+reference_payload_field(Path, Key, Value) ->
+ FieldPath = reference_path_push(Key, Path),
+ case reference_field_kind(Path, Key, Value) of
+ snowflake -> reference_payload_snowflake(Value, FieldPath);
+ permission -> reference_payload_permission(Value, FieldPath);
+ snowflake_list -> reference_payload_snowflake_list(Value, FieldPath);
+ id_string -> reference_payload_id_string(Value, FieldPath);
+ generic -> reference_payload(Value, FieldPath)
end.
--spec payload_snowflake(term(), path()) -> term().
-payload_snowflake(Value, _Path) when is_integer(Value) ->
+-spec reference_payload_snowflake(term(), path()) -> term().
+reference_payload_snowflake(Value, _Path) when is_integer(Value) ->
integer_to_binary(Value);
-payload_snowflake(Value, Path) ->
- payload(Value, Path).
+reference_payload_snowflake(Value, Path) ->
+ reference_payload(Value, Path).
--spec payload_permission(term(), path()) -> term().
-payload_permission(Value, _Path) when is_integer(Value) ->
+-spec reference_payload_permission(term(), path()) -> term().
+reference_payload_permission(Value, _Path) when is_integer(Value) ->
integer_to_binary(Value);
-payload_permission(Value, Path) ->
- payload(Value, Path).
+reference_payload_permission(Value, Path) ->
+ reference_payload(Value, Path).
--spec payload_snowflake_list(term(), path()) -> term().
-payload_snowflake_list(Values, Path) when is_list(Values) ->
- [payload_snowflake(Item, Path) || Item <- Values];
-payload_snowflake_list(Value, Path) ->
- payload(Value, Path).
+-spec reference_payload_snowflake_list(term(), path()) -> term().
+reference_payload_snowflake_list(Values, Path) when is_list(Values) ->
+ [reference_payload_snowflake(Item, Path) || Item <- Values];
+reference_payload_snowflake_list(Value, Path) ->
+ reference_payload(Value, Path).
--spec payload_id_string(term(), path()) -> term().
-payload_id_string(Value, _Path) when is_integer(Value) ->
+-spec reference_payload_id_string(term(), path()) -> term().
+reference_payload_id_string(Value, _Path) when is_integer(Value) ->
integer_to_binary(Value);
-payload_id_string(Value, Path) ->
- payload(Value, Path).
+reference_payload_id_string(Value, Path) ->
+ reference_payload(Value, Path).
--spec field_kind(path(), term(), binary() | undefined, term()) -> field_kind().
-field_kind(Path, Key, KeyBinary, Value) ->
- case is_snowflake_record_list_value(Key, Value) of
+-spec reference_field_kind(path(), term(), term()) -> field_kind().
+reference_field_kind(Path, Key, Value) ->
+ case reference_is_snowflake_record_list_value(Key, Value) of
true -> snowflake_list;
- false -> field_kind_binary(Path, KeyBinary, Value)
+ false -> reference_field_kind_binary(Path, reference_key_binary(Key), Value)
end.
--spec field_kind_binary(path(), binary() | undefined, term()) -> field_kind().
-field_kind_binary(_Path, <<"permissions">>, _Value) ->
+-spec reference_field_kind_binary(path(), binary() | undefined, term()) -> field_kind().
+reference_field_kind_binary(_Path, <<"permissions">>, _Value) ->
permission;
-field_kind_binary(_Path, <<"allow">>, _Value) ->
+reference_field_kind_binary(_Path, <<"allow">>, _Value) ->
permission;
-field_kind_binary(_Path, <<"deny">>, _Value) ->
+reference_field_kind_binary(_Path, <<"deny">>, _Value) ->
permission;
-field_kind_binary(Path, Key, Value) when is_binary(Key) ->
- field_kind_for_binary(Path, Key, Value);
-field_kind_binary(_Path, _Key, _Value) ->
+reference_field_kind_binary(Path, Key, Value) when is_binary(Key) ->
+ reference_field_kind_for_binary(Path, Key, Value);
+reference_field_kind_binary(_Path, _Key, _Value) ->
generic.
--spec field_kind_for_binary(path(), binary(), term()) -> field_kind().
-field_kind_for_binary(Path, Key, Value) ->
- case is_snowflake_list_key(Key, Value) of
+-spec reference_field_kind_for_binary(path(), binary(), term()) -> field_kind().
+reference_field_kind_for_binary(Path, Key, Value) ->
+ case reference_is_snowflake_list_key(Key, Value) of
true -> snowflake_list;
- false -> field_kind_scalar(Path, Key)
+ false -> reference_field_kind_scalar(Path, Key)
end.
--spec field_kind_scalar(path(), binary()) -> snowflake | id_string | generic.
-field_kind_scalar(Path, Key) ->
- case is_snowflake_key(Path, Key) of
+-spec reference_field_kind_scalar(path(), binary()) -> snowflake | id_string | generic.
+reference_field_kind_scalar(Path, Key) ->
+ case reference_is_snowflake_key(Path, Key) of
true -> snowflake;
- false -> opaque_id_kind(Key)
+ false -> reference_opaque_id_kind(Key)
end.
--spec opaque_id_kind(binary()) -> id_string | generic.
-opaque_id_kind(Key) ->
- case is_opaque_id_key(Key) of
+-spec reference_opaque_id_kind(binary()) -> id_string | generic.
+reference_opaque_id_kind(Key) ->
+ case reference_is_opaque_id_key(Key) of
true -> id_string;
false -> generic
end.
--spec is_snowflake_key(path(), binary()) -> boolean().
-is_snowflake_key(Path, <<"id">>) ->
- not has_any_path([<<"guild_folders">>, <<"rtc_regions">>], Path);
-is_snowflake_key(_Path, Key) ->
- has_suffix(Key, <<"_id">>) andalso not is_opaque_id_key(Key).
-
--spec is_opaque_id_key(binary()) -> boolean().
-is_opaque_id_key(<<"session_id">>) -> true;
-is_opaque_id_key(<<"connection_id">>) -> true;
-is_opaque_id_key(<<"subscription_id">>) -> true;
-is_opaque_id_key(<<"app_id">>) -> true;
-is_opaque_id_key(<<"device_id">>) -> true;
-is_opaque_id_key(<<"region_id">>) -> true;
-is_opaque_id_key(<<"server_id">>) -> true;
-is_opaque_id_key(<<"target_id">>) -> true;
-is_opaque_id_key(_) -> false.
-
--spec is_snowflake_list_key(binary(), term()) -> boolean().
-is_snowflake_list_key(<<"mention_roles">>, _Value) -> true;
-is_snowflake_list_key(<<"participants">>, _Value) -> true;
-is_snowflake_list_key(<<"ringing">>, _Value) -> true;
-is_snowflake_list_key(<<"nsfw_emojis">>, _Value) -> true;
-is_snowflake_list_key(<<"pinned_dms">>, _Value) -> true;
-is_snowflake_list_key(<<"restricted_guilds">>, _Value) -> true;
-is_snowflake_list_key(<<"bot_restricted_guilds">>, _Value) -> true;
-is_snowflake_list_key(<<"roles">>, Value) -> is_scalar_snowflake_list(Value);
-is_snowflake_list_key(<<"mentions">>, Value) -> is_scalar_snowflake_list(Value);
-is_snowflake_list_key(<<"recipients">>, Value) -> is_scalar_snowflake_list(Value);
-is_snowflake_list_key(Key, _Value) -> has_suffix(Key, <<"_ids">>).
-
--spec is_snowflake_record_list_value(term(), term()) -> boolean().
-is_snowflake_record_list_value(Key, Value) ->
- key_is_snowflake(Key) andalso is_scalar_snowflake_list(Value).
-
--spec key_is_snowflake(term()) -> boolean().
-key_is_snowflake(Key) when is_integer(Key), Key > 0 ->
+-spec reference_is_snowflake_key(path(), binary()) -> boolean().
+reference_is_snowflake_key(Path, <<"id">>) ->
+ not reference_has_any_path([<<"guild_folders">>, <<"rtc_regions">>], Path);
+reference_is_snowflake_key(_Path, Key) ->
+ has_suffix(Key, <<"_id">>) andalso not reference_is_opaque_id_key(Key).
+
+-spec reference_is_opaque_id_key(binary()) -> boolean().
+reference_is_opaque_id_key(<<"session_id">>) -> true;
+reference_is_opaque_id_key(<<"connection_id">>) -> true;
+reference_is_opaque_id_key(<<"subscription_id">>) -> true;
+reference_is_opaque_id_key(<<"app_id">>) -> true;
+reference_is_opaque_id_key(<<"device_id">>) -> true;
+reference_is_opaque_id_key(<<"region_id">>) -> true;
+reference_is_opaque_id_key(<<"server_id">>) -> true;
+reference_is_opaque_id_key(<<"target_id">>) -> true;
+reference_is_opaque_id_key(_) -> false.
+
+-spec reference_is_snowflake_list_key(binary(), term()) -> boolean().
+reference_is_snowflake_list_key(<<"mention_roles">>, _Value) ->
+ true;
+reference_is_snowflake_list_key(<<"participants">>, _Value) ->
+ true;
+reference_is_snowflake_list_key(<<"ringing">>, _Value) ->
+ true;
+reference_is_snowflake_list_key(<<"nsfw_emojis">>, _Value) ->
+ true;
+reference_is_snowflake_list_key(<<"pinned_dms">>, _Value) ->
+ true;
+reference_is_snowflake_list_key(<<"restricted_guilds">>, _Value) ->
+ true;
+reference_is_snowflake_list_key(<<"bot_restricted_guilds">>, _Value) ->
+ true;
+reference_is_snowflake_list_key(<<"roles">>, Value) ->
+ reference_is_scalar_snowflake_list(Value);
+reference_is_snowflake_list_key(<<"mentions">>, Value) ->
+ reference_is_scalar_snowflake_list(Value);
+reference_is_snowflake_list_key(<<"recipients">>, Value) ->
+ reference_is_scalar_snowflake_list(Value);
+reference_is_snowflake_list_key(Key, _Value) ->
+ has_suffix(Key, <<"_ids">>).
+
+-spec reference_is_snowflake_record_list_value(term(), term()) -> boolean().
+reference_is_snowflake_record_list_value(Key, Value) ->
+ reference_key_is_snowflake(Key) andalso reference_is_scalar_snowflake_list(Value).
+
+-spec reference_key_is_snowflake(term()) -> boolean().
+reference_key_is_snowflake(Key) when is_integer(Key), Key > 0 ->
true;
-key_is_snowflake(<<First, _/binary>>) when First < $1; First > $9 ->
- false;
-key_is_snowflake(Key) when is_binary(Key) ->
+reference_key_is_snowflake(Key) when is_binary(Key) ->
snowflake_id:is_valid(Key);
-key_is_snowflake(_) ->
+reference_key_is_snowflake(_) ->
false.
--spec is_scalar_snowflake_list(term()) -> boolean().
-is_scalar_snowflake_list([]) ->
+-spec reference_is_scalar_snowflake_list(term()) -> boolean().
+reference_is_scalar_snowflake_list([]) ->
true;
-is_scalar_snowflake_list(Values) when is_list(Values) ->
- lists:all(fun is_snowflake_scalar/1, Values);
-is_scalar_snowflake_list(_) ->
+reference_is_scalar_snowflake_list(Values) when is_list(Values) ->
+ lists:all(fun reference_is_snowflake_scalar/1, Values);
+reference_is_scalar_snowflake_list(_) ->
false.
--spec is_snowflake_scalar(term()) -> boolean().
-is_snowflake_scalar(Value) when is_integer(Value), Value > 0 ->
+-spec reference_is_snowflake_scalar(term()) -> boolean().
+reference_is_snowflake_scalar(Value) when is_integer(Value), Value > 0 ->
true;
-is_snowflake_scalar(Value) when is_binary(Value) ->
+reference_is_snowflake_scalar(Value) when is_binary(Value) ->
snowflake_id:is_valid(Value);
-is_snowflake_scalar(_) ->
+reference_is_snowflake_scalar(_) ->
false.
--spec has_suffix(binary(), binary()) -> boolean().
-has_suffix(Value, Suffix) ->
- Size = byte_size(Value),
- SuffixSize = byte_size(Suffix),
- case Size >= SuffixSize of
- true ->
- has_suffix(Value, Suffix, Size - SuffixSize, SuffixSize);
- false ->
- false
- end.
-
--spec has_suffix(binary(), binary(), non_neg_integer(), non_neg_integer()) -> boolean().
-has_suffix(Value, Suffix, PrefixSize, SuffixSize) ->
- case Value of
- <<_:PrefixSize/binary, Suffix:SuffixSize/binary>> -> true;
- _ -> false
- end.
-
--spec key_binary(term()) -> binary() | undefined.
-key_binary(Key) when is_binary(Key) ->
+-spec reference_key_binary(term()) -> binary() | undefined.
+reference_key_binary(Key) when is_binary(Key) ->
Key;
-key_binary(Key) when is_atom(Key) ->
+reference_key_binary(Key) when is_atom(Key) ->
atom_to_binary(Key, utf8);
-key_binary(_) ->
+reference_key_binary(_) ->
undefined.
--spec path_push(binary() | undefined, path()) -> path().
-path_push(undefined, Path) ->
- Path;
-path_push(Binary, Path) ->
- [Binary | Path].
+-spec reference_path_push(term(), path()) -> path().
+reference_path_push(Key, Path) ->
+ case reference_key_binary(Key) of
+ undefined -> Path;
+ Binary -> [Binary | Path]
+ end.
--spec has_any_path([binary()], path()) -> boolean().
-has_any_path(Keys, Path) ->
+-spec reference_has_any_path([binary()], path()) -> boolean().
+reference_has_any_path(Keys, Path) ->
lists:any(fun(Key) -> lists:member(Key, Path) end, Keys).
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_dispatch_member_list.erl b/fluxer_gateway/src/guild/guild_dispatch_member_list.erl
index 88441d289..08a2b14fe 100644
--- a/fluxer_gateway/src/guild/guild_dispatch_member_list.erl
+++ b/fluxer_gateway/src/guild/guild_dispatch_member_list.erl
@@ -43,12 +43,12 @@ dispatch_member_list_event(channel_create, _EventData, _OldState, UpdatedState)
broadcast_all_updates(UpdatedState);
dispatch_member_list_event(channel_delete, _EventData, _OldState, UpdatedState) ->
broadcast_all_updates(UpdatedState);
-dispatch_member_list_event(channel_update, EventData, _OldState, UpdatedState) ->
- broadcast_channel_update(EventData, UpdatedState);
-dispatch_member_list_event(channel_update_bulk, EventData, _OldState, UpdatedState) ->
+dispatch_member_list_event(channel_update, EventData, OldState, UpdatedState) ->
+ broadcast_channel_update(EventData, OldState, UpdatedState);
+dispatch_member_list_event(channel_update_bulk, EventData, OldState, UpdatedState) ->
Channels = maps:get(<<"channels">>, EventData, []),
lists:foldl(
- fun broadcast_channel_update/2,
+ fun(Channel, AccState) -> broadcast_channel_update(Channel, OldState, AccState) end,
UpdatedState,
Channels
);
@@ -73,19 +73,59 @@ broadcast_all_updates(UpdatedState) ->
{ok, NewState} = guild_member_list:broadcast_all_member_list_updates(UpdatedState),
NewState.
--spec broadcast_channel_update(event_data(), guild_state()) -> guild_state().
-broadcast_channel_update(EventData, UpdatedState) ->
+-spec broadcast_channel_update(event_data(), guild_state(), guild_state()) -> guild_state().
+broadcast_channel_update(EventData, OldState, UpdatedState) ->
ChannelIdBin = maps:get(<<"id">>, EventData, undefined),
case guild_dispatch_decorate:parse_snowflake(<<"id">>, ChannelIdBin) of
undefined ->
UpdatedState;
ChannelId ->
+ broadcast_channel_update_for_id(ChannelId, OldState, UpdatedState)
+ end.
+
+%% guild_state:post_update_channel/2 has already rebuilt this channel's engine
+%% earlier in the same dispatch, through
+%% guild_member_list_write:rebuild_channels_for_permission_change/2. Rebuilding
+%% again here is a second full O(members) build over the same state, so when the
+%% engine already exists and nothing in the window can have changed what a
+%% rebuild would see, queue the sync and skip the rebuild.
+-spec broadcast_channel_update_for_id(integer(), guild_state(), guild_state()) -> guild_state().
+broadcast_channel_update_for_id(ChannelId, OldState, UpdatedState) ->
+ case rebuild_is_redundant(ChannelId, OldState, UpdatedState) of
+ true ->
+ sync_channel_list_without_rebuild(ChannelId, UpdatedState);
+ false ->
{ok, NewState} = guild_member_list:broadcast_member_list_updates_for_channel(
ChannelId, UpdatedState
),
NewState
end.
+%% The engine must already exist, because the earlier pass only rebuilds
+%% ALREADY-LOADED engines while this pass would build a missing one. Virtual
+%% channel access is compared across the whole dispatch, which is a conservative
+%% superset of the window between the two rebuilds: guild_visibility can grant a
+%% user virtual access to a channel they just lost permission for, and that does
+%% change what a rebuild sees.
+-spec rebuild_is_redundant(integer(), guild_state(), guild_state()) -> boolean().
+rebuild_is_redundant(ChannelId, OldState, UpdatedState) ->
+ guild_member_list_channel_engine:ref(integer_to_binary(ChannelId), UpdatedState) =/=
+ undefined andalso
+ maps:get(virtual_channel_access, OldState, #{}) =:=
+ maps:get(virtual_channel_access, UpdatedState, #{}).
+
+%% Mirrors guild_member_list_write:broadcast_list_by_id/4 with the rebuild
+%% removed: a list nobody is subscribed to is left alone, otherwise the sync is
+%% queued exactly as that function would have queued it.
+-spec sync_channel_list_without_rebuild(integer(), guild_state()) -> guild_state().
+sync_channel_list_without_rebuild(ChannelId, State) ->
+ ListId = integer_to_binary(ChannelId),
+ SubsTab = maps:get(member_list_subscriptions, State),
+ case map_size(guild_member_list_subs:get_list_subs(ListId, SubsTab)) of
+ 0 -> State;
+ _ -> guild_member_list_sync_batch:queue_list_sync(ListId, State)
+ end.
+
-spec extract_user_id_from_event(event_data()) -> user_id() | undefined.
extract_user_id_from_event(EventData) ->
MUser = maps:get(<<"user">>, EventData, #{}),
diff --git a/fluxer_gateway/src/guild/guild_dispatch_push.erl b/fluxer_gateway/src/guild/guild_dispatch_push.erl
index 2e72a38b6..700737321 100644
--- a/fluxer_gateway/src/guild/guild_dispatch_push.erl
+++ b/fluxer_gateway/src/guild/guild_dispatch_push.erl
@@ -8,11 +8,30 @@
collect_and_send_push_notifications/3
]).
+-define(MAX_FORMAT_MEMBERS, 50).
+-define(CONCURRENCY_LIMIT_KEY, guild_push_concurrency_limit).
+-define(DEFAULT_PUSH_CONCURRENCY, 2).
+-define(MAX_PUSH_CONCURRENCY, 8).
+-define(PUSH_WORKER_MAX_AGE_MS, 60000).
+-define(PUSH_COUNTERS, guild_push_counters).
+-define(PUSH_COUNTER_KEYS, [
+ worker_started,
+ worker_completed,
+ worker_failed,
+ dropped_at_limit,
+ spawn_failed,
+ slot_reclaimed,
+ slot_deduped,
+ members_table_missing,
+ scan_table_unavailable
+]).
+
-type event() :: atom().
-type event_data() :: map().
-type guild_state() :: map().
-type guild_id() :: integer().
-type user_id() :: integer().
+-type push_worker() :: {integer(), pid(), integer()}.
-export_type([event/0, event_data/0, guild_state/0, guild_id/0]).
-spec maybe_send_push_notifications(event(), event_data(), guild_id(), guild_state()) -> ok.
@@ -26,32 +45,496 @@ maybe_send_push_notifications(_Event, _FinalData, _GuildId, _UpdatedState) ->
-spec maybe_spawn_push(event_data(), guild_id(), guild_state()) -> ok.
maybe_spawn_push(FinalData, GuildId, UpdatedState) ->
- case push_inflight_alive() of
- true -> ok;
- false -> spawn_push(FinalData, GuildId, UpdatedState)
+ Limit = push_concurrency_limit(),
+ Workers = live_push_workers(),
+ put(push_inflight_workers, Workers),
+ case length(Workers) < Limit of
+ true -> spawn_push(FinalData, GuildId, UpdatedState);
+ false -> count_push_event(dropped_at_limit)
end.
--spec push_inflight_alive() -> boolean().
-push_inflight_alive() ->
- case get(push_inflight) of
- Pid when is_pid(Pid) -> is_process_alive(Pid);
- _ -> false
+-spec local_process_alive(pid()) -> boolean().
+local_process_alive(Pid) ->
+ node(Pid) =:= node() andalso erlang:is_process_alive(Pid).
+
+-spec push_concurrency_limit() -> pos_integer().
+push_concurrency_limit() ->
+ case application:get_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, undefined) of
+ Value when is_integer(Value), Value > 0 -> min(Value, ?MAX_PUSH_CONCURRENCY);
+ _ -> ?DEFAULT_PUSH_CONCURRENCY
+ end.
+
+-spec live_push_workers() -> [push_worker()].
+live_push_workers() ->
+ Now = erlang:monotonic_time(millisecond),
+ prune_push_workers(tracked_push_workers(Now), Now, #{}, []).
+
+-spec tracked_push_workers(integer()) -> [term()].
+tracked_push_workers(Now) ->
+ merge_inflight_push_worker(get(push_inflight), tracked_worker_list(), Now).
+
+-spec tracked_worker_list() -> [term()].
+tracked_worker_list() ->
+ case get(push_inflight_workers) of
+ Workers when is_list(Workers) -> Workers;
+ _ -> []
+ end.
+
+-spec merge_inflight_push_worker(term(), [term()], integer()) -> [term()].
+merge_inflight_push_worker(Pid, Workers, Now) when is_pid(Pid) ->
+ case lists:any(fun(Worker) -> push_worker_pid(Worker) =:= Pid end, Workers) of
+ true -> Workers;
+ false -> Workers ++ [{0, Pid, Now}]
+ end;
+merge_inflight_push_worker(_Inflight, Workers, _Now) ->
+ Workers.
+
+-spec push_worker_pid(term()) -> pid() | undefined.
+push_worker_pid({_Gen, Pid, _StartedMs}) when is_pid(Pid) -> Pid;
+push_worker_pid(_Worker) -> undefined.
+
+-spec prune_push_workers([term()], integer(), map(), [push_worker()]) -> [push_worker()].
+prune_push_workers([], _Now, _Seen, Acc) ->
+ lists:reverse(Acc);
+prune_push_workers([Worker | Rest], Now, Seen, Acc) ->
+ case keep_push_worker(Worker, Now, Seen) of
+ {keep, {_Gen, Pid, _StartedMs} = Kept} ->
+ prune_push_workers(Rest, Now, Seen#{Pid => true}, [Kept | Acc]);
+ drop ->
+ prune_push_workers(Rest, Now, Seen, Acc)
+ end.
+
+-spec keep_push_worker(term(), integer(), map()) -> {keep, push_worker()} | drop.
+keep_push_worker({Gen, Pid, StartedMs} = Worker, Now, Seen) when
+ is_integer(Gen), is_pid(Pid), is_integer(StartedMs)
+->
+ case maps:is_key(Pid, Seen) of
+ true -> drop_duplicate_push_worker();
+ false -> keep_live_push_worker(Worker, Now - StartedMs, local_process_alive(Pid))
+ end;
+keep_push_worker(_Worker, _Now, _Seen) ->
+ drop.
+
+-spec drop_duplicate_push_worker() -> drop.
+drop_duplicate_push_worker() ->
+ count_push_event(slot_deduped),
+ drop.
+
+-spec keep_live_push_worker(push_worker(), integer(), boolean()) ->
+ {keep, push_worker()} | drop.
+keep_live_push_worker(_Worker, _AgeMs, false) ->
+ drop;
+keep_live_push_worker(Worker, AgeMs, true) when AgeMs < ?PUSH_WORKER_MAX_AGE_MS ->
+ {keep, Worker};
+keep_live_push_worker(_Worker, _AgeMs, true) ->
+ count_push_event(slot_reclaimed),
+ drop.
+
+-spec note_push_worker(pid()) -> ok.
+note_push_worker(Pid) ->
+ Gen = erlang:unique_integer([monotonic, positive]),
+ StartedMs = erlang:monotonic_time(millisecond),
+ put(push_inflight_workers, [{Gen, Pid, StartedMs} | tracked_worker_list()]),
+ ok.
+
+-spec count_push_event(atom()) -> ok.
+count_push_event(Key) ->
+ bump_push_counter(Key, true).
+
+-spec bump_push_counter(atom(), boolean()) -> ok.
+bump_push_counter(Key, Retry) ->
+ try ets:update_counter(?PUSH_COUNTERS, Key, {2, 1}) of
+ _ -> ok
+ catch
+ error:badarg -> retry_push_counter(Key, Retry)
+ end.
+
+-spec retry_push_counter(atom(), boolean()) -> ok.
+retry_push_counter(Key, true) ->
+ ok = ensure_push_counters(),
+ bump_push_counter(Key, false);
+retry_push_counter(_Key, false) ->
+ ok.
+
+-spec ensure_push_counters() -> ok.
+ensure_push_counters() ->
+ case ets:whereis(?PUSH_COUNTERS) of
+ undefined -> create_push_counters();
+ _ -> ensure_push_counter_keys()
+ end.
+
+-spec create_push_counters() -> ok.
+create_push_counters() ->
+ ok = guild_ets_utils:ensure_table(?PUSH_COUNTERS, push_counter_table_options()),
+ ensure_push_counter_keys().
+
+-spec push_counter_table_options() -> [term()].
+push_counter_table_options() ->
+ [named_table, public, set, {read_concurrency, true}, {write_concurrency, true}].
+
+-spec ensure_push_counter_keys() -> ok.
+ensure_push_counter_keys() ->
+ lists:foreach(fun ensure_push_counter_key/1, ?PUSH_COUNTER_KEYS).
+
+-spec ensure_push_counter_key(atom()) -> ok.
+ensure_push_counter_key(Key) ->
+ try
+ _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}),
+ ok
+ catch
+ error:badarg -> ok
end.
-spec spawn_push(event_data(), guild_id(), guild_state()) -> ok.
spawn_push(FinalData, GuildId, UpdatedState) ->
- PushState = #{
+ Data = maps:get(data, UpdatedState, #{}),
+ case maps:get(members_ets, Data, undefined) of
+ MembersTab when is_reference(MembersTab) ->
+ CompactState = compact_push_state(MembersTab, Data, GuildId, UpdatedState),
+ spawn_compact_push(FinalData, GuildId, CompactState);
+ _ ->
+ missing_members_table(FinalData, GuildId, UpdatedState)
+ end.
+
+-spec compact_push_state(ets:tid(), map(), guild_id(), guild_state()) -> guild_state().
+compact_push_state(MembersTab, Data, GuildId, UpdatedState) ->
+ Sessions = maps:get(sessions, UpdatedState, #{}),
+ #{
+ id => maps:get(id, UpdatedState, GuildId),
+ data => compact_push_data(Data),
+ virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{}),
+ members_ets => MembersTab,
+ member_presence => maps:get(member_presence, UpdatedState, undefined),
+ session_eligibility => build_push_session_eligibility(Sessions, UpdatedState),
+ member_count => maps:get(member_count, UpdatedState, undefined)
+ }.
+
+-spec compact_push_data(map()) -> map().
+compact_push_data(Data) ->
+ maps:with(
+ [
+ <<"guild">>,
+ <<"roles">>,
+ <<"role_index">>,
+ role_perms_cache,
+ <<"channels">>,
+ <<"channel_index">>,
+ overwrite_perms_cache,
+ members_ets
+ ],
+ Data
+ ).
+
+-spec spawn_compact_push(event_data(), guild_id(), guild_state()) -> ok.
+spawn_compact_push(FinalData, GuildId, CompactState) ->
+ spawn_push_worker(
+ fun() ->
+ collect_and_send_compact_push_notifications(FinalData, GuildId, CompactState)
+ end,
+ GuildId
+ ).
+
+-spec missing_members_table(event_data(), guild_id(), guild_state()) -> ok.
+missing_members_table(FinalData, GuildId, UpdatedState) ->
+ count_push_event(members_table_missing),
+ logger:warning(
+ "guild_push_members_table_unavailable: guild_id=~p phase=spawn",
+ [GuildId]
+ ),
+ spawn_legacy_push(FinalData, GuildId, UpdatedState).
+
+-spec spawn_legacy_push(event_data(), guild_id(), guild_state()) -> ok.
+spawn_legacy_push(FinalData, GuildId, UpdatedState) ->
+ LegacyState = legacy_push_state(GuildId, UpdatedState),
+ spawn_push_worker(
+ fun() ->
+ collect_and_send_push_notifications(FinalData, GuildId, LegacyState)
+ end,
+ GuildId
+ ).
+
+-spec legacy_push_state(guild_id(), guild_state()) -> guild_state().
+legacy_push_state(GuildId, UpdatedState) ->
+ #{
id => maps:get(id, UpdatedState, GuildId),
data => maps:get(data, UpdatedState, #{}),
sessions => maps:get(sessions, UpdatedState, #{}),
- virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{})
- },
- Pid = spawn(fun() ->
- collect_and_send_push_notifications(FinalData, GuildId, PushState)
- end),
- put(push_inflight, Pid),
+ member_presence => maps:get(member_presence, UpdatedState, undefined),
+ virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{}),
+ member_count => maps:get(member_count, UpdatedState, undefined)
+ }.
+
+-spec spawn_push_worker(fun(() -> ok), guild_id()) -> ok.
+spawn_push_worker(Worker, GuildId) ->
+ Counted = fun() -> run_counted_push_worker(Worker) end,
+ case try_spawn_push_worker(Counted, GuildId) of
+ {ok, Pid} ->
+ put(push_inflight, Pid),
+ note_push_worker(Pid),
+ count_push_event(worker_started);
+ error ->
+ count_push_event(spawn_failed)
+ end.
+
+-spec run_counted_push_worker(fun(() -> ok)) -> ok.
+run_counted_push_worker(Worker) ->
+ try Worker() of
+ _ -> count_push_event(worker_completed)
+ catch
+ _Class:_Reason -> count_push_event(worker_failed)
+ end.
+
+-spec try_spawn_push_worker(fun(() -> ok), guild_id()) -> {ok, pid()} | error.
+try_spawn_push_worker(Worker, GuildId) ->
+ try
+ {ok, spawn(fun() -> run_deprioritised_push_worker(Worker) end)}
+ catch
+ error:system_limit ->
+ logger:warning(
+ "guild_push_worker_spawn_failed: guild_id=~p reason=system_limit",
+ [GuildId]
+ ),
+ error
+ end.
+
+-spec run_deprioritised_push_worker(fun(() -> ok)) -> ok.
+run_deprioritised_push_worker(Worker) ->
+ ok = apply_push_worker_priority(),
+ Worker().
+
+-spec apply_push_worker_priority() -> ok.
+apply_push_worker_priority() ->
+ _ = erlang:process_flag(priority, low),
+ ok.
+
+-spec large_guild_meta(guild_state()) -> map() | undefined.
+large_guild_meta(State) ->
+ case maps:get(member_count, State, undefined) of
+ MemberCount when is_integer(MemberCount), MemberCount >= 0 ->
+ #{member_count => MemberCount, features => guild_features(State)};
+ _ ->
+ undefined
+ end.
+
+-spec guild_features(guild_state()) -> [binary()].
+guild_features(State) ->
+ Data = maps:get(data, State, #{}),
+ Guild = maps:get(<<"guild">>, Data, #{}),
+ case maps:get(<<"features">>, Guild, []) of
+ Features when is_list(Features) -> Features;
+ _ -> []
+ end.
+
+-spec collect_and_send_compact_push_notifications(event_data(), guild_id(), guild_state()) ->
+ ok.
+collect_and_send_compact_push_notifications(MessageData, GuildId, State) ->
+ case guild_dispatch_config:should_send_push_notifications(State) of
+ false -> ok;
+ true -> send_compact_push_notifications(MessageData, GuildId, State)
+ end.
+
+-spec send_compact_push_notifications(event_data(), guild_id(), guild_state()) -> ok.
+send_compact_push_notifications(MessageData, GuildId, State) ->
+ ChannelIdBin = maps:get(<<"channel_id">>, MessageData, undefined),
+ case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of
+ undefined -> ok;
+ ChannelId -> scan_and_send_compact_push(MessageData, GuildId, ChannelId, State)
+ end.
+
+-spec scan_and_send_compact_push(event_data(), guild_id(), integer(), guild_state()) -> ok.
+scan_and_send_compact_push(MessageData, GuildId, ChannelId, State) ->
+ Context = compact_scan_context(MessageData, ChannelId, State),
+ MembersTab = maps:get(members_ets, State),
+ case scan_push_members(MembersTab, Context, State) of
+ {ok, #{eligible_user_ids := []}} ->
+ ok;
+ {ok, Scan} ->
+ send_compact_scanned_push(MessageData, GuildId, Scan, State);
+ {error, table_unavailable} ->
+ scan_table_unavailable(GuildId)
+ end.
+
+-spec scan_table_unavailable(guild_id()) -> ok.
+scan_table_unavailable(GuildId) ->
+ count_push_event(scan_table_unavailable),
+ logger:warning(
+ "guild_push_members_table_unavailable: guild_id=~p phase=scan",
+ [GuildId]
+ ),
ok.
+-spec compact_scan_context(event_data(), integer(), guild_state()) -> map().
+compact_scan_context(MessageData, ChannelId, State) ->
+ MentionRoles = maps:get(<<"mention_roles">>, MessageData, []),
+ #{
+ mention_everyone => maps:get(<<"mention_everyone">>, MessageData, false) =:= true,
+ direct_mentions => direct_mention_id_set(MessageData),
+ mention_roles => mention_role_id_set(MentionRoles),
+ format_members => format_member_id_set(MessageData),
+ channel_id => ChannelId,
+ session_eligibility => maps:get(session_eligibility, State)
+ }.
+
+-spec scan_push_members(ets:tid(), map(), guild_state()) ->
+ {ok, map()} | {error, table_unavailable}.
+scan_push_members(MembersTab, Context, State) ->
+ try
+ {ok, fold_push_members(member_id_snapshot(MembersTab), MembersTab, Context, State)}
+ catch
+ error:badarg -> {error, table_unavailable}
+ end.
+
+-spec initial_scan_acc() -> map().
+initial_scan_acc() ->
+ #{eligible_user_ids => [], user_roles => #{}, format_members => #{}}.
+
+-spec member_id_snapshot(ets:tid()) -> [user_id()].
+member_id_snapshot(MembersTab) ->
+ ets:select(MembersTab, [{{'$1', '_'}, [], ['$1']}]).
+
+-spec fold_push_members([user_id()], ets:tid(), map(), guild_state()) -> map().
+fold_push_members(UserIds, MembersTab, Context, State) ->
+ lists:foldl(
+ fun(UserId, Acc) -> scan_push_member_id(UserId, MembersTab, Context, State, Acc) end,
+ initial_scan_acc(),
+ UserIds
+ ).
+
+-spec scan_push_member_id(user_id(), ets:tid(), map(), guild_state(), map()) -> map().
+scan_push_member_id(UserId, MembersTab, Context, State, Acc) ->
+ case ets:lookup(MembersTab, UserId) of
+ [Row] -> scan_push_member(Row, Context, State, Acc);
+ _ -> Acc
+ end.
+
+-spec scan_push_member(term(), map(), guild_state(), map()) -> map().
+scan_push_member({UserId, Member}, Context, State, Acc) when
+ is_integer(UserId), UserId > 0, is_map(Member)
+->
+ Acc1 = maybe_collect_format_member(UserId, Member, Context, Acc),
+ case scanned_member_is_eligible(UserId, Member, Context, State) of
+ true -> add_scanned_member(UserId, Member, Acc1);
+ false -> Acc1
+ end;
+scan_push_member(_Row, _Context, _State, Acc) ->
+ Acc.
+
+-spec scanned_member_is_eligible(user_id(), map(), map(), guild_state()) -> boolean().
+scanned_member_is_eligible(UserId, Member, Context, State) ->
+ is_push_candidate(UserId, Member, Context) andalso
+ guild_permissions:can_view_channel(
+ UserId, maps:get(channel_id, Context), Member, State
+ ).
+
+-spec add_scanned_member(user_id(), map(), map()) -> map().
+add_scanned_member(UserId, Member, Acc) ->
+ Eligible = maps:get(eligible_user_ids, Acc),
+ UserRoles = maps:get(user_roles, Acc),
+ Acc#{
+ eligible_user_ids := [UserId | Eligible],
+ user_roles := UserRoles#{UserId => extract_role_ids(Member)}
+ }.
+
+-spec maybe_collect_format_member(user_id(), map(), map(), map()) -> map().
+maybe_collect_format_member(UserId, Member, Context, Acc) ->
+ case maps:is_key(UserId, maps:get(format_members, Context)) of
+ true ->
+ Members = maps:get(format_members, Acc),
+ Acc#{format_members := Members#{UserId => Member}};
+ false ->
+ Acc
+ end.
+
+-spec is_push_candidate(user_id(), map(), map()) -> boolean().
+is_push_candidate(UserId, Member, Context) ->
+ maps:get(UserId, maps:get(session_eligibility, Context), true) orelse
+ (not mentions_respect_active_session() andalso
+ is_mention_candidate(UserId, Member, Context)).
+
+-spec is_mention_candidate(user_id(), map(), map()) -> boolean().
+is_mention_candidate(UserId, Member, Context) ->
+ maps:get(mention_everyone, Context) orelse
+ maps:is_key(UserId, maps:get(direct_mentions, Context)) orelse
+ member_has_mentioned_role(Member, maps:get(mention_roles, Context)).
+
+-spec mentions_respect_active_session() -> boolean().
+mentions_respect_active_session() ->
+ application:get_env(fluxer_gateway, push_mentions_respect_active_session, true) =:= true.
+
+-spec send_compact_scanned_push(event_data(), guild_id(), map(), guild_state()) -> ok.
+send_compact_scanned_push(MessageData, GuildId, Scan, State) ->
+ FormatData = compact_format_data(maps:get(data, State), maps:get(format_members, Scan)),
+ send_push_to_eligible_users(
+ MessageData,
+ GuildId,
+ lists:reverse(maps:get(eligible_user_ids, Scan)),
+ maps:get(user_roles, Scan),
+ maps:get(session_eligibility, State),
+ FormatData,
+ large_guild_meta(State)
+ ).
+
+-spec compact_format_data(map(), map()) -> map().
+compact_format_data(Data, FormatMembers) ->
+ Data#{
+ <<"members">> => FormatMembers,
+ members_normalized => FormatMembers,
+ members_sorted_ids => lists:sort(maps:keys(FormatMembers)),
+ <<"member_role_index">> =>
+ guild_data_index_members:build_member_role_index(FormatMembers)
+ }.
+
+-spec direct_mention_id_set(event_data()) -> map().
+direct_mention_id_set(MessageData) ->
+ lists:foldl(
+ fun add_direct_mention_id/2,
+ #{},
+ maps:get(<<"mentions">>, MessageData, [])
+ ).
+
+-spec add_direct_mention_id(term(), map()) -> map().
+add_direct_mention_id(Mention, Acc) ->
+ case direct_mention_user_id(Mention) of
+ {true, UserId} -> Acc#{UserId => true};
+ false -> Acc
+ end.
+
+-spec format_member_id_set(event_data()) -> map().
+format_member_id_set(MessageData) ->
+ AuthorSet = author_format_id_set(maps:get(<<"author">>, MessageData, undefined)),
+ collect_format_mention_ids(
+ maps:get(<<"mentions">>, MessageData, []),
+ ?MAX_FORMAT_MEMBERS,
+ AuthorSet
+ ).
+
+-spec author_format_id_set(term()) -> map().
+author_format_id_set(Author) when is_map(Author) ->
+ case direct_mention_user_id(Author) of
+ {true, UserId} -> #{UserId => true};
+ false -> #{}
+ end;
+author_format_id_set(_Author) ->
+ #{}.
+
+-spec collect_format_mention_ids([term()], non_neg_integer(), map()) -> map().
+collect_format_mention_ids(_Mentions, 0, Acc) ->
+ Acc;
+collect_format_mention_ids([], _Remaining, Acc) ->
+ Acc;
+collect_format_mention_ids([Mention | Rest], Remaining, Acc) ->
+ case direct_mention_user_id(Mention) of
+ {true, UserId} -> collect_format_mention_id(UserId, Rest, Remaining, Acc);
+ false -> collect_format_mention_ids(Rest, Remaining, Acc)
+ end.
+
+-spec collect_format_mention_id(user_id(), [term()], non_neg_integer(), map()) -> map().
+collect_format_mention_id(UserId, Rest, Remaining, Acc) ->
+ case maps:is_key(UserId, Acc) of
+ true -> collect_format_mention_ids(Rest, Remaining, Acc);
+ false -> collect_format_mention_ids(Rest, Remaining - 1, Acc#{UserId => true})
+ end.
+
-spec collect_and_send_push_notifications(event_data(), guild_id(), guild_state()) -> ok.
collect_and_send_push_notifications(MessageData, GuildId, State) ->
case guild_dispatch_config:should_send_push_notifications(State) of
@@ -64,7 +547,7 @@ send_push_notifications(MessageData, GuildId, State) ->
Data = maps:get(data, State),
Members = guild_data_index:member_map(Data),
Sessions = maps:get(sessions, State, #{}),
- SessionEligibility = build_push_session_eligibility(Sessions),
+ SessionEligibility = build_push_session_eligibility(Sessions, State),
CandidateUserIds = push_candidate_user_ids(Members, SessionEligibility, MessageData),
ChannelIdBin = maps:get(<<"channel_id">>, MessageData, undefined),
case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of
@@ -114,7 +597,8 @@ send_to_eligible(
EligibleUserIds,
UserRolesMap,
SessionEligibility,
- Data
+ Data,
+ large_guild_meta(State)
)
end.
@@ -128,8 +612,9 @@ push_candidate_user_ids(Members, SessionEligibility, MessageData) ->
).
-spec push_candidate_user_ids(term(), map(), map(), event_data()) -> [user_id()].
-push_candidate_user_ids(true, Members, _SessionEligibility, _MessageData) ->
- maps:keys(Members);
+push_candidate_user_ids(true, Members, SessionEligibility, _MessageData) ->
+ {CandidateUserIds, _Seen} = base_candidate_acc(Members, SessionEligibility),
+ lists:reverse(CandidateUserIds);
push_candidate_user_ids(_MentionEveryone, Members, SessionEligibility, MessageData) ->
BaseAcc = base_candidate_acc(Members, SessionEligibility),
{CandidateUserIds, _Seen} =
@@ -276,6 +761,72 @@ view_to_filtermap(UserId, ChannelId, Member, State) ->
false -> false
end.
+-spec build_push_session_eligibility(map(), guild_state()) -> #{user_id() => boolean()}.
+build_push_session_eligibility(Sessions, State) ->
+ case presence_eligibility_enabled() of
+ true -> build_push_presence_eligibility(Sessions, State);
+ false -> build_push_session_eligibility(Sessions)
+ end.
+
+-spec presence_eligibility_enabled() -> boolean().
+presence_eligibility_enabled() ->
+ application:get_env(fluxer_gateway, push_presence_eligibility, true) =:= true.
+
+-spec build_push_presence_eligibility(map(), guild_state()) -> #{user_id() => boolean()}.
+build_push_presence_eligibility(Sessions, State) ->
+ Presences = maps:get(member_presence, State, undefined),
+ maps:fold(
+ fun(_Sid, Session, Acc) ->
+ accumulate_presence_eligibility(Session, Presences, Acc)
+ end,
+ #{},
+ Sessions
+ ).
+
+-spec accumulate_presence_eligibility(map(), term(), #{user_id() => boolean()}) ->
+ #{user_id() => boolean()}.
+accumulate_presence_eligibility(Session, Presences, Acc) ->
+ case maps:get(user_id, Session, undefined) of
+ UserId when is_integer(UserId) ->
+ Acc#{
+ UserId =>
+ maps:get(UserId, Acc, true) andalso
+ not actively_engaged(UserId, Presences)
+ };
+ _ ->
+ Acc
+ end.
+
+-spec actively_engaged(user_id(), term()) -> boolean().
+actively_engaged(_UserId, undefined) ->
+ false;
+actively_engaged(UserId, Presences) ->
+ case lookup_presence_safe(UserId, Presences) of
+ undefined -> false;
+ Presence -> presence_is_active(Presence)
+ end.
+
+-spec lookup_presence_safe(user_id(), term()) -> map() | undefined.
+lookup_presence_safe(UserId, Presences) ->
+ try guild_state_member:lookup_presence(Presences, UserId) of
+ Presence when is_map(Presence) -> Presence;
+ _ -> undefined
+ catch
+ error:badarg -> undefined
+ end.
+
+-spec presence_is_active(map()) -> boolean().
+presence_is_active(Presence) ->
+ Status = maps:get(<<"status">>, Presence, <<"offline">>),
+ Afk = maps:get(<<"afk">>, Presence, false) =:= true,
+ Mobile = maps:get(<<"mobile">>, Presence, false) =:= true,
+ is_engaged_status(Status) andalso not Afk andalso not Mobile.
+
+-spec is_engaged_status(term()) -> boolean().
+is_engaged_status(<<"online">>) -> true;
+is_engaged_status(<<"dnd">>) -> true;
+is_engaged_status(_Status) -> false.
+
-spec build_push_session_eligibility(map()) -> #{user_id() => boolean()}.
build_push_session_eligibility(Sessions) ->
maps:fold(
@@ -299,22 +850,23 @@ accumulate_session_eligibility(Session, Acc) ->
Acc
end.
--spec send_push_to_eligible_users(event_data(), guild_id(), [user_id()], map(), map(), map()) ->
- ok.
+-spec send_push_to_eligible_users(
+ event_data(), guild_id(), [user_id()], map(), map(), map(), map() | undefined
+) -> ok.
send_push_to_eligible_users(
MessageData,
GuildId,
EligibleUserIds,
UserRolesMap,
ConnectedUsers,
- Data
+ Data,
+ LargeGuildMeta
) ->
AuthorIdBin = maps:get(<<"id">>, maps:get(<<"author">>, MessageData, #{}), undefined),
case guild_dispatch_decorate:parse_snowflake(<<"author.id">>, AuthorIdBin) of
undefined ->
ok;
AuthorId ->
- Guild = maps:get(<<"guild">>, Data),
ChannelIdBin = maps:get(<<"channel_id">>, MessageData),
ChannelName = find_channel_name(ChannelIdBin, Data),
RoleNames = build_role_names_map(Data),
@@ -324,11 +876,11 @@ send_push_to_eligible_users(
EligibleUserIds,
UserRolesMap,
ConnectedUsers,
- Guild,
ChannelName,
RoleNames,
Data,
- AuthorId
+ AuthorId,
+ LargeGuildMeta
)
end.
@@ -338,11 +890,11 @@ send_push_to_eligible_users(
[user_id()],
map(),
map(),
- map(),
binary(),
map(),
map(),
- integer()
+ integer(),
+ map() | undefined
) -> ok.
do_send_push(
MessageData,
@@ -350,12 +902,13 @@ do_send_push(
EligibleUserIds,
UserRolesMap,
ConnectedUsers,
- Guild,
ChannelName,
RoleNames,
Data,
- AuthorId
+ AuthorId,
+ LargeGuildMeta
) ->
+ Guild = maps:get(<<"guild">>, Data),
DefaultMessageNotifications = maps:get(<<"default_message_notifications">>, Guild, 0),
GuildName = maps:get(<<"name">>, Guild, <<"Unknown">>),
push:handle_message_create(#{
@@ -372,9 +925,19 @@ do_send_push(
MessageData, GuildId, RoleNames, Data
),
user_roles => UserRolesMap,
- connected_users => ConnectedUsers
+ connected_users => ConnectedUsers,
+ guild_member_count => meta_member_count(LargeGuildMeta),
+ guild_features => meta_features(LargeGuildMeta)
}).
+-spec meta_member_count(map() | undefined) -> non_neg_integer() | undefined.
+meta_member_count(#{member_count := MemberCount}) -> MemberCount;
+meta_member_count(_Meta) -> undefined.
+
+-spec meta_features(map() | undefined) -> [binary()] | undefined.
+meta_features(#{features := Features}) -> Features;
+meta_features(_Meta) -> undefined.
+
-spec find_channel_name(binary(), map()) -> binary().
find_channel_name(ChannelIdBin, Data) ->
case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of
@@ -444,6 +1007,134 @@ collect_role_id(Role, Acc) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
+presence_eligibility_suppresses_active_desktop_test() ->
+ Tab = ets:new(test_member_presence, [set, public]),
+ try
+ ets:insert(
+ Tab, {1, #{<<"status">> => <<"online">>, <<"mobile">> => false, <<"afk">> => false}}
+ ),
+ ?assertEqual(
+ #{1 => false},
+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{
+ member_presence => Tab
+ })
+ )
+ after
+ ets:delete(Tab)
+ end.
+
+presence_eligibility_allows_mobile_only_session_test() ->
+ Tab = ets:new(test_member_presence, [set, public]),
+ try
+ ets:insert(
+ Tab, {1, #{<<"status">> => <<"online">>, <<"mobile">> => true, <<"afk">> => false}}
+ ),
+ ?assertEqual(
+ #{1 => true},
+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{
+ member_presence => Tab
+ })
+ )
+ after
+ ets:delete(Tab)
+ end.
+
+presence_eligibility_allows_idle_and_afk_test() ->
+ Tab = ets:new(test_member_presence, [set, public]),
+ try
+ ets:insert(
+ Tab, {1, #{<<"status">> => <<"idle">>, <<"mobile">> => false, <<"afk">> => false}}
+ ),
+ ets:insert(
+ Tab, {2, #{<<"status">> => <<"online">>, <<"mobile">> => false, <<"afk">> => true}}
+ ),
+ ?assertEqual(
+ #{1 => true, 2 => true},
+ build_push_presence_eligibility(
+ #{<<"s1">> => #{user_id => 1}, <<"s2">> => #{user_id => 2}},
+ #{member_presence => Tab}
+ )
+ )
+ after
+ ets:delete(Tab)
+ end.
+
+presence_eligibility_allows_unknown_presence_test() ->
+ Tab = ets:new(test_member_presence, [set, public]),
+ try
+ ?assertEqual(
+ #{7 => true},
+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 7}}, #{
+ member_presence => Tab
+ })
+ )
+ after
+ ets:delete(Tab)
+ end.
+
+presence_eligibility_allows_when_presence_table_missing_test() ->
+ ?assertEqual(
+ #{1 => true},
+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{})
+ ).
+
+legacy_session_eligibility_suppresses_every_real_session_test() ->
+ RealSession = #{
+ session_id => <<"s1">>,
+ user_id => 1,
+ pid => self(),
+ active_guilds => sets:new(),
+ bot => false,
+ is_staff => false,
+ pending_connect => false,
+ viewable_channels => #{}
+ },
+ ?assertEqual(#{1 => false}, build_push_session_eligibility(#{<<"s1">> => RealSession})).
+
+is_push_candidate_suppresses_mention_for_active_user_test() ->
+ Context = #{
+ session_eligibility => #{1 => false},
+ mention_everyone => true,
+ direct_mentions => #{1 => true},
+ mention_roles => #{}
+ },
+ ?assertEqual(false, is_push_candidate(1, #{}, Context)).
+
+is_push_candidate_allows_eligible_user_test() ->
+ Context = #{
+ session_eligibility => #{1 => true},
+ mention_everyone => false,
+ direct_mentions => #{},
+ mention_roles => #{}
+ },
+ ?assertEqual(true, is_push_candidate(1, #{}, Context)).
+
+is_push_candidate_allows_sessionless_user_test() ->
+ Context = #{
+ session_eligibility => #{},
+ mention_everyone => false,
+ direct_mentions => #{},
+ mention_roles => #{}
+ },
+ ?assertEqual(true, is_push_candidate(9, #{}, Context)).
+
+push_candidate_user_ids_everyone_respects_active_sessions_test() ->
+ Members = #{1 => #{}, 2 => #{}, 3 => #{}},
+ ?assertEqual(
+ [],
+ push_candidate_user_ids(
+ Members, #{1 => false, 2 => false, 3 => false}, #{<<"mention_everyone">> => true}
+ )
+ ),
+ ?assertEqual(
+ [1, 3],
+ lists:sort(
+ push_candidate_user_ids(
+ Members, #{1 => true, 2 => false}, #{<<"mention_everyone">> => true}
+ )
+ )
+ ).
+
build_push_session_eligibility_test() ->
Sessions = #{
<<"s1">> => #{user_id => 1, mobile => false, afk => true},
@@ -477,14 +1168,6 @@ push_candidate_user_ids_includes_connected_mentioned_users_test() ->
CandidateUserIds = push_candidate_user_ids(Members, SessionEligibility, MessageData),
?assertEqual([1, 3], lists:sort(CandidateUserIds)).
-push_candidate_user_ids_includes_everyone_even_when_connected_test() ->
- Members = #{1 => #{}, 2 => #{}, 3 => #{}},
- SessionEligibility = #{1 => false, 2 => false, 3 => false},
- CandidateUserIds = push_candidate_user_ids(
- Members, SessionEligibility, #{<<"mention_everyone">> => true}
- ),
- ?assertEqual([1, 2, 3], lists:sort(CandidateUserIds)).
-
push_candidate_user_ids_deduplicates_mentions_test() ->
Members = #{1 => #{<<"roles">> => [<<"10">>]}, 2 => #{<<"roles">> => []}},
SessionEligibility = #{1 => false, 2 => false},
@@ -557,13 +1240,18 @@ send_push_to_eligible_users_uses_full_data_for_channel_name_test() ->
}
},
?assertEqual(
- ok, send_push_to_eligible_users(MessageData, 10, [1], #{1 => []}, #{}, Data)
+ ok,
+ send_push_to_eligible_users(
+ MessageData, 10, [1], #{1 => []}, #{}, Data, undefined
+ )
),
receive
{push_params, Params} ->
?assertEqual(<<"general">>, maps:get(channel_name, Params)),
?assertEqual(<<"Test Guild">>, maps:get(guild_name, Params)),
- ?assertEqual(#{200 => <<"Alerts">>}, maps:get(role_names, Params))
+ ?assertEqual(#{200 => <<"Alerts">>}, maps:get(role_names, Params)),
+ ?assertEqual(undefined, maps:get(guild_member_count, Params)),
+ ?assertEqual(undefined, maps:get(guild_features, Params))
after 1000 ->
?assert(false)
end,
@@ -602,4 +1290,518 @@ extract_role_ids_missing_key_test() ->
Member = #{},
?assertEqual([], extract_role_ids(Member)).
+meta_accessors_default_to_undefined_test() ->
+ ?assertEqual(undefined, meta_member_count(undefined)),
+ ?assertEqual(undefined, meta_features(undefined)),
+ ?assertEqual(5, meta_member_count(#{member_count => 5, features => []})),
+ ?assertEqual([], meta_features(#{member_count => 5, features => []})).
+
+compact_push_state_drops_members_and_keeps_member_count_test() ->
+ Tab = ets:new(test_members, [set, public]),
+ try
+ Data = compact_test_data(Tab),
+ UpdatedState = #{
+ id => 7,
+ data => Data,
+ sessions => #{<<"s1">> => #{user_id => 1}},
+ virtual_channel_access => #{},
+ member_count => 49435
+ },
+ Compact = compact_push_state(Tab, Data, 7, UpdatedState),
+ CompactData = maps:get(data, Compact),
+ ?assertEqual(49435, maps:get(member_count, Compact)),
+ ?assertEqual(Tab, maps:get(members_ets, Compact)),
+ ?assertNot(maps:is_key(sessions, Compact)),
+ ?assertEqual(#{1 => true}, maps:get(session_eligibility, Compact)),
+ ?assertNot(maps:is_key(<<"members">>, CompactData)),
+ ?assertNot(maps:is_key(members_normalized, CompactData)),
+ ?assertNot(maps:is_key(<<"member_role_index">>, CompactData)),
+ ?assertEqual(
+ #{member_count => 49435, features => [<<"COMMUNITY">>]},
+ large_guild_meta(Compact)
+ )
+ after
+ ets:delete(Tab)
+ end.
+
+legacy_push_state_carries_member_count_test() ->
+ UpdatedState = #{id => 7, data => #{}, sessions => #{}, member_count => 1234},
+ Legacy = legacy_push_state(7, UpdatedState),
+ ?assertEqual(1234, maps:get(member_count, Legacy)),
+ ?assertEqual(#{}, maps:get(sessions, Legacy)),
+ ?assertEqual(#{member_count => 1234, features => []}, large_guild_meta(Legacy)).
+
+compact_scan_collects_eligible_users_and_format_members_test() ->
+ Tab = ets:new(test_members, [set, public]),
+ try
+ true = ets:insert(Tab, {1, #{<<"roles">> => []}}),
+ true = ets:insert(Tab, {2, #{<<"roles">> => []}}),
+ true = ets:insert(Tab, {3, #{<<"roles">> => [<<"200">>]}}),
+ State = compact_scan_state(Tab),
+ MessageData = #{<<"mentions">> => [#{<<"id">> => <<"3">>}]},
+ Context = compact_scan_context(MessageData, 10, State),
+ {ok, Scan} = scan_push_members(Tab, Context, State),
+ ?assertEqual([1], lists:sort(maps:get(eligible_user_ids, Scan))),
+ ?assertEqual(#{3 => #{<<"roles">> => [<<"200">>]}}, maps:get(format_members, Scan)),
+ ?assertNot(maps:is_key(3, maps:get(user_roles, Scan))),
+ ?assertEqual([], maps:get(1, maps:get(user_roles, Scan)))
+ after
+ ets:delete(Tab)
+ end.
+
+compact_scan_includes_mentioned_user_when_bypass_is_re_enabled_test() ->
+ Tab = ets:new(test_members, [set, public]),
+ application:set_env(fluxer_gateway, push_mentions_respect_active_session, false),
+ try
+ true = ets:insert(Tab, {1, #{<<"roles">> => []}}),
+ true = ets:insert(Tab, {3, #{<<"roles">> => [<<"200">>]}}),
+ State = compact_scan_state(Tab),
+ MessageData = #{<<"mentions">> => [#{<<"id">> => <<"3">>}]},
+ Context = compact_scan_context(MessageData, 10, State),
+ {ok, Scan} = scan_push_members(Tab, Context, State),
+ ?assertEqual([1, 3], lists:sort(maps:get(eligible_user_ids, Scan)))
+ after
+ application:unset_env(fluxer_gateway, push_mentions_respect_active_session),
+ ets:delete(Tab)
+ end.
+
+scan_push_members_reports_unavailable_table_test() ->
+ Tab = ets:new(test_members, [set, public]),
+ State = compact_scan_state(Tab),
+ Context = compact_scan_context(#{}, 10, State),
+ true = ets:delete(Tab),
+ ?assertEqual({error, table_unavailable}, scan_push_members(Tab, Context, State)).
+
+collect_format_mention_ids_is_bounded_test() ->
+ Mentions = [#{<<"id">> => integer_to_binary(Id)} || Id <- lists:seq(1, 80)],
+ Result = collect_format_mention_ids(Mentions, ?MAX_FORMAT_MEMBERS, #{}),
+ ?assertEqual(?MAX_FORMAT_MEMBERS, map_size(Result)).
+
+format_member_id_set_includes_author_test() ->
+ MessageData = #{
+ <<"author">> => #{<<"id">> => <<"99">>},
+ <<"mentions">> => [#{<<"id">> => <<"1">>}, #{<<"id">> => <<"1">>}]
+ },
+ ?assertEqual(#{1 => true, 99 => true}, format_member_id_set(MessageData)).
+
+spawn_push_without_members_table_falls_back_to_legacy_push_test() ->
+ reset_push_worker_state(),
+ try
+ ?assertEqual(ok, spawn_push(#{}, 7, #{id => 7, data => #{}, sessions => #{}})),
+ ?assert(is_pid(get(push_inflight)))
+ after
+ reset_push_worker_state()
+ end.
+
+compact_push_carries_large_guild_metadata_test() ->
+ Self = self(),
+ Tab = ets:new(test_members, [set, public]),
+ ok = meck:new(push, [passthrough, no_link]),
+ try
+ ok = meck:expect(push, handle_message_create, fun(Params) ->
+ Self ! {push_params, Params},
+ ok
+ end),
+ State = compact_scan_state(Tab),
+ Scan = #{
+ eligible_user_ids => [1],
+ user_roles => #{1 => []},
+ format_members => #{1 => #{<<"nick">> => <<"one">>}}
+ },
+ MessageData = #{
+ <<"channel_id">> => <<"10">>,
+ <<"author">> => #{<<"id">> => <<"42">>}
+ },
+ ?assertEqual(ok, send_compact_scanned_push(MessageData, 7, Scan, State)),
+ receive
+ {push_params, Params} ->
+ ?assertEqual(49435, maps:get(guild_member_count, Params)),
+ ?assertEqual([<<"COMMUNITY">>], maps:get(guild_features, Params)),
+ ?assertEqual([1], maps:get(user_ids, Params)),
+ ?assertEqual(<<"general">>, maps:get(channel_name, Params))
+ after 1000 ->
+ ?assert(false)
+ end,
+ ?assert(meck:validate(push))
+ after
+ meck:unload(push),
+ ets:delete(Tab)
+ end.
+
+compact_format_data_restricts_member_map_test() ->
+ Data = #{<<"guild">> => #{}, <<"members">> => #{1 => #{}, 2 => #{}, 3 => #{}}},
+ FormatMembers = #{2 => #{<<"roles">> => [<<"200">>]}},
+ Result = compact_format_data(Data, FormatMembers),
+ ?assertEqual(FormatMembers, maps:get(<<"members">>, Result)),
+ ?assertEqual(FormatMembers, maps:get(members_normalized, Result)),
+ ?assertEqual([2], maps:get(members_sorted_ids, Result)),
+ ?assertEqual(#{200 => #{2 => true}}, maps:get(<<"member_role_index">>, Result)).
+
+inflight_pid_without_a_slot_still_counts_against_the_limit_test() ->
+ reset_push_worker_state(),
+ Blocker = blocking_push_worker(),
+ put(push_inflight, Blocker),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1),
+ try
+ ?assertEqual([Blocker], worker_pids(live_push_workers())),
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertEqual(Blocker, get(push_inflight)),
+ ?assertEqual([Blocker], worker_pids(get(push_inflight_workers)))
+ after
+ stop_push_worker(Blocker),
+ reset_push_worker_state()
+ end.
+
+unknown_worker_entries_are_dropped_test() ->
+ reset_push_worker_state(),
+ Blocker = blocking_push_worker(),
+ put(push_inflight_workers, [Blocker, {Blocker, 0}, not_a_worker]),
+ try
+ ?assertEqual([], live_push_workers())
+ after
+ stop_push_worker(Blocker),
+ reset_push_worker_state()
+ end.
+
+remote_worker_pid_is_dropped_without_badarg_test() ->
+ reset_push_worker_state(),
+ Remote = remote_test_pid(),
+ put(push_inflight_workers, [{1, Remote, erlang:monotonic_time(millisecond)}]),
+ try
+ ?assertNotEqual(node(), node(Remote)),
+ ?assertError(badarg, erlang:is_process_alive(Remote)),
+ ?assertEqual(false, local_process_alive(Remote)),
+ ?assertEqual([], live_push_workers())
+ after
+ reset_push_worker_state()
+ end.
+
+reused_worker_pid_consumes_only_one_slot_test() ->
+ reset_push_worker_state(),
+ Blocker = blocking_push_worker(),
+ Now = erlang:monotonic_time(millisecond),
+ put(push_inflight_workers, [{2, Blocker, Now}, {1, Blocker, Now - 10}]),
+ Before = read_push_counter(slot_deduped),
+ try
+ ?assertEqual([{2, Blocker, Now}], live_push_workers()),
+ ?assertEqual(Before + 1, read_push_counter(slot_deduped))
+ after
+ stop_push_worker(Blocker),
+ reset_push_worker_state()
+ end.
+
+stale_push_worker_slot_is_reclaimed_and_counted_test() ->
+ reset_push_worker_state(),
+ Stale = blocking_push_worker(),
+ Aged = erlang:monotonic_time(millisecond) - ?PUSH_WORKER_MAX_AGE_MS - 1,
+ put(push_inflight_workers, [{1, Stale, Aged}]),
+ Before = read_push_counter(slot_reclaimed),
+ try
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertEqual(Before + 1, read_push_counter(slot_reclaimed)),
+ ?assertEqual(1, length(get(push_inflight_workers))),
+ ?assertNot(lists:member(Stale, worker_pids(get(push_inflight_workers))))
+ after
+ stop_push_worker(Stale),
+ reset_push_worker_state()
+ end.
+
+push_drop_is_readable_from_named_ets_table_test() ->
+ reset_push_worker_state(),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1),
+ Blocker = blocking_push_worker(),
+ put(push_inflight_workers, [{1, Blocker, erlang:monotonic_time(millisecond)}]),
+ Before = read_push_counter(dropped_at_limit),
+ try
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertEqual(Before + 1, read_push_counter(dropped_at_limit)),
+ ?assertMatch([{dropped_at_limit, _}], ets:lookup(?PUSH_COUNTERS, dropped_at_limit)),
+ ?assertEqual(
+ lists:sort(?PUSH_COUNTER_KEYS),
+ lists:sort([Key || {Key, _} <- ets:tab2list(?PUSH_COUNTERS)])
+ )
+ after
+ stop_push_worker(Blocker),
+ reset_push_worker_state()
+ end.
+
+started_push_worker_is_counted_test() ->
+ reset_push_worker_state(),
+ Before = read_push_counter(worker_started),
+ try
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertEqual(Before + 1, read_push_counter(worker_started)),
+ ?assert(is_pid(get(push_inflight))),
+ ?assertEqual(1, length(get(push_inflight_workers)))
+ after
+ reset_push_worker_state()
+ end.
+
+worker_completion_is_counted_test() ->
+ reset_push_worker_state(),
+ Before = read_push_counter(worker_completed),
+ BeforeFailed = read_push_counter(worker_failed),
+ try
+ ?assertEqual(ok, run_counted_push_worker(fun() -> ok end)),
+ ?assertEqual(Before + 1, read_push_counter(worker_completed)),
+ ?assertEqual(BeforeFailed, read_push_counter(worker_failed))
+ after
+ reset_push_worker_state()
+ end.
+
+worker_crash_is_counted_as_failure_not_completion_test() ->
+ reset_push_worker_state(),
+ Before = read_push_counter(worker_failed),
+ BeforeCompleted = read_push_counter(worker_completed),
+ try
+ ?assertEqual(ok, run_counted_push_worker(fun() -> error(boom) end)),
+ ?assertEqual(Before + 1, read_push_counter(worker_failed)),
+ ?assertEqual(BeforeCompleted, read_push_counter(worker_completed))
+ after
+ reset_push_worker_state()
+ end.
+
+spawned_worker_reports_started_and_completed_test() ->
+ reset_push_worker_state(),
+ Self = self(),
+ Started = read_push_counter(worker_started),
+ Completed = read_push_counter(worker_completed),
+ try
+ ?assertEqual(
+ ok,
+ spawn_push_worker(
+ fun() ->
+ Self ! {ran, self()},
+ ok
+ end,
+ 7
+ )
+ ),
+ Pid = get(push_inflight),
+ receive
+ {ran, Pid} -> ok
+ after 1000 -> ?assert(false)
+ end,
+ wait_for_counter(worker_completed, Completed + 1),
+ ?assertEqual(Started + 1, read_push_counter(worker_started)),
+ ?assertEqual(Completed + 1, read_push_counter(worker_completed)),
+ Messages = element(2, process_info(self(), messages)),
+ ?assertEqual([], [M || {'DOWN', _, process, P, _} = M <- Messages, P =:= Pid])
+ after
+ reset_push_worker_state()
+ end.
+
+missing_members_table_is_counted_test() ->
+ reset_push_worker_state(),
+ Before = read_push_counter(members_table_missing),
+ try
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertEqual(Before + 1, read_push_counter(members_table_missing))
+ after
+ reset_push_worker_state()
+ end.
+
+scan_table_unavailable_is_counted_test() ->
+ reset_push_worker_state(),
+ Tab = ets:new(test_members, [set, public]),
+ State = compact_scan_state(Tab),
+ true = ets:delete(Tab),
+ Before = read_push_counter(scan_table_unavailable),
+ try
+ MessageData = #{<<"channel_id">> => <<"10">>},
+ ?assertEqual(ok, send_compact_push_notifications(MessageData, 7, State)),
+ ?assertEqual(Before + 1, read_push_counter(scan_table_unavailable))
+ after
+ reset_push_worker_state()
+ end.
+
+bounded_scan_leaves_the_members_table_unfixed_test() ->
+ reset_push_worker_state(),
+ Tab = ets:new(test_members, [set, public]),
+ try
+ true = ets:insert(Tab, {1, #{<<"roles">> => []}}),
+ true = ets:insert(Tab, {2, #{<<"roles">> => []}}),
+ State = compact_scan_state(Tab),
+ Context = compact_scan_context(#{}, 10, State),
+ ?assertEqual(false, ets:info(Tab, fixed)),
+ {ok, Scan} = scan_push_members(Tab, Context, State),
+ ?assertEqual([1], lists:sort(maps:get(eligible_user_ids, Scan))),
+ ?assertEqual(false, ets:info(Tab, fixed)),
+ ?assertEqual(false, ets:info(Tab, safe_fixed_monotonic_time))
+ after
+ ets:delete(Tab),
+ reset_push_worker_state()
+ end.
+
+bounded_push_concurrency_spawns_second_worker_under_limit_test() ->
+ reset_push_worker_state(),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 3),
+ Blocker = blocking_push_worker(),
+ put(push_inflight, Blocker),
+ put(push_inflight_workers, [{1, Blocker, erlang:monotonic_time(millisecond)}]),
+ try
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertNotEqual(Blocker, get(push_inflight)),
+ ?assertEqual(2, length(get(push_inflight_workers))),
+ ?assert(lists:member(Blocker, worker_pids(get(push_inflight_workers))))
+ after
+ stop_push_worker(Blocker),
+ reset_push_worker_state()
+ end.
+
+bounded_push_concurrency_counts_drops_at_limit_test() ->
+ reset_push_worker_state(),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 2),
+ First = blocking_push_worker(),
+ Second = blocking_push_worker(),
+ Now = erlang:monotonic_time(millisecond),
+ put(push_inflight, Second),
+ put(push_inflight_workers, [{2, Second, Now}, {1, First, Now}]),
+ Before = read_push_counter(dropped_at_limit),
+ try
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertEqual(Second, get(push_inflight)),
+ ?assertEqual([Second, First], worker_pids(get(push_inflight_workers))),
+ ?assertEqual(Before + 1, read_push_counter(dropped_at_limit)),
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertEqual(Before + 2, read_push_counter(dropped_at_limit)),
+ ?assertEqual([Second, First], worker_pids(get(push_inflight_workers)))
+ after
+ stop_push_worker(First),
+ stop_push_worker(Second),
+ reset_push_worker_state()
+ end.
+
+bounded_push_concurrency_reaps_finished_workers_test() ->
+ reset_push_worker_state(),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1),
+ Finished = blocking_push_worker(),
+ stop_push_worker(Finished),
+ put(push_inflight, Finished),
+ put(push_inflight_workers, [{1, Finished, erlang:monotonic_time(millisecond)}]),
+ try
+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),
+ ?assertNot(lists:member(Finished, worker_pids(get(push_inflight_workers))))
+ after
+ reset_push_worker_state()
+ end.
+
+bounded_worker_list_stays_bounded_by_the_limit_test() ->
+ reset_push_worker_state(),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 2),
+ Blockers = [blocking_push_worker() || _ <- lists:seq(1, 6)],
+ Now = erlang:monotonic_time(millisecond),
+ put(push_inflight_workers, [
+ {Gen, Pid, Now}
+ || {Gen, Pid} <- lists:zip(lists:seq(1, 6), Blockers)
+ ]),
+ try
+ lists:foreach(
+ fun(_) -> ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())) end,
+ lists:seq(1, 20)
+ ),
+ ?assert(length(get(push_inflight_workers)) =< 6)
+ after
+ lists:foreach(fun stop_push_worker/1, Blockers),
+ reset_push_worker_state()
+ end.
+
+push_concurrency_limit_defaults_and_clamps_test() ->
+ reset_push_worker_state(),
+ try
+ ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 0),
+ ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, not_a_number),
+ ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1000),
+ ?assertEqual(?MAX_PUSH_CONCURRENCY, push_concurrency_limit()),
+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 6),
+ ?assertEqual(6, push_concurrency_limit())
+ after
+ reset_push_worker_state()
+ end.
+
+legacy_test_state() ->
+ #{id => 7, data => #{}, sessions => #{}}.
+
+worker_pids(Workers) ->
+ [Pid || {_Gen, Pid, _StartedMs} <- Workers].
+
+wait_for_counter(Key, Target) ->
+ wait_for_counter(Key, Target, 200).
+
+wait_for_counter(Key, Target, 0) ->
+ ?assertEqual(Target, read_push_counter(Key));
+wait_for_counter(Key, Target, Retries) ->
+ case read_push_counter(Key) >= Target of
+ true ->
+ ok;
+ false ->
+ timer:sleep(5),
+ wait_for_counter(Key, Target, Retries - 1)
+ end.
+
+remote_test_pid() ->
+ binary_to_term(<<131, 88, 119, 12, "fake@nowhere", 1:32, 0:32, 1:32>>).
+
+read_push_counter(Key) ->
+ try ets:lookup(?PUSH_COUNTERS, Key) of
+ [{Key, Value}] when is_integer(Value) -> Value;
+ _ -> 0
+ catch
+ error:badarg -> 0
+ end.
+
+blocking_push_worker() ->
+ spawn(fun() ->
+ receive
+ stop -> ok
+ end
+ end).
+
+stop_push_worker(Pid) ->
+ Ref = monitor(process, Pid),
+ Pid ! stop,
+ receive
+ {'DOWN', Ref, process, Pid, _Reason} -> ok
+ after 5000 -> error(push_worker_did_not_stop)
+ end.
+
+reset_push_worker_state() ->
+ erase(push_inflight),
+ erase(push_inflight_workers),
+ ok = application:unset_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY).
+
+compact_test_data(Tab) ->
+ #{
+ <<"guild">> => #{
+ <<"id">> => <<"7">>,
+ <<"owner_id">> => <<"999">>,
+ <<"features">> => [<<"COMMUNITY">>],
+ <<"default_message_notifications">> => 0
+ },
+ <<"roles">> => [],
+ <<"role_index">> => #{},
+ <<"channels">> => [#{<<"id">> => <<"10">>, <<"name">> => <<"general">>}],
+ <<"channel_index">> => #{10 => #{<<"id">> => <<"10">>, <<"name">> => <<"general">>}},
+ <<"members">> => #{1 => #{}, 2 => #{}, 3 => #{}},
+ members_normalized => #{1 => #{}, 2 => #{}, 3 => #{}},
+ <<"member_role_index">> => #{},
+ members_ets => Tab
+ }.
+
+compact_scan_state(Tab) ->
+ #{
+ id => 7,
+ data => compact_push_data(compact_test_data(Tab)),
+ virtual_channel_access => #{
+ 1 => sets:from_list([10]),
+ 3 => sets:from_list([10])
+ },
+ members_ets => Tab,
+ session_eligibility => #{1 => true, 2 => true, 3 => false},
+ member_count => 49435
+ }.
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_dispatch_send.erl b/fluxer_gateway/src/guild/guild_dispatch_send.erl
index c3b535513..bf3344ebc 100644
--- a/fluxer_gateway/src/guild/guild_dispatch_send.erl
+++ b/fluxer_gateway/src/guild/guild_dispatch_send.erl
@@ -10,6 +10,8 @@
filter_visible_channels/4
]).
+-define(MAX_BULK_ENCODE_GROUPS, 1024).
+
-type event() :: atom().
-type event_data() :: map().
-type guild_state() :: map().
@@ -17,6 +19,9 @@
-type guild_id() :: integer().
-type user_id() :: integer().
-type session_pair() :: {session_id(), map()}.
+-type bulk_group_key() :: [term()].
+-type bulk_groups() :: #{bulk_group_key() => {[map()], [pid()]}}.
+-type bulk_acc() :: {bulk_groups(), non_neg_integer()}.
-export_type([event/0, event_data/0, guild_state/0, user_id/0, session_pair/0]).
-spec dispatch_to_sessions([session_pair()], event(), event_data(), guild_state()) ->
@@ -45,44 +50,76 @@ dispatch_bulk_update(FilteredSessions, Event, FinalData, UpdatedState) ->
}
|| Ch <- BulkChannels
],
- SuccessCount = lists:foldl(
+ {Groups, Dispatched} = lists:foldl(
fun({_Sid, SessionData}, Acc) ->
- dispatch_bulk_to_one_session_indexed(
+ collect_bulk_recipient(
SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc
)
end,
- 0,
+ {#{}, 0},
FilteredSessions
),
+ SuccessCount = dispatch_bulk_groups(Groups, Event, FinalData, GuildId, Dispatched),
normalize_success(SuccessCount).
--spec dispatch_bulk_to_one_session_indexed(
+-spec collect_bulk_recipient(
map(),
event(),
event_data(),
[{integer() | undefined, map()}],
guild_id(),
guild_state(),
- non_neg_integer()
-) -> non_neg_integer().
-dispatch_bulk_to_one_session_indexed(
- SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc
-) ->
+ bulk_acc()
+) -> bulk_acc().
+collect_bulk_recipient(SessionData, Event, FinalData, IndexedChannels, GuildId, State, Acc) ->
Pid = maps:get(pid, SessionData),
- case
- session_passive:should_receive_event(
- Event, FinalData, GuildId, SessionData, UpdatedState
- )
- of
+ Eligible =
+ is_pid(Pid) andalso
+ session_passive:should_receive_event(Event, FinalData, GuildId, SessionData, State),
+ case Eligible of
false ->
Acc;
true ->
- FilteredChannels = filter_indexed_for_session(
- SessionData, IndexedChannels, UpdatedState
+ Filtered = filter_indexed_for_session(SessionData, IndexedChannels, State),
+ add_bulk_recipient(Pid, Filtered, Event, FinalData, GuildId, Acc)
+ end.
+
+-spec add_bulk_recipient(pid(), [map()], event(), event_data(), guild_id(), bulk_acc()) ->
+ bulk_acc().
+add_bulk_recipient(_Pid, [], _Event, _FinalData, _GuildId, Acc) ->
+ Acc;
+add_bulk_recipient(Pid, FilteredChannels, Event, FinalData, GuildId, {Groups, Dispatched}) ->
+ Key = bulk_group_key(FilteredChannels),
+ case Groups of
+ #{Key := {Channels, Pids}} ->
+ {Groups#{Key := {Channels, [Pid | Pids]}}, Dispatched};
+ _ when map_size(Groups) >= ?MAX_BULK_ENCODE_GROUPS ->
+ Sent = dispatch_bulk_to_pid(
+ Pid, Event, FinalData, FilteredChannels, GuildId, Dispatched
),
- dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc)
+ {Groups, Sent};
+ _ ->
+ {Groups#{Key => {FilteredChannels, [Pid]}}, Dispatched}
end.
+-spec bulk_group_key([map()]) -> bulk_group_key().
+bulk_group_key(FilteredChannels) ->
+ [maps:get(<<"id">>, Ch, undefined) || Ch <- FilteredChannels].
+
+-spec dispatch_bulk_groups(
+ bulk_groups(), event(), event_data(), guild_id(), non_neg_integer()
+) -> non_neg_integer().
+dispatch_bulk_groups(Groups, Event, FinalData, GuildId, Dispatched) ->
+ maps:fold(
+ fun(_Key, {FilteredChannels, Pids}, Acc) ->
+ dispatch_bulk_to_pids(
+ lists:reverse(Pids), Event, FinalData, FilteredChannels, GuildId, Acc
+ )
+ end,
+ Dispatched,
+ Groups
+ ).
+
-spec filter_indexed_for_session(map(), [{integer() | undefined, map()}], guild_state()) ->
[map()].
filter_indexed_for_session(SessionData, IndexedChannels, UpdatedState) ->
@@ -128,12 +165,7 @@ is_channel_visible(Channel, UserId, Member, State) ->
dispatch_bulk_to_pid(_, _, _, [], _GuildId, Acc) ->
Acc;
dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) when is_pid(Pid) ->
- CustomData = FinalData#{<<"channels">> => FilteredChannels},
- EncodedData =
- {pre_encoded,
- iolist_to_binary(
- json:encode(guild_data_wire:payload(CustomData), fun json:encode_value/2)
- )},
+ EncodedData = encode_bulk_payload(FinalData, FilteredChannels),
try
gateway_dispatch_relay:dispatch(Pid, Event, EncodedData, GuildId),
Acc + 1
@@ -143,6 +175,28 @@ dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) when
dispatch_bulk_to_pid(_, _, _, _, _GuildId, Acc) ->
Acc.
+-spec dispatch_bulk_to_pids(
+ [pid()], event(), event_data(), [map()], guild_id(), non_neg_integer()
+) -> non_neg_integer().
+dispatch_bulk_to_pids([], _Event, _FinalData, _FilteredChannels, _GuildId, Acc) ->
+ Acc;
+dispatch_bulk_to_pids(Pids, Event, FinalData, FilteredChannels, GuildId, Acc) ->
+ EncodedData = encode_bulk_payload(FinalData, FilteredChannels),
+ try
+ gateway_dispatch_relay:dispatch_many(Pids, Event, EncodedData, GuildId),
+ Acc + length(Pids)
+ catch
+ _:_ -> Acc
+ end.
+
+-spec encode_bulk_payload(event_data(), [map()]) -> {pre_encoded, binary()}.
+encode_bulk_payload(FinalData, FilteredChannels) ->
+ CustomData = FinalData#{<<"channels">> => FilteredChannels},
+ {pre_encoded,
+ iolist_to_binary(
+ json:encode(guild_data_wire:payload(CustomData), fun json:encode_value/2)
+ )}.
+
-spec dispatch_standard([session_pair()], event(), event_data(), guild_id(), guild_state()) ->
non_neg_integer().
dispatch_standard(FilteredSessions, Event, FinalData, GuildId, State) ->
@@ -205,6 +259,8 @@ normalize_dispatched([_ | _]) -> 1.
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
+-define(OVERFLOW_CHANNEL_BITS, 11).
+
normalize_success_test() ->
?assertEqual(1, normalize_success(5)),
?assertEqual(1, normalize_success(1)),
@@ -287,6 +343,150 @@ passive_channel_update_bulk_dispatches_visible_channels_test() ->
maps:get(<<"channels">>, Payload)
).
+grouped_bulk_encoding_matches_ungrouped_test() ->
+ flush_dispatches(),
+ Data = bulk_channel_data(),
+ State = passive_dispatch_state(),
+ Sessions = [bulk_session_pair(<<"a">>, #{100 => true})],
+ ?assertEqual(1, reference_dispatch_bulk_update(Sessions, channel_update_bulk, Data, State)),
+ Ungrouped = receive_pre_encoded_binary(channel_update_bulk),
+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),
+ ?assertEqual(Ungrouped, receive_pre_encoded_binary(channel_update_bulk)).
+
+reference_dispatch_bulk_update(FilteredSessions, Event, FinalData, UpdatedState) ->
+ GuildId = maps:get(id, UpdatedState),
+ BulkChannels = maps:get(<<"channels">>, FinalData, []),
+ IndexedChannels = [
+ {
+ guild_dispatch_decorate:parse_snowflake(
+ <<"id">>,
+ maps:get(<<"id">>, Ch, undefined)
+ ),
+ Ch
+ }
+ || Ch <- BulkChannels
+ ],
+ SuccessCount = lists:foldl(
+ fun({_Sid, SessionData}, Acc) ->
+ reference_dispatch_bulk_to_one_session_indexed(
+ SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc
+ )
+ end,
+ 0,
+ FilteredSessions
+ ),
+ normalize_success(SuccessCount).
+
+reference_dispatch_bulk_to_one_session_indexed(
+ SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc
+) ->
+ Pid = maps:get(pid, SessionData),
+ case
+ session_passive:should_receive_event(
+ Event, FinalData, GuildId, SessionData, UpdatedState
+ )
+ of
+ false ->
+ Acc;
+ true ->
+ FilteredChannels = filter_indexed_for_session(
+ SessionData, IndexedChannels, UpdatedState
+ ),
+ dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc)
+ end.
+
+grouped_bulk_encodes_once_per_visible_set_test() ->
+ flush_dispatches(),
+ Data = bulk_channel_data(),
+ State = passive_dispatch_state(),
+ Sessions = [
+ bulk_session_pair(<<"a">>, #{100 => true}),
+ bulk_session_pair(<<"b">>, #{100 => true}),
+ bulk_session_pair(<<"c">>, #{200 => true})
+ ],
+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),
+ Bins = [receive_pre_encoded_binary(channel_update_bulk) || _ <- lists:seq(1, 3)],
+ ?assertEqual(2, length(lists:usort(Bins))),
+ Sets = [maps:get(<<"channels">>, json:decode(Bin)) || Bin <- Bins],
+ ?assertEqual(2, length([S || S <- Sets, S =:= [bulk_channel(<<"100">>, <<"first">>)]])),
+ ?assertEqual(1, length([S || S <- Sets, S =:= [bulk_channel(<<"200">>, <<"second">>)]])).
+
+grouped_bulk_skips_sessions_without_visible_channels_test() ->
+ flush_dispatches(),
+ Data = bulk_channel_data(),
+ State = passive_dispatch_state(),
+ Sessions = [
+ bulk_session_pair(<<"a">>, #{}),
+ bulk_session_pair(<<"b">>, #{200 => true})
+ ],
+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),
+ ?assertEqual(
+ [bulk_channel(<<"200">>, <<"second">>)],
+ maps:get(<<"channels">>, receive_pre_encoded_payload(channel_update_bulk))
+ ),
+ ?assertEqual(ok, assert_no_further_dispatch()).
+
+grouped_bulk_dispatches_overflow_recipients_without_retaining_them_test() ->
+ flush_dispatches(),
+ Total = ?MAX_BULK_ENCODE_GROUPS + 1,
+ State = passive_dispatch_state(),
+ Data = #{
+ <<"guild_id">> => <<"42">>,
+ <<"channels">> => [
+ bulk_channel(integer_to_binary(Id), <<"c">>)
+ || Id <- lists:seq(1, ?OVERFLOW_CHANNEL_BITS)
+ ]
+ },
+ Sessions = [overflow_session_pair(N) || N <- lists:seq(1, Total)],
+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),
+ Received = collect_bulk_channel_ids(Total, []),
+ Expected = [overflow_channel_ids(N) || N <- lists:seq(1, Total)],
+ ?assertEqual(lists:sort(Expected), lists:sort(Received)),
+ ?assertEqual(ok, assert_no_further_dispatch()).
+
+overflow_session_pair(N) ->
+ Base = passive_session_data(),
+ Sid = integer_to_binary(N),
+ Viewable = maps:from_list([{Id, true} || Id <- overflow_visible_ids(N)]),
+ {Sid, Base#{session_id => Sid, viewable_channels => Viewable}}.
+
+overflow_visible_ids(N) ->
+ [Id || Id <- lists:seq(1, ?OVERFLOW_CHANNEL_BITS), (N bsr (Id - 1)) band 1 =:= 1].
+
+overflow_channel_ids(N) ->
+ [integer_to_binary(Id) || Id <- overflow_visible_ids(N)].
+
+collect_bulk_channel_ids(0, Acc) ->
+ Acc;
+collect_bulk_channel_ids(N, Acc) ->
+ Payload = receive_pre_encoded_payload(channel_update_bulk),
+ Ids = [maps:get(<<"id">>, Ch) || Ch <- maps:get(<<"channels">>, Payload)],
+ collect_bulk_channel_ids(N - 1, [Ids | Acc]).
+
+bulk_channel_data() ->
+ #{
+ <<"guild_id">> => <<"42">>,
+ <<"channels">> => [
+ bulk_channel(<<"100">>, <<"first">>),
+ bulk_channel(<<"200">>, <<"second">>)
+ ]
+ }.
+
+bulk_channel(Id, Name) ->
+ #{<<"id">> => Id, <<"name">> => Name}.
+
+bulk_session_pair(Sid, ViewableChannels) ->
+ Base = passive_session_data(),
+ {Sid, Base#{session_id => Sid, viewable_channels => ViewableChannels}}.
+
+assert_no_further_dispatch() ->
+ receive
+ {'$gen_cast', {dispatch, Event, _Payload}} ->
+ ?assert(false, {unexpected_dispatch, Event})
+ after 0 ->
+ ok
+ end.
+
standard_dispatch_without_eligible_sessions_test() ->
flush_dispatches(),
Session = (passive_session_data())#{pid => undefined},
@@ -364,9 +564,12 @@ passive_dispatch_state() ->
}.
receive_pre_encoded_payload(Event) ->
+ json:decode(receive_pre_encoded_binary(Event)).
+
+receive_pre_encoded_binary(Event) ->
receive
{'$gen_cast', {dispatch, Event, {pre_encoded, Bin}}} ->
- json:decode(Bin)
+ Bin
after 1000 ->
?assert(false, {dispatch_not_received, Event})
end.
diff --git a/fluxer_gateway/src/guild/guild_init.erl b/fluxer_gateway/src/guild/guild_init.erl
index d1612c538..6640624ec 100644
--- a/fluxer_gateway/src/guild/guild_init.erl
+++ b/fluxer_gateway/src/guild/guild_init.erl
@@ -13,7 +13,11 @@
handle_reload/2
]).
+-define(MAX_RELOAD_CARRY_WARN_MEMBERS, 100000).
+
-type guild_state() :: map().
+-type user_id() :: integer().
+-type member() :: map().
-export_type([guild_state/0]).
@@ -122,7 +126,8 @@ handle_reload(NewData, State) ->
{ReloadVoiceStates, ReloadData} = extract_voice_states_from_data(
NewData, ExistingVoiceStates
),
- NormalizedNewData = guild_data_index:normalize_map(ReloadData),
+ NormalizedNewData0 = guild_data_index:normalize_map(ReloadData),
+ NormalizedNewData = carry_members_table(OldData, NormalizedNewData0),
NewState0 = State#{voice_states => ReloadVoiceStates, data => NormalizedNewData},
NewState1 = guild_availability:handle_unavailability_transition(State, NewState0),
NewState2 = guild_sessions:refresh_all_viewable_channels(NewState1),
@@ -139,6 +144,89 @@ handle_reload(NewData, State) ->
ok = guild_maintenance:maybe_put_permission_cache(NewState),
{reply, ok, NewState}.
+-spec carry_members_table(term(), term()) -> term().
+carry_members_table(OldData, NewData) when is_map(OldData), is_map(NewData) ->
+ carry_healthy_members_table(OldData, NewData);
+carry_members_table(_OldData, NewData) ->
+ NewData.
+
+-spec carry_healthy_members_table(map(), map()) -> map().
+carry_healthy_members_table(OldData, NewData) ->
+ case members_ets_table(OldData) of
+ Tab when is_reference(Tab) -> carry_live_members_table(Tab, OldData, NewData);
+ undefined -> maps:remove(members_ets, NewData)
+ end.
+
+-spec members_ets_table(map()) -> ets:tid() | undefined.
+members_ets_table(#{members_ets := Tab}) ->
+ Tab;
+members_ets_table(_Data) ->
+ undefined.
+
+-spec carry_live_members_table(ets:tid(), map(), map()) -> map().
+carry_live_members_table(Tab, OldData, NewData) ->
+ case guild_members_table_repair:members_table_healthy(Tab) of
+ true -> apply_members_table_delta(Tab, OldData, NewData);
+ false -> maps:remove(members_ets, NewData)
+ end.
+
+-spec apply_members_table_delta(ets:tid(), map(), map()) -> map().
+apply_members_table_delta(Tab, OldData, NewData) ->
+ OldMap = guild_data_index_members:member_map(OldData),
+ NewMap = guild_data_index_members:member_map(NewData),
+ maybe_warn_large_carry(map_size(NewMap)),
+ try
+ ok = insert_changed_members(Tab, OldMap, NewMap),
+ ok = delete_removed_members(Tab, OldMap, NewMap),
+ NewData#{members_ets => Tab}
+ catch
+ error:badarg ->
+ logger:warning(
+ "guild_reload_members_table_carry_failed: members=~p", [map_size(NewMap)]
+ ),
+ maps:remove(members_ets, NewData)
+ end.
+
+-spec insert_changed_members(ets:tid(), map(), map()) -> ok.
+insert_changed_members(Tab, OldMap, NewMap) ->
+ maps:foreach(
+ fun(UserId, Member) -> insert_changed_member(Tab, OldMap, UserId, Member) end,
+ NewMap
+ ).
+
+-spec insert_changed_member(ets:tid(), map(), user_id(), member()) -> ok.
+insert_changed_member(Tab, OldMap, UserId, Member) ->
+ case maps:get(UserId, OldMap, undefined) of
+ Member -> ok;
+ _ -> insert_member_row(Tab, UserId, Member)
+ end.
+
+-spec insert_member_row(ets:tid(), user_id(), member()) -> ok.
+insert_member_row(Tab, UserId, Member) ->
+ true = ets:insert(Tab, {UserId, Member}),
+ ok.
+
+-spec delete_removed_members(ets:tid(), map(), map()) -> ok.
+delete_removed_members(Tab, OldMap, NewMap) ->
+ OldIds = sets:from_list(maps:keys(OldMap)),
+ NewIds = sets:from_list(maps:keys(NewMap)),
+ RemovedIds = sets:to_list(sets:subtract(OldIds, NewIds)),
+ lists:foreach(fun(UserId) -> delete_member_row(Tab, UserId) end, RemovedIds).
+
+-spec delete_member_row(ets:tid(), user_id()) -> ok.
+delete_member_row(Tab, UserId) ->
+ true = ets:delete(Tab, UserId),
+ ok.
+
+-spec maybe_warn_large_carry(non_neg_integer()) -> ok.
+maybe_warn_large_carry(Size) when Size > ?MAX_RELOAD_CARRY_WARN_MEMBERS ->
+ logger:warning(
+ "guild_reload_members_table_carry_large: members=~p threshold=~p",
+ [Size, ?MAX_RELOAD_CARRY_WARN_MEMBERS]
+ );
+maybe_warn_large_carry(_Size) ->
+ ok.
+
-spec collect_active_pid(term(), map(), [pid()]) -> [pid()].
collect_active_pid(_Sid, S, Acc) ->
case maps:get(pending_connect, S, false) of
@@ -168,3 +256,92 @@ maybe_index_voice_state(_, Acc) ->
-spec guild_id(guild_state()) -> integer() | undefined.
guild_id(State) ->
snowflake_id:parse_optional(maps:get(id, State, undefined)).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+carry_members_table_carries_healthy_tid_test() ->
+ Tab = ets:new(carry_members, [set, public]),
+ try
+ OldMap = #{1 => carry_member(1), 2 => carry_member(2)},
+ NewMap = #{1 => carry_member(1), 3 => carry_member(3)},
+ seed_carry_table(Tab, OldMap),
+ OldData = carry_data(OldMap, Tab),
+ Carried = carry_members_table(OldData, carry_data(NewMap, undefined)),
+ ?assertEqual(Tab, maps:get(members_ets, Carried)),
+ ?assertEqual([1, 3], carry_table_ids(Tab))
+ after
+ ets:delete(Tab)
+ end.
+
+carry_members_table_skips_dead_tid_test() ->
+ Tab = ets:new(dead_carry_members, [set, public]),
+ true = ets:delete(Tab),
+ OldData = carry_data(#{1 => carry_member(1)}, Tab),
+ NewData = carry_data(#{1 => carry_member(1)}, undefined),
+ ?assertEqual(NewData, carry_members_table(OldData, NewData)).
+
+carry_members_table_without_old_tid_leaves_new_data_test() ->
+ OldData = carry_data(#{1 => carry_member(1)}, undefined),
+ NewData = carry_data(#{2 => carry_member(2)}, undefined),
+ ?assertEqual(NewData, carry_members_table(OldData, NewData)).
+
+carry_members_table_delta_failure_falls_back_to_new_data_test() ->
+ Tab = ets:new(failing_members, [set, public]),
+ true = ets:delete(Tab),
+ OldData = carry_data(#{}, undefined),
+ NewData = carry_data(#{1 => carry_member(1)}, undefined),
+ ?assertEqual(NewData, apply_members_table_delta(Tab, OldData, NewData)).
+
+carry_delta_never_empties_table_test() ->
+ Tab = ets:new(delta_members, [set, public]),
+ try
+ OldMap = #{1 => carry_member(1), 2 => carry_member(2), 3 => carry_member(3)},
+ NewMap = #{
+ 1 => carry_member(1),
+ 2 => carry_member(2, <<"changed">>),
+ 4 => carry_member(4)
+ },
+ seed_carry_table(Tab, OldMap),
+ ?assertEqual(3, ets:info(Tab, size)),
+ ok = insert_changed_members(Tab, OldMap, NewMap),
+ ?assertEqual(4, ets:info(Tab, size)),
+ ?assertEqual([{1, carry_member(1)}], ets:lookup(Tab, 1)),
+ ok = delete_removed_members(Tab, OldMap, NewMap),
+ ?assertEqual(3, ets:info(Tab, size)),
+ ?assertEqual([{1, carry_member(1)}], ets:lookup(Tab, 1)),
+ ?assertEqual([1, 2, 4], carry_table_ids(Tab)),
+ ?assertEqual([{2, carry_member(2, <<"changed">>)}], ets:lookup(Tab, 2))
+ after
+ ets:delete(Tab)
+ end.
+
+carry_delta_skips_unchanged_members_test() ->
+ Tab = ets:new(unchanged_members, [set, public]),
+ try
+ MemberMap = #{1 => carry_member(1)},
+ true = ets:insert(Tab, {1, sentinel}),
+ ok = insert_changed_members(Tab, MemberMap, MemberMap),
+ ?assertEqual([{1, sentinel}], ets:tab2list(Tab))
+ after
+ ets:delete(Tab)
+ end.
+
+carry_data(MemberMap, undefined) ->
+ #{<<"members">> => MemberMap, members_normalized => MemberMap};
+carry_data(MemberMap, Tab) ->
+ #{<<"members">> => MemberMap, members_normalized => MemberMap, members_ets => Tab}.
+
+seed_carry_table(Tab, MemberMap) ->
+ maps:foreach(fun(UserId, Member) -> ets:insert(Tab, {UserId, Member}) end, MemberMap).
+
+carry_member(UserId) ->
+ #{<<"user">> => #{<<"id">> => UserId}}.
+
+carry_table_ids(Tab) ->
+ lists:sort([Id || {Id, _} <- ets:tab2list(Tab)]).
+
+carry_member(UserId, Nick) ->
+ #{<<"user">> => #{<<"id">> => UserId}, <<"nick">> => Nick}.
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_maintenance.erl b/fluxer_gateway/src/guild/guild_maintenance.erl
index e654c44e0..67e36d156 100644
--- a/fluxer_gateway/src/guild/guild_maintenance.erl
+++ b/fluxer_gateway/src/guild/guild_maintenance.erl
@@ -22,15 +22,18 @@
-type session_id() :: binary().
-type subscription_state() :: guild_subscriptions:subscription_state().
-type guild_state() :: map().
+-type viewable_memo() :: #{exceptions := sets:set(user_id()), cache := #{term() => map()}}.
+-type prune_acc() :: {subscription_state(), #{user_id() => non_neg_integer()}, viewable_memo()}.
-spec update_counts(guild_state()) -> guild_state().
update_counts(State) ->
- Data = maps:get(data, State, #{}),
+ State0 = guild_members_table_repair:maybe_repair(State),
+ Data = maps:get(data, State0, #{}),
MemberCount = guild_data_index:member_count(Data),
- OnlineCount = guild_member_list:get_online_count(State),
- PublicOnlineCount = guild_public_online:compute_count(State),
- ok = maybe_put_guild_count_cache(State, MemberCount, PublicOnlineCount),
- State#{
+ OnlineCount = guild_member_list:get_online_count(State0),
+ PublicOnlineCount = guild_public_online:compute_count(State0),
+ ok = maybe_put_guild_count_cache(State0, MemberCount, PublicOnlineCount),
+ State0#{
member_count => MemberCount,
online_count => OnlineCount,
public_online_count => PublicOnlineCount
@@ -160,30 +163,36 @@ require_subscription_state(MemberSubs) when is_map(MemberSubs) ->
-spec prune_member_subscription_map(subscription_state(), map(), guild_state()) ->
{subscription_state(), #{user_id() => non_neg_integer()}}.
+prune_member_subscription_map(MemberSubs, _Sessions, _State) when map_size(MemberSubs) =:= 0 ->
+ {MemberSubs, #{}};
prune_member_subscription_map(MemberSubs, Sessions, State) ->
- maps:fold(
+ {NewMemberSubs, Counts, _Memo} = maps:fold(
fun(MemberId, Subscribers, Acc) ->
prune_member_subscribers(MemberId, Subscribers, Sessions, State, Acc)
end,
- {MemberSubs, #{}},
+ {MemberSubs, #{}, new_viewable_memo(State)},
MemberSubs
- ).
+ ),
+ {NewMemberSubs, Counts}.
-spec prune_member_subscribers(
term(),
sets:set(session_id()),
map(),
guild_state(),
- {subscription_state(), #{user_id() => non_neg_integer()}}
-) -> {subscription_state(), #{user_id() => non_neg_integer()}}.
-prune_member_subscribers(MemberId, Subscribers, Sessions, State, Acc) when
+ prune_acc()
+) -> prune_acc().
+prune_member_subscribers(MemberId, Subscribers, Sessions, State, {Subs, Counts, Memo}) when
is_integer(MemberId)
->
- MemberViewable = member_viewable_channel_map(MemberId, State),
+ {MemberViewable, Memo1} = memoised_member_viewable_channel_map(MemberId, State, Memo),
{KeptSubscribers, RemovedCount} = prune_subscriber_set(
Subscribers, Sessions, MemberViewable, State
),
- update_pruned_member_subscription(MemberId, KeptSubscribers, RemovedCount, Acc);
+ {NewSubs, NewCounts} = update_pruned_member_subscription(
+ MemberId, KeptSubscribers, RemovedCount, {Subs, Counts}
+ ),
+ {NewSubs, NewCounts, Memo1};
prune_member_subscribers(_MemberId, _Subscribers, _Sessions, _State, Acc) ->
Acc.
@@ -204,24 +213,152 @@ member_viewable_channel_map(MemberId, State) ->
guild_visibility:get_user_viewable_channels(MemberId, State)
).
+%% One pass reads a fixed state, and get_user_viewable_channels/2 reads nothing from a
+%% member except its raw roles term, so two members holding the same term see the same
+%% channels. The user-specific inputs are the guild owner, virtual channel access and
+%% user-type permission overwrites, and every user named by one of those bypasses the
+%% memo. A term is still computed the first time the pass meets it, so a term whose
+%% computation raises raises on the same member it does without the memo.
+-spec new_viewable_memo(guild_state()) -> viewable_memo().
+new_viewable_memo(State) ->
+ #{exceptions => viewable_exceptions(State), cache => #{}}.
+
+-spec memoised_member_viewable_channel_map(user_id(), guild_state(), viewable_memo()) ->
+ {map(), viewable_memo()}.
+memoised_member_viewable_channel_map(MemberId, State, Memo) ->
+ #{exceptions := Exceptions, cache := Cache} = Memo,
+ case memo_key(MemberId, Exceptions, State) of
+ bypass ->
+ {member_viewable_channel_map(MemberId, State), Memo};
+ {ok, RawRoles} ->
+ cached_member_viewable_channel_map(MemberId, RawRoles, State, Cache, Memo)
+ end.
+
+-spec cached_member_viewable_channel_map(
+ user_id(), term(), guild_state(), #{term() => map()}, viewable_memo()
+) -> {map(), viewable_memo()}.
+cached_member_viewable_channel_map(MemberId, RawRoles, State, Cache, Memo) ->
+ case maps:find(RawRoles, Cache) of
+ {ok, Viewable} ->
+ {Viewable, Memo};
+ error ->
+ Viewable = member_viewable_channel_map(MemberId, State),
+ {Viewable, Memo#{cache := Cache#{RawRoles => Viewable}}}
+ end.
+
+-spec memo_key(user_id(), sets:set(user_id()), guild_state()) -> {ok, term()} | bypass.
+memo_key(MemberId, Exceptions, State) ->
+ case sets:is_element(MemberId, Exceptions) of
+ true -> bypass;
+ false -> member_roles_key(guild_permissions:find_member_by_user_id(MemberId, State))
+ end.
+
+-spec member_roles_key(map() | undefined) -> {ok, term()} | bypass.
+member_roles_key(Member) when is_map(Member) ->
+ {ok, maps:get(<<"roles">>, Member, [])};
+member_roles_key(_Member) ->
+ bypass.
+
+-spec viewable_exceptions(guild_state()) -> sets:set(user_id()).
+viewable_exceptions(State) ->
+ Data = map_utils:ensure_map(map_utils:get_safe(State, data, #{})),
+ Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})),
+ Owner = owner_ids(maps:get(<<"owner_id">>, Guild, undefined)),
+ Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))),
+ sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(Data)).
+
+%% parse_maybe/1, not parse_optional/1: an id this rejects makes the permission check
+%% itself raise before it can match a user, so leaving it out keeps the memo exact.
+-spec owner_ids(term()) -> [user_id()].
+owner_ids(OwnerIdRaw) ->
+ case snowflake_id:parse_maybe(OwnerIdRaw) of
+ OwnerId when is_integer(OwnerId) -> [OwnerId];
+ _ -> []
+ end.
+
+%% Both places a permission check can take overwrites from, and role targets alongside
+%% user targets. Role ids can never collide with user ids, so the superset only costs a
+%% few members their memo entry.
+-spec overwrite_target_ids(map()) -> [user_id()].
+overwrite_target_ids(Data) ->
+ Cached = maps:fold(
+ fun(_ChannelId, Entries, Acc) -> cached_target_ids(Entries, Acc) end,
+ [],
+ map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{}))
+ ),
+ maps:fold(
+ fun(_ChannelId, Channel, Acc) -> channel_target_ids(Channel, Acc) end,
+ Cached,
+ guild_data_index:channel_index(Data)
+ ).
+
+-spec cached_target_ids(term(), [user_id()]) -> [user_id()].
+cached_target_ids(Entries, Acc) when is_list(Entries) ->
+ [Id || {Id, _Type, _Allow, _Deny} <- Entries, is_integer(Id)] ++ Acc;
+cached_target_ids(_Entries, Acc) ->
+ Acc.
+
+-spec channel_target_ids(term(), [user_id()]) -> [user_id()].
+channel_target_ids(Channel, Acc) when is_map(Channel) ->
+ lists:foldl(
+ fun overwrite_target_id/2,
+ Acc,
+ map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, []))
+ );
+channel_target_ids(_Channel, Acc) ->
+ Acc.
+
+-spec overwrite_target_id(term(), [user_id()]) -> [user_id()].
+overwrite_target_id(Overwrite, Acc) when is_map(Overwrite) ->
+ case snowflake_id:parse_maybe(maps:get(<<"id">>, Overwrite, undefined)) of
+ Id when is_integer(Id) -> [Id | Acc];
+ _ -> Acc
+ end;
+overwrite_target_id(_Overwrite, Acc) ->
+ Acc.
+
+%% update_pruned_member_subscription/4 drops the whole result when nothing was removed,
+%% so the kept set is only ever built when Removed is non-empty, and then by the same
+%% fold over the same subscribers as before.
-spec prune_subscriber_set(sets:set(session_id()), map(), map(), guild_state()) ->
{sets:set(session_id()), non_neg_integer()}.
prune_subscriber_set(Subscribers, Sessions, MemberViewable, State) ->
- sets:fold(
+ Removed = sets:fold(
fun(SessionId, Acc) ->
- prune_subscriber(SessionId, Sessions, MemberViewable, State, Acc)
+ collect_removed_subscriber(SessionId, Sessions, MemberViewable, State, Acc)
end,
- {sets:new(), 0},
+ #{},
Subscribers
- ).
+ ),
+ keep_remaining_subscribers(Subscribers, Removed).
--spec prune_subscriber(
- session_id(), map(), map(), guild_state(), {sets:set(session_id()), non_neg_integer()}
-) -> {sets:set(session_id()), non_neg_integer()}.
-prune_subscriber(SessionId, Sessions, MemberViewable, State, {Kept, RemovedCount}) ->
+-spec collect_removed_subscriber(
+ session_id(), map(), map(), guild_state(), #{session_id() => true}
+) -> #{session_id() => true}.
+collect_removed_subscriber(SessionId, Sessions, MemberViewable, State, Removed) ->
case subscriber_can_still_view_member(SessionId, Sessions, MemberViewable, State) of
- true -> {sets:add_element(SessionId, Kept), RemovedCount};
- false -> {Kept, RemovedCount + 1}
+ true -> Removed;
+ false -> Removed#{SessionId => true}
+ end.
+
+-spec keep_remaining_subscribers(sets:set(session_id()), #{session_id() => true}) ->
+ {sets:set(session_id()), non_neg_integer()}.
+keep_remaining_subscribers(Subscribers, Removed) when map_size(Removed) =:= 0 ->
+ {Subscribers, 0};
+keep_remaining_subscribers(Subscribers, Removed) ->
+ Kept = sets:fold(
+ fun(SessionId, Acc) -> add_kept_subscriber(SessionId, Removed, Acc) end,
+ sets:new(),
+ Subscribers
+ ),
+ {Kept, map_size(Removed)}.
+
+-spec add_kept_subscriber(session_id(), #{session_id() => true}, sets:set(session_id())) ->
+ sets:set(session_id()).
+add_kept_subscriber(SessionId, Removed, Kept) ->
+ case maps:is_key(SessionId, Removed) of
+ true -> Kept;
+ false -> sets:add_element(SessionId, Kept)
end.
-spec subscriber_can_still_view_member(session_id(), map(), map(), guild_state()) ->
@@ -399,8 +536,8 @@ prune_test_state() ->
30 => prune_member(30, [OtherRole])
},
<<"channels">> => [
- prune_channel(ChannelA, ViewerRole),
- prune_channel(ChannelB, OtherRole)
+ prune_channel(ChannelA, [prune_overwrite(ViewerRole, 0)]),
+ prune_channel(ChannelB, [prune_overwrite(OtherRole, 0)])
]
}
}.
@@ -417,20 +554,186 @@ prune_member(UserId, RoleIds) ->
<<"roles">> => [integer_to_binary(RoleId) || RoleId <- RoleIds]
}.
-prune_channel(ChannelId, RoleId) ->
- ViewPerm = constants:view_channel_permission(),
+update_counts_repairs_missing_members_table_test() ->
+ ok = application:set_env(fluxer_gateway, guild_members_table_self_heal, true),
+ try
+ Result = update_counts(repair_counts_state()),
+ Tab = maps:get(members_ets, maps:get(data, Result)),
+ ?assert(is_reference(Tab)),
+ ?assertNot(maps:is_key(members_ets_repair, Result)),
+ ?assertEqual([1, 2], lists:sort([Id || {Id, _} <- ets:tab2list(Tab)])),
+ ?assertEqual(2, maps:get(member_count, Result)),
+ ets:delete(Tab)
+ after
+ application:unset_env(fluxer_gateway, guild_members_table_self_heal)
+ end.
+
+repair_counts_state() ->
+ MemberMap = #{
+ 1 => #{<<"user">> => #{<<"id">> => 1}},
+ 2 => #{<<"user">> => #{<<"id">> => 2}}
+ },
+ #{
+ id => 42,
+ disable_guild_count_cache_updates => true,
+ data => #{
+ <<"channels">> => [],
+ <<"members">> => MemberMap,
+ members_normalized => MemberMap
+ }
+ }.
+
+prune_channel(ChannelId, Overwrites) ->
#{
<<"id">> => integer_to_binary(ChannelId),
- <<"permission_overwrites">> => [
- #{
- <<"id">> => integer_to_binary(RoleId),
- <<"type">> => 0,
- <<"allow">> => integer_to_binary(ViewPerm),
- <<"deny">> => <<"0">>
- }
- ]
+ <<"permission_overwrites">> => Overwrites
+ }.
+
+prune_overwrite(TargetId, Type) ->
+ #{
+ <<"id">> => integer_to_binary(TargetId),
+ <<"type">> => Type,
+ <<"allow">> => integer_to_binary(constants:view_channel_permission()),
+ <<"deny">> => <<"0">>
+ }.
+
+%% Members 30, 31, 99 and 4242 all carry the same roles term and reach only channel 600
+%% through it, but 99 holds virtual access to channel 500 and 4242 is a user-overwrite
+%% target on 500, so neither may be answered from that term. Member 7 owns the guild,
+%% 777 is subscribed without being a member, and session s3 has no cached viewable map.
+memo_prune_state() ->
+ GuildId = 42,
+ ViewerRole = 1000,
+ OtherRole = 2000,
+ Members = maps:from_list(
+ [{7, prune_member(7, [])}] ++
+ [{Id, prune_member(Id, [ViewerRole])} || Id <- [10, 20, 21]] ++
+ [{Id, prune_member(Id, [OtherRole])} || Id <- [30, 31, 99, 4242]]
+ ),
+ #{
+ id => GuildId,
+ virtual_channel_access => #{99 => sets:from_list([500])},
+ sessions => #{
+ <<"s1">> => #{user_id => 10, viewable_channels => #{500 => true}},
+ <<"s2">> => #{user_id => 30, viewable_channels => #{600 => true}},
+ <<"s3">> => #{user_id => 10}
+ },
+ member_subscriptions => memo_prune_subscriptions(),
+ presence_subscriptions => maps:from_list(
+ [{Id, 3} || Id <- [7, 20, 21, 30, 31, 99, 4242, 777]]
+ ),
+ data => #{
+ <<"guild">> => #{<<"owner_id">> => <<"7">>},
+ <<"roles">> => [
+ prune_role(GuildId, 0),
+ prune_role(ViewerRole, 0),
+ prune_role(OtherRole, 0)
+ ],
+ <<"members">> => Members,
+ <<"channels">> => [
+ prune_channel(500, [prune_overwrite(ViewerRole, 0), prune_overwrite(4242, 1)]),
+ prune_channel(600, [prune_overwrite(OtherRole, 0)])
+ ]
+ }
}.
+memo_prune_subscriptions() ->
+ lists:foldl(
+ fun({SessionId, MemberIds}, Acc) ->
+ guild_subscriptions:update_subscriptions(SessionId, MemberIds, Acc)
+ end,
+ guild_subscriptions:init_state(),
+ [
+ {<<"s1">>, [20, 21, 30, 31, 7, 99, 4242, 777]},
+ {<<"s2">>, [20, 30]},
+ {<<"s3">>, [20, 30]},
+ {<<"missing">>, [31]}
+ ]
+ ).
+
+%% prune_member_subscription_map/3 as it read before the memo: every subscribed member
+%% materialises its own viewable channel map and every subscriber set is rebuilt.
+reference_prune_member_subscription_map(MemberSubs, Sessions, State) ->
+ maps:fold(
+ fun(MemberId, Subscribers, Acc) ->
+ reference_prune_member(MemberId, Subscribers, Sessions, State, Acc)
+ end,
+ {MemberSubs, #{}},
+ MemberSubs
+ ).
+
+reference_prune_member(MemberId, Subscribers, Sessions, State, Acc) when is_integer(MemberId) ->
+ MemberViewable = member_viewable_channel_map(MemberId, State),
+ {Kept, RemovedCount} = reference_prune_subscriber_set(
+ Subscribers, Sessions, MemberViewable, State
+ ),
+ update_pruned_member_subscription(MemberId, Kept, RemovedCount, Acc);
+reference_prune_member(_MemberId, _Subscribers, _Sessions, _State, Acc) ->
+ Acc.
+
+reference_prune_subscriber_set(Subscribers, Sessions, MemberViewable, State) ->
+ sets:fold(
+ fun(SessionId, Acc) ->
+ reference_prune_subscriber(SessionId, Sessions, MemberViewable, State, Acc)
+ end,
+ {sets:new(), 0},
+ Subscribers
+ ).
+
+reference_prune_subscriber(SessionId, Sessions, MemberViewable, State, {Kept, RemovedCount}) ->
+ case subscriber_can_still_view_member(SessionId, Sessions, MemberViewable, State) of
+ true -> {sets:add_element(SessionId, Kept), RemovedCount};
+ false -> {Kept, RemovedCount + 1}
+ end.
+
+prune_member_subscription_map_matches_unmemoised_reference_test() ->
+ State = memo_prune_state(),
+ MemberSubs = member_subscriptions(State),
+ Sessions = maps:get(sessions, State),
+ ?assertEqual(
+ reference_prune_member_subscription_map(MemberSubs, Sessions, State),
+ prune_member_subscription_map(MemberSubs, Sessions, State)
+ ).
+
+prune_keeps_exception_members_a_session_can_still_see_test() ->
+ Result = maybe_prune_invalid_member_subscriptions(guild_role_update, memo_prune_state()),
+ MemberSubs = maps:get(member_subscriptions, Result),
+ Presence = #{7 => 3, 20 => 2, 21 => 3, 30 => 1, 31 => 1, 99 => 3, 4242 => 3, 777 => 2},
+ ?assertEqual(
+ [[<<"s1">>], [<<"s1">>], [<<"s1">>], [], []],
+ [
+ lists:sort(guild_subscriptions:get_subscribed_sessions(Id, MemberSubs))
+ || Id <- [7, 99, 4242, 31, 777]
+ ]
+ ),
+ ?assertEqual(Presence, maps:get(presence_subscriptions, Result)).
+
+memo_reuses_one_entry_per_roles_term_test() ->
+ State = memo_prune_state(),
+ {First, Memo1} = memoised_member_viewable_channel_map(20, State, new_viewable_memo(State)),
+ {Second, Memo2} = memoised_member_viewable_channel_map(21, State, Memo1),
+ ?assertEqual(member_viewable_channel_map(21, State), Second),
+ ?assertEqual(First, Second),
+ ?assertEqual([1, 1], [map_size(maps:get(cache, M)) || M <- [Memo1, Memo2]]).
+
+memo_never_answers_owner_virtual_or_overwrite_targets_from_the_cache_test() ->
+ State = memo_prune_state(),
+ {_, Memo} = memoised_member_viewable_channel_map(30, State, new_viewable_memo(State)),
+ Answers = [memoised_member_viewable_channel_map(Id, State, Memo) || Id <- [7, 99, 4242]],
+ ?assertEqual(
+ [member_viewable_channel_map(Id, State) || Id <- [7, 99, 4242]],
+ [Viewable || {Viewable, _} <- Answers]
+ ),
+ ?assertEqual([Memo, Memo, Memo], [M || {_, M} <- Answers]).
+
+prune_subscriber_set_returns_the_original_set_when_nothing_is_removed_test() ->
+ State = memo_prune_state(),
+ Subscribers = sets:from_list([<<"s1">>, <<"s3">>]),
+ Viewable = member_viewable_channel_map(20, State),
+ ?assertEqual(
+ {Subscribers, 0},
+ prune_subscriber_set(Subscribers, maps:get(sessions, State), Viewable, State)
+ ).
permission_cache_skips_nick_only_member_update_test() ->
UserId = 4242,
OldState = permission_cache_state(
diff --git a/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl b/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl
index 8fb67b034..f18f3fec0 100644
--- a/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl
+++ b/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl
@@ -23,6 +23,8 @@
-type guild_state() :: map().
-type list_id() :: binary().
+-type visibility_memo() :: disabled | #{exceptions := sets:set(integer()), cache := map()}.
+-type role_cache() :: #{term() => {[integer()], [integer()]}}.
-type engine_ref() :: ets:table().
-export_type([guild_state/0, list_id/0, engine_ref/0]).
@@ -244,13 +246,21 @@ build_inputs(ChannelId, State) ->
MemberMap = guild_data_index:member_map(Data),
ConnectedUserIds = guild_member_list_common:connected_session_user_ids(State),
PresenceTab = maps:get(member_presence, State),
- Tuples = maps:fold(
- fun(UserId, Member, Acc) ->
+ {Tuples, _Memo, _RoleCache} = maps:fold(
+ fun(UserId, Member, {Acc, Memo, RoleCache}) ->
maybe_prepare_visible_member_tuple(
- UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc
+ UserId,
+ Member,
+ ChannelId,
+ State,
+ PresenceTab,
+ ConnectedUserIds,
+ Acc,
+ Memo,
+ RoleCache
)
end,
- [],
+ {[], new_visibility_memo(ChannelId, Data, State), #{}},
MemberMap
),
Roles = map_utils:ensure_list(maps:get(<<"roles">>, Data, [])),
@@ -264,30 +274,165 @@ build_inputs(ChannelId, State) ->
{Tuples, HoistedRoleIds}.
-spec maybe_prepare_visible_member_tuple(
- integer(), map(), pos_integer(), guild_state(), ets:tid() | map(), sets:set(integer()), [
- guild_member_list_store:member_tuple()
- ]
-) -> [guild_member_list_store:member_tuple()].
+ integer(),
+ map(),
+ pos_integer(),
+ guild_state(),
+ ets:tid() | map(),
+ sets:set(integer()),
+ [guild_member_list_store:member_tuple()],
+ visibility_memo(),
+ role_cache()
+) -> {[guild_member_list_store:member_tuple()], visibility_memo(), role_cache()}.
maybe_prepare_visible_member_tuple(
- UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc
+ UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc, Memo, RoleCache
) ->
- case can_view(UserId, ChannelId, Member, State) of
- true -> prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, Acc);
- false -> Acc
+ {RoleIds, SortedRoleIds, RoleCache1} = member_role_ids(Member, RoleCache),
+ {Visible, Memo1} = memoised_can_view(UserId, ChannelId, Member, SortedRoleIds, State, Memo),
+ case Visible of
+ true ->
+ {
+ prepare_member_tuple(
+ UserId, Member, PresenceTab, ConnectedUserIds, RoleIds, Acc
+ ),
+ Memo1,
+ RoleCache1
+ };
+ false ->
+ {Acc, Memo1, RoleCache1}
+ end.
+
+%% extract_role_ids/1 and the sort that forms the visibility memo key are both
+%% pure functions of the member's raw roles term, so caching on that term
+%% collapses them to the number of distinct role combinations.
+-spec member_role_ids(map(), role_cache()) -> {[integer()], [integer()], role_cache()}.
+member_role_ids(Member, RoleCache) ->
+ RawRoles = maps:get(<<"roles">>, Member, []),
+ case maps:find(RawRoles, RoleCache) of
+ {ok, {RoleIds, SortedRoleIds}} ->
+ {RoleIds, SortedRoleIds, RoleCache};
+ error ->
+ RoleIds = guild_member_list_store:extract_role_ids(Member),
+ SortedRoleIds = lists:sort(RoleIds),
+ {RoleIds, SortedRoleIds, RoleCache#{RawRoles => {RoleIds, SortedRoleIds}}}
end.
--spec prepare_member_tuple(integer(), map(), ets:tid() | map(), sets:set(integer()), [
- guild_member_list_store:member_tuple()
-]) -> [guild_member_list_store:member_tuple()].
-prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, Acc) ->
+%% can_view_channel/4 reads nothing from the member except its role ids, so for a
+%% non-category channel its result is a function of the role set. The only
+%% user-specific inputs are the guild owner, virtual channel access, and the
+%% channel's own permission overwrites, and those users bypass the memo.
+-spec new_visibility_memo(pos_integer(), map(), guild_state()) -> visibility_memo().
+new_visibility_memo(ChannelId, Data, State) ->
+ case memoisable_channel(ChannelId, Data) of
+ true ->
+ #{
+ exceptions => visibility_exceptions(ChannelId, Data, State),
+ cache => #{}
+ };
+ false ->
+ disabled
+ end.
+
+-spec memoised_can_view(
+ integer(), pos_integer(), map(), [integer()], guild_state(), visibility_memo()
+) -> {boolean(), visibility_memo()}.
+memoised_can_view(UserId, ChannelId, Member, _SortedRoleIds, State, disabled) ->
+ {can_view(UserId, ChannelId, Member, State), disabled};
+memoised_can_view(UserId, ChannelId, Member, SortedRoleIds, State, Memo) ->
+ #{exceptions := Exceptions, cache := Cache} = Memo,
+ case sets:is_element(UserId, Exceptions) of
+ true ->
+ {can_view(UserId, ChannelId, Member, State), Memo};
+ false ->
+ case maps:find(SortedRoleIds, Cache) of
+ {ok, Visible} ->
+ {Visible, Memo};
+ error ->
+ Visible = can_view(UserId, ChannelId, Member, State),
+ {Visible, Memo#{cache := Cache#{SortedRoleIds => Visible}}}
+ end
+ end.
+
+%% Category visibility recurses into child channels, each with its own
+%% overwrites, so the exception set is not local to this channel. Do not memoise.
+-spec memoisable_channel(pos_integer(), map()) -> boolean().
+memoisable_channel(ChannelId, Data) ->
+ case maps:get(ChannelId, guild_data_index:channel_index(Data), undefined) of
+ Channel when is_map(Channel) -> maps:get(<<"type">>, Channel, undefined) =/= 4;
+ _ -> false
+ end.
+
+-spec visibility_exceptions(pos_integer(), map(), guild_state()) -> sets:set(integer()).
+visibility_exceptions(ChannelId, Data, State) ->
+ Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})),
+ Owner =
+ case snowflake_id:parse_optional(maps:get(<<"owner_id">>, Guild, undefined)) of
+ OwnerId when is_integer(OwnerId) -> [OwnerId];
+ _ -> []
+ end,
+ Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))),
+ sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(ChannelId, Data)).
+
+%% Every overwrite target on the channel, role and user alike. Role ids can never
+%% collide with user ids, so the superset costs only a few direct computations.
+-spec overwrite_target_ids(pos_integer(), map()) -> [integer()].
+overwrite_target_ids(ChannelId, Data) ->
+ Cache = map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{})),
+ case maps:get(ChannelId, Cache, undefined) of
+ Cached when is_list(Cached) ->
+ [Id || {Id, _Type, _Allow, _Deny} <- Cached, is_integer(Id)];
+ _ ->
+ channel_overwrite_target_ids(ChannelId, Data)
+ end.
+
+-spec channel_overwrite_target_ids(pos_integer(), map()) -> [integer()].
+channel_overwrite_target_ids(ChannelId, Data) ->
+ case maps:get(ChannelId, guild_data_index:channel_index(Data), undefined) of
+ Channel when is_map(Channel) ->
+ lists:filtermap(
+ fun(Overwrite) -> overwrite_target_id(Overwrite) end,
+ map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, []))
+ );
+ _ ->
+ []
+ end.
+
+-spec overwrite_target_id(term()) -> {true, integer()} | false.
+overwrite_target_id(Overwrite) when is_map(Overwrite) ->
+ case snowflake_id:parse_optional(maps:get(<<"id">>, Overwrite, undefined)) of
+ Id when is_integer(Id) -> {true, Id};
+ _ -> false
+ end;
+overwrite_target_id(_Overwrite) ->
+ false.
+
+-spec prepare_member_tuple(
+ integer(),
+ map(),
+ ets:tid() | map(),
+ sets:set(integer()),
+ [integer()],
+ [guild_member_list_store:member_tuple()]
+) -> [guild_member_list_store:member_tuple()].
+prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, RoleIds, Acc) ->
DisplayName = guild_member_list_common:get_member_display_name(Member),
SortKey = guild_member_list_common:casefold_binary(DisplayName),
- RoleIds = guild_member_list_store:extract_role_ids(Member),
+ IsOnline = member_is_online(UserId, PresenceTab, ConnectedUserIds),
+ [{UserId, SortKey, RoleIds, IsOnline} | Acc].
+
+%% Presence is consulted only to decide IsOnline, and IsOnline is false for anyone
+%% without a connected session, so the ETS read and its default-presence
+%% allocation are skipped for every member who is not connected.
+-spec member_is_online(integer(), ets:tid() | map(), sets:set(integer())) -> boolean().
+member_is_online(UserId, PresenceTab, ConnectedUserIds) ->
+ sets:is_element(UserId, ConnectedUserIds) andalso
+ presence_status_is_online(UserId, PresenceTab).
+
+-spec presence_status_is_online(integer(), ets:tid() | map()) -> boolean().
+presence_status_is_online(UserId, PresenceTab) ->
Presence = guild_state_member:lookup_presence(PresenceTab, UserId),
Status = maps:get(<<"status">>, Presence, <<"offline">>),
- IsConnected = sets:is_element(UserId, ConnectedUserIds),
- IsOnline = IsConnected andalso Status =/= <<"offline">> andalso Status =/= <<"invisible">>,
- [{UserId, SortKey, RoleIds, IsOnline} | Acc].
+ Status =/= <<"offline">> andalso Status =/= <<"invisible">>.
-spec can_view(integer(), pos_integer(), map(), guild_state()) -> boolean().
can_view(UserId, ChannelId, Member, State) when is_integer(UserId), UserId > 0 ->
@@ -346,4 +491,91 @@ sync_online_noop_without_engines_test() ->
ok, sync_online(1, true, #{data => #{<<"guild">> => #{<<"features">> => []}}})
).
+memo_channel_state() ->
+ #{
+ data => #{
+ <<"guild">> => #{<<"features">> => [], <<"owner_id">> => <<"7">>},
+ <<"channel_index">> => #{
+ 500 => #{
+ <<"id">> => <<"500">>,
+ <<"type">> => 0,
+ <<"permission_overwrites">> => [#{<<"id">> => <<"42">>, <<"type">> => 1}]
+ },
+ 600 => #{<<"id">> => <<"600">>, <<"type">> => 4}
+ }
+ },
+ virtual_channel_access => #{99 => sets:from_list([500])}
+ }.
+
+memoisable_channel_skips_categories_test() ->
+ Data = maps:get(data, memo_channel_state()),
+ ?assert(memoisable_channel(500, Data)),
+ ?assertNot(memoisable_channel(600, Data)),
+ ?assertNot(memoisable_channel(777, Data)).
+
+visibility_exceptions_cover_owner_virtual_and_overwrites_test() ->
+ State = memo_channel_state(),
+ Exceptions = visibility_exceptions(500, maps:get(data, State), State),
+ ?assert(sets:is_element(7, Exceptions)),
+ ?assert(sets:is_element(99, Exceptions)),
+ ?assert(sets:is_element(42, Exceptions)),
+ ?assertNot(sets:is_element(1234, Exceptions)).
+
+new_visibility_memo_is_disabled_for_categories_test() ->
+ State = memo_channel_state(),
+ ?assertEqual(disabled, new_visibility_memo(600, maps:get(data, State), State)),
+ ?assertNotEqual(disabled, new_visibility_memo(500, maps:get(data, State), State)).
+
+memoised_can_view_caches_one_entry_per_role_set_test() ->
+ State = memo_channel_state(),
+ Memo0 = new_visibility_memo(500, maps:get(data, State), State),
+ {First, Memo1} = memoised_can_view(
+ 1000, 500, #{<<"roles">> => [2, 1]}, lists:sort([2, 1]), State, Memo0
+ ),
+ ?assertEqual(1, map_size(maps:get(cache, Memo1))),
+ {Second, Memo2} = memoised_can_view(
+ 1001, 500, #{<<"roles">> => [1, 2]}, lists:sort([1, 2]), State, Memo1
+ ),
+ ?assertEqual(First, Second),
+ ?assertEqual(1, map_size(maps:get(cache, Memo2))).
+
+memoised_can_view_bypasses_exception_users_test() ->
+ State = memo_channel_state(),
+ Memo0 = new_visibility_memo(500, maps:get(data, State), State),
+ {_, Memo1} = memoised_can_view(7, 500, #{<<"roles">> => []}, [], State, Memo0),
+ ?assertEqual(0, map_size(maps:get(cache, Memo1))),
+ {_, Memo2} = memoised_can_view(42, 500, #{<<"roles">> => []}, [], State, Memo1),
+ ?assertEqual(0, map_size(maps:get(cache, Memo2))).
+
+memoised_can_view_disabled_never_caches_test() ->
+ State = memo_channel_state(),
+ ?assertMatch(
+ {_, disabled},
+ memoised_can_view(1000, 600, #{<<"roles">> => [1]}, [1], State, disabled)
+ ).
+
+member_role_ids_caches_and_sorts_test() ->
+ Member = #{<<"roles">> => [3, 1, 2]},
+ {RoleIds, Sorted, Cache1} = member_role_ids(Member, #{}),
+ ?assertEqual([1, 2, 3], Sorted),
+ ?assertEqual(lists:sort(RoleIds), Sorted),
+ ?assertEqual(1, map_size(Cache1)),
+ {RoleIds2, Sorted2, Cache2} = member_role_ids(#{<<"roles">> => [3, 1, 2]}, Cache1),
+ ?assertEqual({RoleIds, Sorted}, {RoleIds2, Sorted2}),
+ ?assertEqual(1, map_size(Cache2)),
+ {_, _, Cache3} = member_role_ids(#{<<"roles">> => [9]}, Cache2),
+ ?assertEqual(2, map_size(Cache3)).
+
+offline_member_skips_the_presence_lookup_test() ->
+ Presences = #{7 => #{<<"status">> => <<"online">>}},
+ Connected = sets:from_list([7]),
+ ?assert(member_is_online(7, Presences, Connected)),
+ ?assertNot(member_is_online(8, Presences, Connected)),
+ ?assertNot(member_is_online(7, Presences, sets:new())).
+
+invisible_member_is_not_online_test() ->
+ Connected = sets:from_list([7]),
+ ?assertNot(member_is_online(7, #{7 => #{<<"status">> => <<"invisible">>}}, Connected)),
+ ?assertNot(member_is_online(7, #{7 => #{<<"status">> => <<"offline">>}}, Connected)).
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_member_list_common.erl b/fluxer_gateway/src/guild/guild_member_list_common.erl
index 6da4efe9f..75800e79e 100644
--- a/fluxer_gateway/src/guild/guild_member_list_common.erl
+++ b/fluxer_gateway/src/guild/guild_member_list_common.erl
@@ -65,11 +65,45 @@ get_member_sort_key(Member) ->
-spec casefold_binary(term()) -> binary().
casefold_binary(Value) ->
Bin = normalize_name(Value),
- case unicode_chardata_to_binary(string:casefold(Bin)) of
- {ok, Folded} -> Folded;
- error -> Bin
+ case ascii_casefold(Bin) of
+ {ok, Folded} ->
+ Folded;
+ not_ascii ->
+ case unicode_chardata_to_binary(string:casefold(Bin)) of
+ {ok, Folded} -> Folded;
+ error -> Bin
+ end
end.
+%% Case folding a pure-ASCII binary is exactly A-Z -> a-z, so a byte pass is
+%% equivalent to string:casefold/1 and skips two full unicode traversals per
+%% member. Anything with a byte above 127 falls through to the unicode path.
+-spec ascii_casefold(binary()) -> {ok, binary()} | not_ascii.
+ascii_casefold(Bin) ->
+ case scan_ascii(Bin, false) of
+ not_ascii -> not_ascii;
+ false -> {ok, Bin};
+ true -> {ok, lower_ascii(Bin, <<>>)}
+ end.
+
+-spec scan_ascii(binary(), boolean()) -> boolean() | not_ascii.
+scan_ascii(<<>>, HasUpper) ->
+ HasUpper;
+scan_ascii(<<C, Rest/binary>>, _HasUpper) when C >= $A, C =< $Z ->
+ scan_ascii(Rest, true);
+scan_ascii(<<C, Rest/binary>>, HasUpper) when C < 128 ->
+ scan_ascii(Rest, HasUpper);
+scan_ascii(_Bin, _HasUpper) ->
+ not_ascii.
+
+-spec lower_ascii(binary(), binary()) -> binary().
+lower_ascii(<<>>, Acc) ->
+ Acc;
+lower_ascii(<<C, Rest/binary>>, Acc) when C >= $A, C =< $Z ->
+ lower_ascii(Rest, <<Acc/binary, (C + 32)>>);
+lower_ascii(<<C, Rest/binary>>, Acc) ->
+ lower_ascii(Rest, <<Acc/binary, C>>).
+
-spec deep_merge_member(map(), map()) -> map().
deep_merge_member(CurrentMember, MemberUpdate) ->
User0 = maps:get(<<"user">>, CurrentMember, #{}),
@@ -153,3 +187,43 @@ default_presence() -> guild_member_list_connected:default_presence().
-spec connected_session_user_ids(map()) -> sets:set(integer()).
connected_session_user_ids(S) -> guild_member_list_connected:connected_session_user_ids(S).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+casefold_reference(Bin) ->
+ {ok, Folded} = unicode_chardata_to_binary(string:casefold(Bin)),
+ Folded.
+
+ascii_casefold_matches_unicode_casefold_test() ->
+ lists:foreach(
+ fun(Bin) ->
+ ?assertEqual({Bin, casefold_reference(Bin)}, {Bin, casefold_binary(Bin)})
+ end,
+ [
+ <<>>,
+ <<"a">>,
+ <<"A">>,
+ <<"Zed">>,
+ <<"already lower">>,
+ <<"MiXeD CaSe 123">>,
+ <<"punct!@#$%^&*()_+-=[]{}">>,
+ <<"0123456789">>,
+ <<"~", 127>>
+ ]
+ ).
+
+ascii_casefold_defers_to_unicode_for_non_ascii_test() ->
+ ?assertEqual(not_ascii, ascii_casefold(<<"Ärger"/utf8>>)),
+ lists:foreach(
+ fun(Bin) ->
+ ?assertEqual({Bin, casefold_reference(Bin)}, {Bin, casefold_binary(Bin)})
+ end,
+ [<<"Ärger"/utf8>>, <<"ÅSTRÖM"/utf8>>, <<"日本語"/utf8>>, <<"ß"/utf8>>]
+ ).
+
+ascii_casefold_returns_input_untouched_when_already_folded_test() ->
+ Bin = <<"no uppercase here">>,
+ ?assertEqual({ok, Bin}, ascii_casefold(Bin)).
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl b/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl
index 73a4f6ce6..91503d829 100644
--- a/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl
+++ b/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl
@@ -25,7 +25,7 @@ members_cursor(Limit, AfterId, State) ->
Data = maps:get(data, State, #{}),
MemberMap = guild_data_index:member_map(Data),
Total = map_size(MemberMap),
- SortedIds = lists:sort(maps:keys(MemberMap)),
+ SortedIds = guild_data_index_members:sorted_member_ids(Data, MemberMap),
FilteredIds = filter_ids_after(SortedIds, AfterId),
ResponseMembers = take_members(FilteredIds, Limit, MemberMap),
{reply, #{members => ResponseMembers, total => Total}, State}.
@@ -33,8 +33,61 @@ members_cursor(Limit, AfterId, State) ->
-spec take_members([integer()], integer(), map()) -> [map()].
take_members(_Ids, Limit, _MemberMap) when Limit =< 0 -> [];
take_members(Ids, Limit, MemberMap) ->
- [maps:get(Id, MemberMap) || Id <- lists:sublist(Ids, Limit)].
+ [Member || Id <- lists:sublist(Ids, Limit), {ok, Member} <- [maps:find(Id, MemberMap)]].
-spec filter_ids_after([integer()], integer() | undefined) -> [integer()].
filter_ids_after(Ids, undefined) -> Ids;
filter_ids_after(Ids, AfterId) -> lists:dropwhile(fun(Id) -> Id =< AfterId end, Ids).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+get_members_cursor_matches_full_sort_test() ->
+ lists:foreach(fun assert_cursor_page_matches/1, cursor_cases()).
+
+assert_cursor_page_matches({State, Limit, After}) ->
+ {reply, Reply, _} = get_members_cursor(cursor_request(Limit, After), State),
+ ?assertEqual(legacy_members_cursor(Limit, After, State), Reply).
+
+legacy_members_cursor(Limit, After, State) ->
+ MemberMap = guild_data_index:member_map(maps:get(data, State, #{})),
+ AfterId = snowflake_id:parse_optional(After),
+ Ids = filter_ids_after(lists:sort(maps:keys(MemberMap)), AfterId),
+ #{members => legacy_take_members(Ids, Limit, MemberMap), total => map_size(MemberMap)}.
+
+legacy_take_members(_Ids, Limit, _MemberMap) when Limit =< 0 -> [];
+legacy_take_members(Ids, Limit, MemberMap) ->
+ [maps:get(Id, MemberMap) || Id <- lists:sublist(Ids, Limit)].
+
+cursor_request(Limit, undefined) -> #{<<"limit">> => Limit};
+cursor_request(Limit, After) -> #{<<"limit">> => Limit, <<"after">> => After}.
+
+cursor_cases() ->
+ [
+ {State, Limit, After}
+ || State <- cursor_states(),
+ Limit <- [0, 1, 2, 99],
+ After <- [undefined, 1, 5, 12, 39, 40, 41, 99]
+ ].
+
+cursor_states() ->
+ Members = cursor_members(cursor_user_ids(40)),
+ Cached = guild_data_index:normalize_data(#{<<"members">> => Members}),
+ [
+ #{data => Cached},
+ #{data => guild_data_index:put_member(cursor_member(500), Cached)},
+ #{data => #{<<"members">> => Members}},
+ #{data => guild_data_index:normalize_data(#{<<"members">> => []})},
+ #{}
+ ].
+
+cursor_members(UserIds) ->
+ [cursor_member(UserId) || UserId <- UserIds].
+
+cursor_user_ids(Count) ->
+ lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)).
+
+cursor_member(UserId) ->
+ #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}.
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_member_list_write.erl b/fluxer_gateway/src/guild/guild_member_list_write.erl
index 1dfdc0a46..867e786bf 100644
--- a/fluxer_gateway/src/guild/guild_member_list_write.erl
+++ b/fluxer_gateway/src/guild/guild_member_list_write.erl
@@ -20,6 +20,10 @@
-type list_id() :: binary().
-type user_id() :: integer().
-type channel_id() :: integer().
+-type engine_ref() :: ets:table().
+-type absence() :: {absent, engine_ref()} | present.
+
+-define(MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT, 1000000000).
-export_type([guild_state/0, list_id/0, user_id/0, channel_id/0]).
@@ -180,14 +184,14 @@ broadcast_channel_engine_connection_change(UserId, State, SubsTab) ->
guild_state().
fold_connection_change_lists(GuildId, UserId, State, SubsTab) ->
Sessions = maps:get(sessions, State, #{}),
- guild_member_list_subs:fold_list_ids(
+ lists:foldl(
fun(ListId, AccState) ->
- sync_connection_change_for_list(
- GuildId, UserId, ListId, SubsTab, Sessions, AccState
+ sync_connection_change_for_subscribed_list(
+ GuildId, UserId, ListId, Sessions, AccState
)
end,
State,
- SubsTab
+ guild_member_list_subs:list_ids(SubsTab)
).
-spec dispatch_user_change_to_subscribed_lists(
@@ -200,29 +204,38 @@ fold_connection_change_lists(GuildId, UserId, State, SubsTab) ->
dispatch_user_change_to_subscribed_lists(
UserId, OldMember, NewMember, SubsTab, State
) ->
- guild_member_list_subs:fold_list_ids(
+ lists:foldl(
fun(ListId, AccState) ->
- dispatch_user_change_to_list(
- UserId, OldMember, NewMember, ListId, SubsTab, AccState
+ dispatch_user_change_to_subscribed_list(
+ UserId, OldMember, NewMember, ListId, AccState
)
end,
State,
- SubsTab
+ guild_member_list_subs:list_ids(SubsTab)
).
--spec dispatch_user_change_to_list(
+-spec dispatch_user_change_to_subscribed_list(
user_id(),
map() | undefined,
map() | undefined,
list_id(),
- ets:table(),
guild_state()
) -> guild_state().
-dispatch_user_change_to_list(
- UserId, OldMember, NewMember, ListId, SubsTab, State
-) ->
+dispatch_user_change_to_subscribed_list(UserId, OldMember, NewMember, ListId, State) ->
+ Absence = user_change_absence(UserId, ListId, OldMember, NewMember, State),
State1 = apply_user_change_to_channel_store(UserId, ListId, OldMember, NewMember, State),
- queue_list_sync_if_subscribed(ListId, SubsTab, State1).
+ case sync_body_unchanged(UserId, ListId, Absence, State1) of
+ true -> record_member_list_sync_skipped_absent(State1);
+ false -> guild_member_list_sync_batch:queue_list_sync(ListId, State1)
+ end.
+
+-spec user_change_absence(
+ user_id(), list_id(), map() | undefined, map() | undefined, guild_state()
+) -> absence().
+user_change_absence(UserId, ListId, Member, Member, State) when is_map(Member) ->
+ list_member_absence(UserId, ListId, State);
+user_change_absence(_UserId, _ListId, _OldMember, _NewMember, _State) ->
+ present.
-spec apply_user_change_to_channel_store(
user_id(), list_id(), map() | undefined, map() | undefined, guild_state()
@@ -249,18 +262,13 @@ apply_channel_member_change(UserId, ListId, OldMember, NewMember, State) when
apply_channel_member_change(_UserId, _ListId, _OldMember, _NewMember, _State) ->
ok.
--spec sync_connection_change_for_list(
- integer(),
- user_id(),
- list_id(),
- ets:table(),
- map(),
- guild_state()
+-spec sync_connection_change_for_subscribed_list(
+ integer(), user_id(), list_id(), map(), guild_state()
) -> guild_state().
-sync_connection_change_for_list(_GuildId, _UserId, ListId, SubsTab, _Sessions, State) ->
+sync_connection_change_for_subscribed_list(_GuildId, UserId, ListId, _Sessions, State) ->
case guild_member_list_channel_engine:is_engine_list(ListId, State) of
true ->
- queue_list_sync_if_subscribed(ListId, SubsTab, State);
+ queue_connection_list_sync_unless_absent(UserId, ListId, State);
false ->
State
end.
@@ -280,12 +288,10 @@ broadcast_list_by_id(_GuildId, _ChannelId, ListId, State) ->
-spec rebuild_subscribed_channel_lists(guild_state(), ets:table()) -> guild_state().
rebuild_subscribed_channel_lists(State, SubsTab) ->
- guild_member_list_subs:fold_lists(
- fun(ListId, _Subs, AccState) ->
- rebuild_channel_store(ListId, AccState)
- end,
+ lists:foldl(
+ fun rebuild_channel_store/2,
State,
- SubsTab
+ guild_member_list_subs:list_ids(SubsTab)
).
-spec rebuild_channel_store(list_id(), guild_state()) -> guild_state().
@@ -295,13 +301,104 @@ rebuild_channel_store(ListId, State) ->
false -> State
end.
--spec queue_list_sync_if_subscribed(list_id(), ets:table(), guild_state()) -> guild_state().
-queue_list_sync_if_subscribed(ListId, SubsTab, State) ->
- case guild_member_list_subs:has_subs(ListId, SubsTab) of
- false -> State;
- true -> guild_member_list_sync_batch:queue_list_sync(ListId, State)
+-spec queue_connection_list_sync_unless_absent(user_id(), list_id(), guild_state()) ->
+ guild_state().
+queue_connection_list_sync_unless_absent(UserId, ListId, State) ->
+ case list_member_absence(UserId, ListId, State) of
+ {absent, _Ref} -> record_member_list_sync_skipped_absent(State);
+ present -> queue_connection_list_sync(ListId, State)
end.
+-spec queue_connection_list_sync(list_id(), guild_state()) -> guild_state().
+queue_connection_list_sync(ListId, State) ->
+ case maps:get(pending_member_list_sync_batch, State, undefined) of
+ #{pending_list_ids := PendingListIds} = Batch when is_map(PendingListIds) ->
+ State#{
+ pending_member_list_sync_batch => Batch#{
+ pending_list_ids => PendingListIds#{ListId => true}
+ }
+ };
+ _ ->
+ TimerRef = erlang:send_after(
+ connection_sync_delay_ms(State), self(), flush_member_list_sync_batch
+ ),
+ State#{
+ pending_member_list_sync_batch => #{
+ pending_list_ids => #{ListId => true},
+ timer_ref => TimerRef
+ }
+ }
+ end.
+
+-spec connection_sync_delay_ms(guild_state()) -> pos_integer().
+connection_sync_delay_ms(State) ->
+ case State of
+ #{member_count := MemberCount} when is_integer(MemberCount), MemberCount >= 5000 ->
+ positive_env(member_list_large_connection_sync_delay_ms, 1000);
+ _ ->
+ positive_env(member_list_connection_sync_delay_ms, 250)
+ end.
+
+-spec positive_env(atom(), pos_integer()) -> pos_integer().
+positive_env(Key, Default) ->
+ case application:get_env(fluxer_gateway, Key, Default) of
+ Value when is_integer(Value), Value >= 1 -> Value;
+ _ -> Default
+ end.
+
+-spec record_member_list_sync_skipped_absent(guild_state()) -> guild_state().
+record_member_list_sync_skipped_absent(State) ->
+ case maps:get(member_list_sync_skipped_absent, State, 0) of
+ Count when
+ is_integer(Count), Count >= 0, Count < ?MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT
+ ->
+ State#{member_list_sync_skipped_absent => Count + 1};
+ ?MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT ->
+ State;
+ Invalid ->
+ erlang:error({invalid_member_list_sync_skipped_absent, Invalid})
+ end.
+
+-spec sync_body_unchanged(user_id(), list_id(), absence(), guild_state()) -> boolean().
+sync_body_unchanged(_UserId, _ListId, present, _State) ->
+ false;
+sync_body_unchanged(UserId, ListId, {absent, Ref}, State) ->
+ list_store_ref(ListId, State) =:= Ref andalso member_row_absent(UserId, Ref).
+
+-spec list_member_absence(user_id(), list_id(), guild_state()) -> absence().
+list_member_absence(UserId, ListId, State) ->
+ absence_for_store(UserId, list_store_ref(ListId, State)).
+
+-spec absence_for_store(user_id(), engine_ref() | undefined) -> absence().
+absence_for_store(_UserId, undefined) ->
+ present;
+absence_for_store(UserId, Ref) ->
+ case member_row_absent(UserId, Ref) of
+ true -> {absent, Ref};
+ false -> present
+ end.
+
+-spec member_row_absent(user_id(), engine_ref()) -> boolean().
+member_row_absent(UserId, Ref) when is_integer(UserId), UserId > 0 ->
+ try guild_member_list_engine:is_member_online(Ref, UserId) of
+ not_present -> true;
+ _Online -> false
+ catch
+ _:_ -> false
+ end;
+member_row_absent(_UserId, _Ref) ->
+ false.
+
+-spec list_store_ref(list_id(), guild_state()) -> engine_ref() | undefined.
+list_store_ref(<<"0">>, State) ->
+ case maps:get(member_list_engine, State, undefined) of
+ undefined -> undefined;
+ Ref when is_reference(Ref); is_atom(Ref) -> Ref;
+ _Other -> undefined
+ end;
+list_store_ref(ListId, State) ->
+ guild_member_list_channel_engine:ref(ListId, State).
+
-spec flush_pending_member_list_syncs(guild_state()) -> guild_state().
flush_pending_member_list_syncs(State) ->
guild_member_list_sync_batch:flush_pending_syncs(State).
@@ -316,12 +413,247 @@ apply_user_change_to_channel_store_updates_engine_test() ->
State1 = apply_user_change_to_channel_store(2, <<"123">>, undefined, Member, State),
?assert(maps:is_key(<<"123">>, maps:get(channel_member_list_engines, State1))).
-sync_connection_change_ignores_zero_list_test() ->
- SubsTab = guild_member_list_subs:new(),
- ?assertEqual(
- #{},
- sync_connection_change_for_list(1, 2, <<"0">>, SubsTab, #{}, #{})
- ),
- guild_member_list_subs:destroy(SubsTab).
+absence_is_absent_for_unloaded_member_test() ->
+ Ref = guild_member_list_engine:new(),
+ State = engine_state(Ref),
+ try
+ ?assertEqual({absent, Ref}, list_member_absence(7, <<"500">>, State))
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+absence_is_present_for_loaded_member_test() ->
+ Ref = guild_member_list_engine:new(),
+ State = engine_state(Ref),
+ try
+ ok = guild_member_list_engine:bulk_load(Ref, [{7, <<"seven">>, [], true}], []),
+ ?assertEqual(present, list_member_absence(7, <<"500">>, State)),
+ ?assertEqual({absent, Ref}, list_member_absence(8, <<"500">>, State))
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+absence_is_present_without_store_test() ->
+ ?assertEqual(present, list_member_absence(7, <<"500">>, #{})),
+ ?assertEqual(present, list_member_absence(7, <<"0">>, #{})),
+ ?assertEqual(present, list_member_absence(0, <<"500">>, #{})).
+
+default_list_absence_reads_default_engine_test() ->
+ Ref = guild_member_list_engine:new(),
+ State = #{member_list_engine => Ref},
+ try
+ ?assertEqual({absent, Ref}, list_member_absence(7, <<"0">>, State)),
+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
+ ?assertEqual(present, list_member_absence(7, <<"0">>, State))
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+sync_body_unchanged_requires_stable_ref_test() ->
+ Ref = guild_member_list_engine:new(),
+ Other = guild_member_list_engine:new(),
+ State = engine_state(Ref),
+ try
+ ?assert(sync_body_unchanged(7, <<"500">>, {absent, Ref}, State)),
+ ?assertNot(sync_body_unchanged(7, <<"500">>, present, State)),
+ ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Other}, State)),
+ ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Ref}, #{}))
+ after
+ guild_member_list_engine:destroy(Ref),
+ guild_member_list_engine:destroy(Other)
+ end.
+
+sync_body_unchanged_false_when_member_reappears_test() ->
+ Ref = guild_member_list_engine:new(),
+ State = engine_state(Ref),
+ try
+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
+ ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Ref}, State))
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+user_change_absence_forces_sync_on_member_change_test() ->
+ Ref = guild_member_list_engine:new(),
+ State = engine_state(Ref),
+ Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []},
+ try
+ ?assertEqual({absent, Ref}, user_change_absence(7, <<"500">>, Member, Member, State)),
+ ?assertEqual(present, user_change_absence(7, <<"500">>, Member, undefined, State)),
+ ?assertEqual(present, user_change_absence(7, <<"500">>, undefined, Member, State)),
+ ?assertEqual(
+ present,
+ user_change_absence(7, <<"500">>, Member, Member#{<<"nick">> => <<"x">>}, State)
+ )
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+subscribed_list_sync_is_skipped_only_for_absent_member_test() ->
+ Ref = guild_member_list_engine:new(),
+ Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []},
+ State = (engine_state(Ref))#{pending_member_list_sync_batch => #{pending_list_ids => #{}}},
+ try
+ with_batch_window(false, fun() ->
+ ?assert(
+ maps:is_key(
+ pending_member_list_sync_batch,
+ dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)
+ )
+ ),
+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
+ ?assertNot(
+ maps:is_key(
+ pending_member_list_sync_batch,
+ dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)
+ )
+ )
+ end)
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+list_id_fold_matches_fold_lists_test() ->
+ Tab = guild_member_list_subs:new(),
+ try
+ guild_member_list_subs:subscribe(<<"s1">>, <<"500">>, [{0, 99}], Tab),
+ guild_member_list_subs:subscribe(<<"s2">>, <<"500">>, [{0, 9}], Tab),
+ guild_member_list_subs:subscribe(<<"s1">>, <<"600">>, [{0, 99}], Tab),
+ Sizes = guild_member_list_subs:fold_lists(
+ fun(ListId, ListSubs, Acc) -> Acc#{ListId => map_size(ListSubs)} end,
+ #{},
+ Tab
+ ),
+ ?assertEqual(lists:sort(maps:keys(Sizes)), guild_member_list_subs:list_ids(Tab)),
+ ?assertEqual([], [ListId || {ListId, 0} <- maps:to_list(Sizes)])
+ after
+ guild_member_list_subs:destroy(Tab)
+ end.
+
+connection_sync_delay_defaults_test() ->
+ ?assertEqual(250, connection_sync_delay_ms(#{})),
+ ?assertEqual(250, connection_sync_delay_ms(#{member_count => 4999})),
+ ?assertEqual(250, connection_sync_delay_ms(#{member_count => undefined})),
+ ?assertEqual(1000, connection_sync_delay_ms(#{member_count => 5000})),
+ ?assertEqual(1000, connection_sync_delay_ms(#{member_count => 60000})).
+
+connection_sync_delay_reads_env_test() ->
+ with_env(member_list_connection_sync_delay_ms, 40, fun() ->
+ ?assertEqual(40, connection_sync_delay_ms(#{member_count => 10}))
+ end),
+ with_env(member_list_large_connection_sync_delay_ms, 90, fun() ->
+ ?assertEqual(90, connection_sync_delay_ms(#{member_count => 5000}))
+ end).
+
+connection_sync_delay_rejects_invalid_env_test() ->
+ lists:foreach(
+ fun(Invalid) ->
+ with_env(member_list_connection_sync_delay_ms, Invalid, fun() ->
+ ?assertEqual(250, connection_sync_delay_ms(#{}))
+ end)
+ end,
+ [0, -1, undefined, 250.0, <<"250">>]
+ ).
+
+connection_list_sync_arms_one_timer_per_window_test() ->
+ State1 = queue_connection_list_sync(<<"500">>, #{}),
+ Batch1 = maps:get(pending_member_list_sync_batch, State1),
+ TimerRef = maps:get(timer_ref, Batch1),
+ try
+ ?assert(is_reference(TimerRef)),
+ ?assertEqual(#{<<"500">> => true}, maps:get(pending_list_ids, Batch1)),
+ State2 = queue_connection_list_sync(<<"600">>, State1),
+ Batch2 = maps:get(pending_member_list_sync_batch, State2),
+ ?assertEqual(TimerRef, maps:get(timer_ref, Batch2)),
+ ?assertEqual(
+ #{<<"500">> => true, <<"600">> => true}, maps:get(pending_list_ids, Batch2)
+ ),
+ State3 = queue_connection_list_sync(<<"500">>, State2),
+ ?assertEqual(Batch2, maps:get(pending_member_list_sync_batch, State3))
+ after
+ _ = erlang:cancel_timer(TimerRef)
+ end.
+
+connection_change_sync_is_debounced_test() ->
+ Ref = guild_member_list_engine:new(),
+ try
+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
+ Next = sync_connection_change_for_subscribed_list(
+ 1, 7, <<"500">>, #{}, engine_state(Ref)
+ ),
+ Batch = maps:get(pending_member_list_sync_batch, Next),
+ ?assertEqual(#{<<"500">> => true}, maps:get(pending_list_ids, Batch)),
+ _ = erlang:cancel_timer(maps:get(timer_ref, Batch))
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+connection_change_skip_absent_arms_no_timer_test() ->
+ Ref = guild_member_list_engine:new(),
+ try
+ Next = sync_connection_change_for_subscribed_list(
+ 1, 7, <<"500">>, #{}, engine_state(Ref)
+ ),
+ ?assertNot(maps:is_key(pending_member_list_sync_batch, Next)),
+ ?assertEqual(1, maps:get(member_list_sync_skipped_absent, Next))
+ after
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+connection_change_skip_absent_keeps_pending_batch_test() ->
+ Ref = guild_member_list_engine:new(),
+ TimerRef = erlang:send_after(60000, self(), flush_member_list_sync_batch),
+ Batch = #{pending_list_ids => #{<<"600">> => true}, timer_ref => TimerRef},
+ State = (engine_state(Ref))#{pending_member_list_sync_batch => Batch},
+ try
+ Next = sync_connection_change_for_subscribed_list(1, 7, <<"500">>, #{}, State),
+ ?assertEqual(Batch, maps:get(pending_member_list_sync_batch, Next))
+ after
+ _ = erlang:cancel_timer(TimerRef),
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+member_update_sync_stays_immediate_test() ->
+ Ref = guild_member_list_engine:new(),
+ TimerRef = erlang:send_after(60000, self(), flush_member_list_sync_batch),
+ Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []},
+ State = (engine_state(Ref))#{
+ pending_member_list_sync_batch => #{
+ pending_list_ids => #{<<"600">> => true},
+ timer_ref => TimerRef
+ }
+ },
+ try
+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),
+ ?assertNot(
+ maps:is_key(
+ pending_member_list_sync_batch,
+ dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)
+ )
+ )
+ after
+ _ = erlang:cancel_timer(TimerRef),
+ guild_member_list_engine:destroy(Ref)
+ end.
+
+engine_state(Ref) ->
+ #{channel_member_list_engines => #{<<"500">> => Ref}, member_presence => #{}}.
+
+with_batch_window(Value, Fun) ->
+ with_env(guild_member_list_sync_batch_window, Value, Fun).
+
+with_env(Key, Value, Fun) ->
+ Previous = application:get_env(fluxer_gateway, Key),
+ application:set_env(fluxer_gateway, Key, Value),
+ try
+ Fun()
+ after
+ restore_env(Key, Previous)
+ end.
+
+restore_env(Key, {ok, Previous}) ->
+ application:set_env(fluxer_gateway, Key, Previous);
+restore_env(Key, undefined) ->
+ application:unset_env(fluxer_gateway, Key).
-endif.
diff --git a/fluxer_gateway/src/guild/guild_members_table_repair.erl b/fluxer_gateway/src/guild/guild_members_table_repair.erl
new file mode 100644
index 000000000..daa508360
--- /dev/null
+++ b/fluxer_gateway/src/guild/guild_members_table_repair.erl
@@ -0,0 +1,542 @@
+%% SPDX-License-Identifier: AGPL-3.0-or-later
+
+-module(guild_members_table_repair).
+-typing([eqwalizer]).
+
+-export([
+ maybe_repair/1,
+ members_table_healthy/1
+]).
+
+-define(REPAIR_KEY, members_ets_repair).
+-define(ATTEMPTS_KEY, members_ets_repair_attempts).
+-define(MAX_MEMBER_ETS_REPAIR_CHUNK, 5000).
+-define(MAX_MEMBER_ETS_REPAIR_MEMBERS, 250000).
+-define(MAX_MEMBER_ETS_REPAIR_TICKS, 200).
+-define(MAX_MEMBER_ETS_REPAIR_ATTEMPTS, 3).
+-define(MAX_MEMBER_ETS_REPAIR_MAILBOX, 1000).
+
+-type guild_state() :: map().
+-type user_id() :: integer().
+-type member() :: map().
+-type member_map() :: #{user_id() => member()}.
+-type pass_result() :: continue | complete | {error, table_unavailable}.
+
+-export_type([guild_state/0]).
+
+-spec maybe_repair(guild_state()) -> guild_state().
+maybe_repair(State) ->
+ case self_heal_enabled() of
+ true -> repair_enabled(State);
+ false -> release_pending_repair(State)
+ end.
+
+-spec repair_enabled(guild_state()) -> guild_state().
+repair_enabled(State) ->
+ case members_table_healthy(members_table(State)) of
+ true -> release_pending_repair(State);
+ false -> repair_unhealthy(State)
+ end.
+
+-spec self_heal_enabled() -> boolean().
+self_heal_enabled() ->
+ application:get_env(fluxer_gateway, guild_members_table_self_heal, true) =:= true.
+
+-spec members_table_healthy(term()) -> boolean().
+members_table_healthy(Tab) when is_reference(Tab) ->
+ table_owner(Tab) =:= self();
+members_table_healthy(_Tab) ->
+ false.
+
+-spec repair_unhealthy(guild_state()) -> guild_state().
+repair_unhealthy(State) ->
+ case mailbox_within_bound(message_queue_len()) of
+ true -> repair_pass(State);
+ false -> State
+ end.
+
+-spec repair_pass(guild_state()) -> guild_state().
+repair_pass(State) ->
+ case maps:get(?REPAIR_KEY, State, undefined) of
+ #{tab := Tab, ticks := Ticks} when is_reference(Tab), is_integer(Ticks) ->
+ continue_repair(Tab, Ticks, State);
+ _ ->
+ start_repair(maps:remove(?REPAIR_KEY, State))
+ end.
+
+-spec start_repair(guild_state()) -> guild_state().
+start_repair(State) ->
+ case attempts(State) < ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS of
+ true -> start_repair_within_bounds(State);
+ false -> State
+ end.
+
+-spec start_repair_within_bounds(guild_state()) -> guild_state().
+start_repair_within_bounds(State) ->
+ Size = map_size(member_map(State)),
+ case Size =< ?MAX_MEMBER_ETS_REPAIR_MEMBERS of
+ true -> create_staging_table(State);
+ false -> refuse_oversized(Size, State)
+ end.
+
+-spec refuse_oversized(non_neg_integer(), guild_state()) -> guild_state().
+refuse_oversized(Size, State) ->
+ logger:warning(
+ "guild_members_table_repair_refused: guild_id=~p members=~p max=~p",
+ [guild_id(State), Size, ?MAX_MEMBER_ETS_REPAIR_MEMBERS]
+ ),
+ State#{?ATTEMPTS_KEY => ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS}.
+
+-spec create_staging_table(guild_state()) -> guild_state().
+create_staging_table(State) ->
+ case new_staging_table() of
+ Tab when is_reference(Tab) -> continue_repair(Tab, 0, State);
+ undefined -> record_start_failure(State)
+ end.
+
+-spec continue_repair(ets:tid(), non_neg_integer(), guild_state()) -> guild_state().
+continue_repair(Tab, Ticks, State) when Ticks >= ?MAX_MEMBER_ETS_REPAIR_TICKS ->
+ abandon_repair(Tab, ticks_exhausted, State);
+continue_repair(Tab, Ticks, State) ->
+ case run_pass(Tab, member_map(State)) of
+ continue -> State#{?REPAIR_KEY => #{tab => Tab, ticks => Ticks + 1}};
+ complete -> publish_repair(Tab, State);
+ {error, table_unavailable} -> abandon_repair(Tab, table_unavailable, State)
+ end.
+
+-spec publish_repair(ets:tid(), guild_state()) -> guild_state().
+publish_repair(Tab, State) ->
+ case maps:get(data, State, #{}) of
+ Data when is_map(Data) -> publish_into_data(Tab, Data, State);
+ _ -> abandon_repair(Tab, invalid_guild_data, State)
+ end.
+
+-spec publish_into_data(ets:tid(), map(), guild_state()) -> guild_state().
+publish_into_data(Tab, Data, State) ->
+ logger:info(
+ "guild_members_table_repair_completed: guild_id=~p members=~p",
+ [guild_id(State), map_size(member_map(State))]
+ ),
+ Published = State#{data => Data#{members_ets => Tab}},
+ maps:remove(?ATTEMPTS_KEY, maps:remove(?REPAIR_KEY, Published)).
+
+-spec abandon_repair(ets:tid(), atom(), guild_state()) -> guild_state().
+abandon_repair(Tab, Reason, State) ->
+ ok = delete_staging_table(Tab),
+ logger:warning(
+ "guild_members_table_repair_abandoned: guild_id=~p reason=~p attempts=~p",
+ [guild_id(State), Reason, attempts(State) + 1]
+ ),
+ bump_attempts(maps:remove(?REPAIR_KEY, State)).
+
+-spec record_start_failure(guild_state()) -> guild_state().
+record_start_failure(State) ->
+ logger:warning(
+ "guild_members_table_repair_start_failed: guild_id=~p attempts=~p",
+ [guild_id(State), attempts(State) + 1]
+ ),
+ bump_attempts(State).
+
+-spec run_pass(ets:tid(), member_map()) -> pass_result().
+run_pass(Tab, MemberMap) ->
+ run_pass(Tab, MemberMap, ?MAX_MEMBER_ETS_REPAIR_CHUNK).
+
+-spec run_pass(ets:tid(), member_map(), pos_integer()) -> pass_result().
+run_pass(Tab, MemberMap, Budget) ->
+ try
+ apply_pass(Tab, MemberMap, Budget)
+ catch
+ error:badarg -> {error, table_unavailable}
+ end.
+
+-spec apply_pass(ets:tid(), member_map(), pos_integer()) -> continue | complete.
+apply_pass(Tab, MemberMap, Budget) ->
+ case collect_missing_rows(Tab, MemberMap, Budget) of
+ [] -> drop_extra_rows(Tab, MemberMap, Budget);
+ Rows -> insert_missing_rows(Tab, MemberMap, Rows, Budget)
+ end.
+
+-spec insert_missing_rows(ets:tid(), member_map(), [{user_id(), member()}], pos_integer()) ->
+ continue | complete.
+insert_missing_rows(Tab, MemberMap, Rows, Budget) ->
+ true = ets:insert(Tab, Rows),
+ case length(Rows) < Budget of
+ true -> drop_extra_rows(Tab, MemberMap, Budget);
+ false -> continue
+ end.
+
+-spec drop_extra_rows(ets:tid(), member_map(), pos_integer()) -> continue | complete.
+drop_extra_rows(Tab, MemberMap, Budget) ->
+ case collect_extra_ids(Tab, MemberMap, Budget) of
+ [] -> complete;
+ Ids -> delete_extra_ids(Tab, Ids, Budget)
+ end.
+
+-spec delete_extra_ids(ets:tid(), [user_id()], pos_integer()) -> continue | complete.
+delete_extra_ids(Tab, Ids, Budget) ->
+ lists:foreach(fun(UserId) -> ets:delete(Tab, UserId) end, Ids),
+ case length(Ids) < Budget of
+ true -> complete;
+ false -> continue
+ end.
+
+-spec collect_missing_rows(ets:tid(), member_map(), pos_integer()) ->
+ [{user_id(), member()}].
+collect_missing_rows(Tab, MemberMap, Budget) ->
+ collect_missing_iter(maps:next(maps:iterator(MemberMap)), Tab, Budget, []).
+
+-spec collect_missing_iter(term(), ets:tid(), non_neg_integer(), [{user_id(), member()}]) ->
+ [{user_id(), member()}].
+collect_missing_iter(_Next, _Tab, 0, Acc) ->
+ Acc;
+collect_missing_iter(none, _Tab, _Budget, Acc) ->
+ Acc;
+collect_missing_iter({UserId, Member, Iter}, Tab, Budget, Acc) ->
+ case ets:member(Tab, UserId) of
+ true ->
+ collect_missing_iter(maps:next(Iter), Tab, Budget, Acc);
+ false ->
+ collect_missing_iter(maps:next(Iter), Tab, Budget - 1, [{UserId, Member} | Acc])
+ end.
+
+-spec collect_extra_ids(ets:tid(), member_map(), pos_integer()) -> [user_id()].
+collect_extra_ids(Tab, MemberMap, Budget) ->
+ collect_extra_iter(ets:first(Tab), Tab, MemberMap, Budget, []).
+
+-spec collect_extra_iter(term(), ets:tid(), member_map(), non_neg_integer(), [user_id()]) ->
+ [user_id()].
+collect_extra_iter(_Key, _Tab, _MemberMap, 0, Acc) ->
+ Acc;
+collect_extra_iter('$end_of_table', _Tab, _MemberMap, _Budget, Acc) ->
+ Acc;
+collect_extra_iter(Key, Tab, MemberMap, Budget, Acc) ->
+ Next = ets:next(Tab, Key),
+ case maps:is_key(Key, MemberMap) of
+ true ->
+ collect_extra_iter(Next, Tab, MemberMap, Budget, Acc);
+ false ->
+ collect_extra_iter(Next, Tab, MemberMap, Budget - 1, [Key | Acc])
+ end.
+
+-spec release_pending_repair(guild_state()) -> guild_state().
+release_pending_repair(State) ->
+ case maps:get(?REPAIR_KEY, State, undefined) of
+ undefined -> State;
+ #{tab := Tab} when is_reference(Tab) -> discard_staging_table(Tab, State);
+ _ -> maps:remove(?REPAIR_KEY, State)
+ end.
+
+-spec discard_staging_table(ets:tid(), guild_state()) -> guild_state().
+discard_staging_table(Tab, State) ->
+ ok = delete_staging_table(Tab),
+ maps:remove(?REPAIR_KEY, State).
+
+-spec new_staging_table() -> ets:tid() | undefined.
+new_staging_table() ->
+ try ets:new(guild_members_data, [set, public, {read_concurrency, true}]) of
+ Tab when is_reference(Tab) -> Tab;
+ _Named -> undefined
+ catch
+ error:badarg -> undefined;
+ error:system_limit -> undefined
+ end.
+
+-spec delete_staging_table(ets:tid()) -> ok.
+delete_staging_table(Tab) ->
+ try ets:delete(Tab) of
+ _ -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+-spec table_owner(term()) -> pid() | undefined.
+table_owner(Tab) ->
+ try ets:info(eqwalizer:dynamic_cast(Tab), owner) of
+ Owner when is_pid(Owner) -> Owner;
+ _ -> undefined
+ catch
+ error:badarg -> undefined
+ end.
+
+-spec members_table(guild_state()) -> ets:tid() | undefined.
+members_table(State) ->
+ members_ets_table(data_map(State)).
+
+-spec members_ets_table(map()) -> ets:tid() | undefined.
+members_ets_table(#{members_ets := Tab}) ->
+ Tab;
+members_ets_table(_Data) ->
+ undefined.
+
+-spec member_map(guild_state()) -> member_map().
+member_map(State) ->
+ guild_data_index_members:member_map(data_map(State)).
+
+-spec data_map(guild_state()) -> map().
+data_map(State) ->
+ case maps:get(data, State, #{}) of
+ Data when is_map(Data) -> Data;
+ _ -> #{}
+ end.
+
+-spec guild_id(guild_state()) -> term().
+guild_id(State) ->
+ maps:get(id, State, undefined).
+
+-spec attempts(guild_state()) -> non_neg_integer().
+attempts(State) ->
+ case maps:get(?ATTEMPTS_KEY, State, 0) of
+ N when is_integer(N), N >= 0 -> N;
+ _ -> 0
+ end.
+
+-spec bump_attempts(guild_state()) -> guild_state().
+bump_attempts(State) ->
+ State#{?ATTEMPTS_KEY => attempts(State) + 1}.
+
+-spec message_queue_len() -> non_neg_integer().
+message_queue_len() ->
+ case erlang:process_info(self(), message_queue_len) of
+ {message_queue_len, Len} when is_integer(Len), Len >= 0 -> Len;
+ _ -> 0
+ end.
+
+-spec mailbox_within_bound(non_neg_integer()) -> boolean().
+mailbox_within_bound(Len) ->
+ Len < ?MAX_MEMBER_ETS_REPAIR_MAILBOX.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+healthy_table_is_left_untouched_test() ->
+ Tab = ets:new(healthy_members, [set, public]),
+ try
+ State = repair_state_with_table(#{1 => test_member(1)}, Tab),
+ ?assertEqual(State, maybe_repair(State)),
+ ?assertEqual(0, ets:info(Tab, size))
+ after
+ ets:delete(Tab)
+ end.
+
+healthy_table_releases_pending_staging_table_test() ->
+ Healthy = ets:new(healthy_members, [set, public]),
+ try
+ Staging = ets:new(rollback_members, [set, public]),
+ State0 = with_repair_key(
+ repair_state_with_table(#{1 => test_member(1)}, Healthy), Staging, 3
+ ),
+ State1 = maybe_repair(State0),
+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),
+ ?assertEqual(undefined, ets:info(Staging, size))
+ after
+ ets:delete(Healthy)
+ end.
+
+repair_publishes_table_matching_member_map_test() ->
+ ok = drain_mailbox(),
+ with_self_heal_enabled(fun() ->
+ MemberMap = #{1 => test_member(1), 2 => test_member(2)},
+ State1 = maybe_repair(repair_state(MemberMap)),
+ Tab = published_table(State1),
+ ?assert(is_reference(Tab)),
+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),
+ ?assertEqual(lists:sort(maps:to_list(MemberMap)), lists:sort(ets:tab2list(Tab))),
+ ?assertEqual(State1, maybe_repair(State1)),
+ ets:delete(Tab)
+ end).
+
+repair_replaces_dead_table_tid_test() ->
+ ok = drain_mailbox(),
+ with_self_heal_enabled(fun() ->
+ Dead = ets:new(dead_members, [set, public]),
+ true = ets:delete(Dead),
+ State0 = repair_state_with_table(#{1 => test_member(1)}, Dead),
+ State1 = maybe_repair(State0),
+ Tab = published_table(State1),
+ ?assert(is_reference(Tab)),
+ ?assertNotEqual(Dead, Tab),
+ ?assertEqual([{1, test_member(1)}], ets:tab2list(Tab)),
+ ets:delete(Tab)
+ end).
+
+repair_replaces_table_owned_by_another_process_test() ->
+ ok = drain_mailbox(),
+ with_self_heal_enabled(fun() ->
+ {Owner, Foreign} = foreign_members_table(),
+ State1 = maybe_repair(repair_state_with_table(#{1 => test_member(1)}, Foreign)),
+ Tab = published_table(State1),
+ ?assert(is_reference(Tab)),
+ ?assertNotEqual(Foreign, Tab),
+ ?assertEqual(Owner, ets:info(Foreign, owner)),
+ Owner ! stop,
+ ets:delete(Tab)
+ end).
+
+repair_pass_is_bounded_by_chunk_test() ->
+ Tab = ets:new(bounded_members, [set, public]),
+ try
+ MemberMap = maps:from_list([{N, test_member(N)} || N <- lists:seq(1, 5)]),
+ ?assertEqual(continue, run_pass(Tab, MemberMap, 2)),
+ ?assertEqual(2, ets:info(Tab, size)),
+ ?assertEqual(continue, run_pass(Tab, MemberMap, 2)),
+ ?assertEqual(4, ets:info(Tab, size)),
+ ?assertEqual(complete, run_pass(Tab, MemberMap, 2)),
+ ?assertEqual(5, ets:info(Tab, size))
+ after
+ ets:delete(Tab)
+ end.
+
+repair_picks_up_member_added_mid_build_test() ->
+ Tab = ets:new(midbuild_members, [set, public]),
+ try
+ Map1 = #{1 => test_member(1), 2 => test_member(2)},
+ ?assertEqual(continue, run_pass(Tab, Map1, 1)),
+ Map2 = Map1#{3 => test_member(3)},
+ ?assertEqual(continue, run_pass(Tab, Map2, 1)),
+ ?assertEqual(continue, run_pass(Tab, Map2, 1)),
+ ?assertEqual(complete, run_pass(Tab, Map2, 1)),
+ ?assertEqual([1, 2, 3], table_ids(Tab))
+ after
+ ets:delete(Tab)
+ end.
+
+repair_deletes_member_removed_mid_build_test() ->
+ Tab = ets:new(midremove_members, [set, public]),
+ try
+ Map1 = #{1 => test_member(1), 2 => test_member(2)},
+ ?assertEqual(continue, run_pass(Tab, Map1, 2)),
+ ?assertEqual(2, ets:info(Tab, size)),
+ ?assertEqual(complete, run_pass(Tab, #{1 => test_member(1)}, 2)),
+ ?assertEqual([{1, test_member(1)}], ets:tab2list(Tab))
+ after
+ ets:delete(Tab)
+ end.
+
+repair_abandons_after_tick_cap_test() ->
+ ok = drain_mailbox(),
+ with_self_heal_enabled(fun() ->
+ Tab = ets:new(ticks_members, [set, public]),
+ State0 = with_repair_key(
+ repair_state(#{1 => test_member(1)}), Tab, ?MAX_MEMBER_ETS_REPAIR_TICKS
+ ),
+ State1 = maybe_repair(State0),
+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),
+ ?assertEqual(1, maps:get(?ATTEMPTS_KEY, State1)),
+ ?assertEqual(undefined, ets:info(Tab, size)),
+ ?assertEqual(undefined, published_table(State1))
+ end).
+
+repair_abandons_when_staging_table_is_dead_test() ->
+ ok = drain_mailbox(),
+ with_self_heal_enabled(fun() ->
+ Tab = ets:new(dying_members, [set, public]),
+ true = ets:delete(Tab),
+ State0 = with_repair_key(repair_state(#{1 => test_member(1)}), Tab, 1),
+ State1 = maybe_repair(State0),
+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),
+ ?assertEqual(1, maps:get(?ATTEMPTS_KEY, State1)),
+ ?assertEqual(undefined, published_table(State1))
+ end).
+
+repair_stops_after_attempt_cap_test() ->
+ State0 = (repair_state(#{1 => test_member(1)}))#{
+ ?ATTEMPTS_KEY => ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS
+ },
+ ?assertEqual(State0, maybe_repair(State0)).
+
+oversized_member_map_is_refused_and_counted_test() ->
+ State0 = repair_state(#{1 => test_member(1)}),
+ State1 = refuse_oversized(?MAX_MEMBER_ETS_REPAIR_MEMBERS + 1, State0),
+ ?assertEqual(?MAX_MEMBER_ETS_REPAIR_ATTEMPTS, maps:get(?ATTEMPTS_KEY, State1)),
+ ?assertEqual(State1, start_repair(State1)).
+
+mailbox_guard_bound_test() ->
+ ?assert(mailbox_within_bound(0)),
+ ?assert(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX - 1)),
+ ?assertNot(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX)),
+ ?assertNot(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX + 1)).
+
+repair_skips_pass_when_mailbox_is_deep_test() ->
+ with_self_heal_enabled(fun() ->
+ Self = self(),
+ Sent = lists:seq(1, ?MAX_MEMBER_ETS_REPAIR_MAILBOX),
+ lists:foreach(fun(N) -> Self ! {mailbox_guard, N} end, Sent),
+ State = repair_state(#{1 => test_member(1)}),
+ ?assertEqual(State, maybe_repair(State)),
+ drain_mailbox_guard(?MAX_MEMBER_ETS_REPAIR_MAILBOX)
+ end).
+
+members_table_healthy_rejects_non_references_test() ->
+ ?assertNot(members_table_healthy(undefined)),
+ ?assertNot(members_table_healthy(not_a_table)),
+ ?assertNot(members_table_healthy(make_ref())).
+
+%% repair_unhealthy/1 bails out when the CURRENT process mailbox exceeds
+%% ?MAX_MEMBER_ETS_REPAIR_MAILBOX, so a test that leaves the eunit process's
+%% mailbox polluted by an earlier test silently exercises the bail-out branch
+%% instead of the repair it means to assert.
+drain_mailbox() ->
+ receive
+ _Any -> drain_mailbox()
+ after 0 -> ok
+ end.
+
+%% Self-heal defaults to off (production enables it via config), so a test that
+%% exercises the actual repair pass must arm it and always disarm it afterwards -
+%% left on, it would make every other test's update_counts/1 call in this shared
+%% eunit VM attempt a real repair too.
+with_self_heal_enabled(Fun) ->
+ ok = application:set_env(fluxer_gateway, guild_members_table_self_heal, true),
+ try
+ Fun()
+ after
+ application:unset_env(fluxer_gateway, guild_members_table_self_heal)
+ end.
+
+repair_state(MemberMap) ->
+ #{
+ id => 42,
+ data => #{<<"members">> => MemberMap, members_normalized => MemberMap}
+ }.
+
+repair_state_with_table(MemberMap, Tab) ->
+ State = repair_state(MemberMap),
+ Data = maps:get(data, State),
+ State#{data => Data#{members_ets => Tab}}.
+
+with_repair_key(State, Tab, Ticks) ->
+ State#{?REPAIR_KEY => #{tab => Tab, ticks => Ticks}}.
+
+published_table(State) ->
+ maps:get(members_ets, maps:get(data, State, #{}), undefined).
+
+test_member(UserId) ->
+ #{<<"user">> => #{<<"id">> => UserId}}.
+
+table_ids(Tab) ->
+ lists:sort([Id || {Id, _} <- ets:tab2list(Tab)]).
+
+foreign_members_table() ->
+ Self = self(),
+ Owner = spawn(fun() -> foreign_table_owner(Self) end),
+ receive
+ {foreign_members_table, Tab} -> {Owner, Tab}
+ after 5000 -> error(foreign_table_timeout)
+ end.
+
+foreign_table_owner(Parent) ->
+ Parent ! {foreign_members_table, ets:new(foreign_members, [set, public])},
+ receive
+ stop -> ok
+ after 30000 -> ok
+ end.
+
+drain_mailbox_guard(0) ->
+ ok;
+drain_mailbox_guard(N) ->
+ receive
+ {mailbox_guard, _} -> drain_mailbox_guard(N - 1)
+ after 0 -> ok
+ end.
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_mutual_online.erl b/fluxer_gateway/src/guild/guild_mutual_online.erl
index e8855f13f..4c4350755 100644
--- a/fluxer_gateway/src/guild/guild_mutual_online.erl
+++ b/fluxer_gateway/src/guild/guild_mutual_online.erl
@@ -13,6 +13,8 @@
-type user_id() :: integer().
-type guild_state() :: map().
+-type viewable_index() :: #{user_id() => map()}.
+-type index_ctx() :: {map(), viewable_index(), guild_state()}.
-spec compute_count(user_id() | term(), guild_state()) -> non_neg_integer().
compute_count(UserId, State) when is_integer(UserId), UserId > 0 ->
@@ -37,7 +39,7 @@ slow_count(UserId, State) ->
true ->
self_online_count(UserId, State);
false ->
- count_mutually_visible(UserId, ViewerSet, State)
+ count_mutually_visible_indexed(UserId, ViewerSet, State)
end.
-spec self_online_count(user_id(), guild_state()) -> non_neg_integer().
@@ -47,38 +49,120 @@ self_online_count(UserId, State) ->
false -> 0
end.
--spec count_mutually_visible(user_id(), sets:set(), guild_state()) -> non_neg_integer().
-count_mutually_visible(UserId, ViewerSet, State) ->
+-spec count_mutually_visible_indexed(user_id(), sets:set(), guild_state()) ->
+ non_neg_integer().
+count_mutually_visible_indexed(UserId, ViewerSet, State) ->
Tab = maps:get(member_presence, State),
+ Ctx = {viewer_channel_map(ViewerSet), build_viewable_index(State), State},
ets:foldl(
fun({OtherUserId, Presence}, Acc) ->
- count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc)
+ count_online_member_indexed(UserId, Ctx, OtherUserId, Presence, Acc)
end,
0,
Tab
).
--spec count_online_member(
- user_id(), sets:set(), guild_state(), term(), term(), non_neg_integer()
+-spec count_online_member_indexed(
+ user_id(), index_ctx(), term(), term(), non_neg_integer()
) -> non_neg_integer().
-count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) when
+count_online_member_indexed(UserId, Ctx, OtherUserId, Presence, Acc) when
is_integer(OtherUserId), is_map(Presence), OtherUserId > 0
->
case is_online(Presence) of
false -> Acc;
true when OtherUserId =:= UserId -> Acc + 1;
- true -> count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc)
+ true -> count_if_mutually_visible_indexed(OtherUserId, Ctx, Acc)
end;
-count_online_member(_UserId, _ViewerSet, _State, _OtherUserId, _Presence, Acc) ->
+count_online_member_indexed(_UserId, _Ctx, _OtherUserId, _Presence, Acc) ->
Acc.
--spec count_if_mutually_visible(user_id(), sets:set(), guild_state(), non_neg_integer()) ->
+-spec count_if_mutually_visible_indexed(user_id(), index_ctx(), non_neg_integer()) ->
non_neg_integer().
-count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) ->
- OtherSet = guild_visibility:viewable_channel_set(OtherUserId, State),
- case sets:is_empty(sets:intersection(ViewerSet, OtherSet)) of
+count_if_mutually_visible_indexed(OtherUserId, Ctx, Acc) ->
+ case shares_viewable_channel(OtherUserId, Ctx) of
+ true -> Acc + 1;
+ false -> Acc
+ end.
+
+-spec shares_viewable_channel(user_id(), index_ctx()) -> boolean().
+shares_viewable_channel(OtherUserId, {ViewerMap, Index, State}) ->
+ case maps:find(OtherUserId, Index) of
+ {ok, OtherMap} -> maps_share_any_key(OtherMap, ViewerMap);
+ error -> channel_list_shares_any(OtherUserId, ViewerMap, State)
+ end.
+
+-spec channel_list_shares_any(user_id(), map(), guild_state()) -> boolean().
+channel_list_shares_any(OtherUserId, ViewerMap, State) ->
+ Channels = guild_visibility:get_user_viewable_channels(OtherUserId, State),
+ lists:any(fun(ChannelId) -> maps:is_key(ChannelId, ViewerMap) end, Channels).
+
+-spec viewer_channel_map(sets:set()) -> map().
+viewer_channel_map(ViewerSet) ->
+ sets:fold(fun(ChannelId, Acc) -> Acc#{ChannelId => true} end, #{}, ViewerSet).
+
+-spec build_viewable_index(guild_state()) -> viewable_index().
+build_viewable_index(State) ->
+ build_index_from_sessions(maps:get(sessions, State, #{})).
+
+-spec build_index_from_sessions(term()) -> viewable_index().
+build_index_from_sessions(Sessions) when is_map(Sessions) ->
+ build_index_iter(maps:iterator(Sessions), #{});
+build_index_from_sessions(_) ->
+ #{}.
+
+-spec build_index_iter(maps:iterator(), viewable_index()) -> viewable_index().
+build_index_iter(Iterator, Acc) ->
+ case maps:next(Iterator) of
+ none ->
+ Acc;
+ {_, SessionData, Next} when is_map(SessionData) ->
+ build_index_iter(Next, index_session(SessionData, Acc));
+ {_, _, Next} ->
+ build_index_iter(Next, Acc)
+ end.
+
+-spec index_session(map(), viewable_index()) -> viewable_index().
+index_session(SessionData, Acc) ->
+ SessionUserId = maps:get(user_id, SessionData, undefined),
+ ViewableChannels = maps:get(viewable_channels, SessionData, undefined),
+ index_session_entry(SessionUserId, ViewableChannels, Acc).
+
+-spec index_session_entry(term(), term(), viewable_index()) -> viewable_index().
+index_session_entry(UserId, ViewableChannels, Acc) when
+ is_integer(UserId), is_map(ViewableChannels)
+->
+ put_first_session_map(UserId, ViewableChannels, Acc);
+index_session_entry(_, _, Acc) ->
+ Acc.
+
+-spec put_first_session_map(user_id(), map(), viewable_index()) -> viewable_index().
+put_first_session_map(UserId, ViewableChannels, Acc) ->
+ case maps:is_key(UserId, Acc) of
true -> Acc;
- false -> Acc + 1
+ false -> Acc#{UserId => ViewableChannels}
+ end.
+
+-spec maps_share_any_key(map(), map()) -> boolean().
+maps_share_any_key(MapA, MapB) ->
+ {Smaller, Larger} =
+ case map_size(MapA) =< map_size(MapB) of
+ true -> {MapA, MapB};
+ false -> {MapB, MapA}
+ end,
+ maps_share_any_key_iter(maps:iterator(Smaller), Larger).
+
+-spec maps_share_any_key_iter(maps:iterator(), map()) -> boolean().
+maps_share_any_key_iter(Iterator, LargerMap) ->
+ case maps:next(Iterator) of
+ none -> false;
+ {Key, _, NextIterator} -> key_matches_or_continue(Key, NextIterator, LargerMap)
+ end.
+
+-spec key_matches_or_continue(term(), maps:iterator(), map()) -> boolean().
+key_matches_or_continue(Key, NextIterator, LargerMap) ->
+ case maps:is_key(Key, LargerMap) of
+ true -> true;
+ false -> maps_share_any_key_iter(NextIterator, LargerMap)
end.
-spec is_self_online(user_id(), guild_state()) -> boolean().
@@ -258,6 +342,69 @@ slow_path_returns_zero_when_viewer_offline_and_no_channels_test() ->
},
?assertEqual(0, compute_count(10, State)).
+index_matches_scan_without_sessions_test() ->
+ State = mutual_visibility_state(1, 5000),
+ ViewerSet = guild_visibility:viewable_channel_set(10, State),
+ Expected = reference_count_mutually_visible(10, ViewerSet, State),
+ ?assertEqual(2, Expected),
+ ?assertEqual(Expected, compute_count(10, State)).
+
+index_matches_scan_with_cached_session_channels_test() ->
+ Base = mutual_visibility_state(1, 5000),
+ State = Base#{sessions => cached_viewable_sessions()},
+ ViewerSet = guild_visibility:viewable_channel_set(10, State),
+ Expected = reference_count_mutually_visible(10, ViewerSet, State),
+ ?assertEqual(3, Expected),
+ ?assertEqual(Expected, compute_count(10, State)).
+
+index_counts_with_cached_session_channels_test() ->
+ Base = mutual_visibility_state(1, 5000),
+ State = Base#{sessions => cached_viewable_sessions()},
+ ?assertEqual(3, compute_count(10, State)).
+
+cached_viewable_sessions() ->
+ #{
+ <<"s20a">> => #{user_id => 20, viewable_channels => undefined},
+ <<"s20b">> => #{user_id => 20, viewable_channels => #{100 => true}},
+ <<"s40">> => #{user_id => 40, viewable_channels => #{101 => true}}
+ }.
+
+-spec reference_count_mutually_visible(user_id(), sets:set(), guild_state()) ->
+ non_neg_integer().
+reference_count_mutually_visible(UserId, ViewerSet, State) ->
+ Tab = maps:get(member_presence, State),
+ ets:foldl(
+ fun({OtherUserId, Presence}, Acc) ->
+ reference_count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc)
+ end,
+ 0,
+ Tab
+ ).
+
+-spec reference_count_online_member(
+ user_id(), sets:set(), guild_state(), term(), term(), non_neg_integer()
+) -> non_neg_integer().
+reference_count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) when
+ is_integer(OtherUserId), is_map(Presence), OtherUserId > 0
+->
+ case is_online(Presence) of
+ false -> Acc;
+ true when OtherUserId =:= UserId -> Acc + 1;
+ true -> reference_count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc)
+ end;
+reference_count_online_member(_UserId, _ViewerSet, _State, _OtherUserId, _Presence, Acc) ->
+ Acc.
+
+-spec reference_count_if_mutually_visible(
+ user_id(), sets:set(), guild_state(), non_neg_integer()
+) -> non_neg_integer().
+reference_count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) ->
+ OtherSet = guild_visibility:viewable_channel_set(OtherUserId, State),
+ case sets:is_empty(sets:intersection(ViewerSet, OtherSet)) of
+ true -> Acc;
+ false -> Acc + 1
+ end.
+
make_presence_tab(Map) ->
Tab = ets:new(test_member_presence, [set, public]),
maps:foreach(fun(K, V) -> ets:insert(Tab, {K, V}) end, Map),
diff --git a/fluxer_gateway/src/guild/guild_passive_sync.erl b/fluxer_gateway/src/guild/guild_passive_sync.erl
index 1de624378..1275b261b 100644
--- a/fluxer_gateway/src/guild/guild_passive_sync.erl
+++ b/fluxer_gateway/src/guild/guild_passive_sync.erl
@@ -15,6 +15,7 @@
-export_type([guild_state/0, channel_id/0, last_message_id/0, version/0, voice_state/0]).
-define(PASSIVE_SYNC_INTERVAL, 30000).
+-define(LARGE_GUILD_MEMBER_COUNT, 250).
-type guild_state() :: map().
-type channel_id() :: binary().
@@ -30,40 +31,53 @@ schedule_passive_sync(State) ->
-spec handle_passive_sync(guild_state()) -> {noreply, guild_state()}.
handle_passive_sync(State) ->
GuildId = maps:get(id, State),
- Sessions = maps:get(sessions, State, #{}),
- Data = maps:get(data, State, #{}),
- MemberCount = maps:get(member_count, State, undefined),
- VoiceStates = maps:get(voice_states, State, #{}),
- _ = spawn(fun() ->
- send_passive_updates(
- GuildId, Sessions, passive_sync_state(State, Data, VoiceStates), MemberCount
- )
- end),
+ ok = maybe_spawn_passive_updates(GuildId, State),
_ = schedule_passive_sync(State),
{noreply, State}.
+%% send_passive_updates/4 keeps no session unless is_large_guild/1 holds, so for any other
+%% guild the spawned child returns without touching a session, a payload or the registry.
+-spec maybe_spawn_passive_updates(integer(), guild_state()) -> ok.
+maybe_spawn_passive_updates(GuildId, State) ->
+ case is_large_guild(maps:get(member_count, State, undefined)) of
+ false -> ok;
+ true -> spawn_passive_updates(GuildId, State)
+ end.
+
+-spec spawn_passive_updates(integer(), guild_state()) -> ok.
+spawn_passive_updates(GuildId, State) ->
+ _ = spawn(fun() -> send_passive_updates_for_state(GuildId, State) end),
+ ok.
+
-spec send_passive_updates_to_sessions(guild_state()) -> guild_state().
send_passive_updates_to_sessions(State) ->
- GuildId = maps:get(id, State),
+ ok = send_passive_updates_for_state(maps:get(id, State), State),
+ State.
+
+-spec send_passive_updates_for_state(integer(), guild_state()) -> ok.
+send_passive_updates_for_state(GuildId, State) ->
Sessions = maps:get(sessions, State, #{}),
Data = maps:get(data, State, #{}),
MemberCount = maps:get(member_count, State, undefined),
VoiceStates = maps:get(voice_states, State, #{}),
send_passive_updates(
GuildId, Sessions, passive_sync_state(State, Data, VoiceStates), MemberCount
- ),
- State.
+ ).
-spec passive_sync_state(guild_state(), map(), map()) -> guild_state().
passive_sync_state(State, Data, VoiceStates) ->
State#{data => Data, voice_states => VoiceStates}.
+-spec is_large_guild(term()) -> boolean().
+is_large_guild(MemberCount) ->
+ is_integer(MemberCount) andalso MemberCount > ?LARGE_GUILD_MEMBER_COUNT.
+
-spec send_passive_updates(integer(), map(), guild_state(), non_neg_integer() | undefined) ->
ok.
send_passive_updates(GuildId, Sessions, State, MemberCount) ->
Data = maps:get(data, State, #{}),
Channels = guild_data_index:channel_list(Data),
- IsLargeGuild = is_integer(MemberCount) andalso MemberCount > 250,
+ IsLargeGuild = is_large_guild(MemberCount),
PassiveSessions = maps:filter(
fun(_SessionId, SessionData) ->
IsLargeGuild andalso session_passive:is_passive(GuildId, SessionData)
@@ -327,3 +341,129 @@ snowflake_binary(FieldName, Value) ->
{ok, Id} -> integer_to_binary(Id);
{error, _, _} -> undefined
end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+is_large_guild_matches_previous_inline_threshold_test() ->
+ lists:foreach(
+ fun(MemberCount) ->
+ Expected = is_integer(MemberCount) andalso MemberCount > 250,
+ ?assertEqual(Expected, is_large_guild(MemberCount))
+ end,
+ [undefined, 0, 1, 249, 250, 251, 1000000]
+ ).
+
+skipped_spawn_drops_no_session_the_old_filter_kept_test() ->
+ GuildId = 4242,
+ lists:foreach(
+ fun({MemberCount, Sessions}) ->
+ Kept = reference_passive_sessions(GuildId, Sessions, MemberCount),
+ ?assert(is_large_guild(MemberCount) orelse map_size(Kept) =:= 0)
+ end,
+ passive_gate_cases()
+ ),
+ Large = reference_passive_sessions(GuildId, passive_test_sessions(), 251),
+ ?assertEqual(1, map_size(Large)).
+
+reference_passive_sessions(GuildId, Sessions, MemberCount) ->
+ IsLargeGuild = is_integer(MemberCount) andalso MemberCount > 250,
+ maps:filter(
+ fun(_SessionId, SessionData) ->
+ IsLargeGuild andalso session_passive:is_passive(GuildId, SessionData)
+ end,
+ Sessions
+ ).
+
+passive_gate_cases() ->
+ Sessions = passive_test_sessions(),
+ [
+ {undefined, #{}},
+ {undefined, Sessions},
+ {0, Sessions},
+ {250, Sessions},
+ {251, Sessions},
+ {251, #{}},
+ {not_a_count, Sessions}
+ ].
+
+passive_test_sessions() ->
+ passive_test_sessions(<<"passive-session">>).
+
+passive_test_sessions(SessionId) ->
+ #{
+ SessionId => #{
+ session_id => SessionId,
+ user_id => 1130650140672000000,
+ pid => self(),
+ active_guilds => sets:new(),
+ bot => false
+ }
+ }.
+
+handle_passive_sync_stays_silent_below_threshold_test() ->
+ flush_passive_dispatches(),
+ ok = passive_sync_registry:init(),
+ GuildId = 1427764661718740995,
+ SessionId = <<"small-guild-session">>,
+ State = passive_sync_test_state(GuildId, SessionId, 250),
+ ?assertMatch({noreply, State}, handle_passive_sync(State)),
+ ?assertEqual(no_dispatch, receive_passive_dispatch(200)),
+ RegState = passive_sync_registry:lookup(SessionId, GuildId),
+ ?assertEqual(#{}, maps:get(previous_passive_updates, RegState)).
+
+handle_passive_sync_still_dispatches_above_threshold_test() ->
+ flush_passive_dispatches(),
+ ok = passive_sync_registry:init(),
+ GuildId = 1427764661718740996,
+ SessionId = <<"large-guild-session">>,
+ ChannelId = passive_test_channel_id(),
+ MessageId = passive_test_message_id(),
+ State = passive_sync_test_state(GuildId, SessionId, 251),
+ ?assertMatch({noreply, State}, handle_passive_sync(State)),
+ Payload = receive_passive_dispatch(5000),
+ ?assertEqual(
+ #{integer_to_binary(ChannelId) => integer_to_binary(MessageId)},
+ maps:get(<<"channels">>, Payload)
+ ).
+
+passive_sync_test_state(GuildId, SessionId, MemberCount) ->
+ UserId = 1130650140672000000,
+ Channel = #{
+ <<"id">> => passive_test_channel_id(),
+ <<"last_message_id">> => passive_test_message_id()
+ },
+ #{
+ id => GuildId,
+ member_count => MemberCount,
+ sessions => passive_test_sessions(SessionId),
+ data => #{
+ <<"guild">> => #{<<"id">> => GuildId, <<"owner_id">> => UserId},
+ <<"channels">> => [Channel],
+ <<"members">> => [#{<<"user">> => #{<<"id">> => UserId}, <<"roles">> => []}],
+ <<"roles">> => []
+ },
+ voice_states => #{}
+ }.
+
+passive_test_channel_id() ->
+ 1497639278555484216.
+
+passive_test_message_id() ->
+ 1500000000000000000.
+
+receive_passive_dispatch(Timeout) ->
+ receive
+ {'$gen_cast', {dispatch, passive_updates, Payload}} when is_map(Payload) ->
+ Payload
+ after Timeout ->
+ no_dispatch
+ end.
+
+flush_passive_dispatches() ->
+ case receive_passive_dispatch(0) of
+ no_dispatch -> ok;
+ _ -> flush_passive_dispatches()
+ end.
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_permission_cache.erl b/fluxer_gateway/src/guild/guild_permission_cache.erl
index a19a3a7ab..52ee39735 100644
--- a/fluxer_gateway/src/guild/guild_permission_cache.erl
+++ b/fluxer_gateway/src/guild/guild_permission_cache.erl
@@ -30,6 +30,7 @@
-export_type([guild_id/0, user_id/0, channel_id/0, guild_state/0, guild_data/0]).
-define(TABLE, guild_permission_cache).
+-define(STRIPPED_MEMBERS_MEMO, guild_permission_cache_stripped_members).
-spec put_state(guild_state()) -> ok.
put_state(State) when is_map(State) ->
@@ -134,7 +135,7 @@ ensure_table() ->
-spec strip_data(guild_data()) -> guild_data().
strip_data(Data) when is_map(Data) ->
Guild = strip_guild(maps:get(<<"guild">>, Data, #{})),
- Members = strip_members(maps:get(<<"members">>, Data, #{})),
+ Members = memoised_strip_members(maps:get(<<"members">>, Data, #{})),
Roles = strip_roles(maps:get(<<"roles">>, Data, [])),
Channels = strip_channels(maps:get(<<"channels">>, Data, [])),
ChannelIndex = strip_channel_index(maps:get(<<"channel_index">>, Data, #{})),
@@ -170,6 +171,54 @@ strip_guild(Guild) when is_map(Guild) ->
strip_guild(_) ->
#{}.
+%% Keyed on the IDENTITY of the source term, never on its value: a hit means this process
+%% already stripped that exact term, and strip_members/1 is pure, so the stored result is
+%% that term's result. Every write to a member map yields a new term and so cannot hit, and
+%% only a completed strip is stored, so a term whose strip raises still raises.
+-spec memoised_strip_members(term()) -> map().
+memoised_strip_members(Members) ->
+ case strip_members_memo_enabled() of
+ true -> strip_members_memoised(Members);
+ false -> strip_members_unmemoised(Members)
+ end.
+
+-spec strip_members_unmemoised(term()) -> map().
+strip_members_unmemoised(Members) ->
+ _ = erlang:erase(?STRIPPED_MEMBERS_MEMO),
+ strip_members(Members).
+
+-spec strip_members_memoised(term()) -> map().
+strip_members_memoised(Members) ->
+ case erlang:get(?STRIPPED_MEMBERS_MEMO) of
+ {Source, Stripped} when is_map(Stripped) ->
+ reuse_stripped_members(Members, Source, Stripped);
+ _ ->
+ store_stripped_members(Members)
+ end.
+
+-spec reuse_stripped_members(term(), term(), map()) -> map().
+reuse_stripped_members(Members, Source, Stripped) ->
+ case erts_debug:same(Members, Source) of
+ true -> Stripped;
+ false -> store_stripped_members(Members)
+ end.
+
+%% Dropped before the new one is built so a member map that shares nothing with the
+%% previous one, a wholesale rebuild, is never held twice on the guild heap at once.
+-spec store_stripped_members(term()) -> map().
+store_stripped_members(Members) ->
+ _ = erlang:erase(?STRIPPED_MEMBERS_MEMO),
+ Stripped = strip_members(Members),
+ _ = erlang:put(?STRIPPED_MEMBERS_MEMO, {Members, Stripped}),
+ Stripped.
+
+-spec strip_members_memo_enabled() -> boolean().
+strip_members_memo_enabled() ->
+ case application:get_env(fluxer_gateway, permission_cache_strip_memo_enabled, true) of
+ false -> false;
+ _ -> true
+ end.
+
-spec strip_members(map() | list() | term()) -> map().
strip_members(Members) when is_map(Members) ->
maps:map(fun(_UserId, Member) -> strip_member(Member) end, Members);
@@ -191,22 +240,30 @@ strip_member_entry(_, Acc) ->
strip_member(Member) when is_map(Member) ->
User = maps:get(<<"user">>, Member, #{}),
StrippedUser = strip_user(User),
- Roles = snowflake_id:parse_list(maps:get(<<"roles">>, Member, [])),
+ Roles = strip_role_ids(maps:get(<<"roles">>, Member, [])),
Base = #{
<<"user">> => StrippedUser,
<<"roles">> => Roles
},
- copy_optional_member_fields([<<"communication_disabled_until">>], Member, Base);
+ copy_if_present(<<"communication_disabled_until">>, Member, Base);
strip_member(_) ->
#{}.
--spec copy_optional_member_fields([binary()], map(), map()) -> map().
-copy_optional_member_fields(Keys, Source, Acc) ->
- lists:foldl(
- fun(Key, Current) -> copy_if_present(Key, Source, Current) end,
- Acc,
- Keys
- ).
+%% snowflake_id:parse/1 returns a positive integer unchanged, so a roles list that already
+%% holds only those is its own parse_list/1 result and does not need to be rebuilt.
+-spec strip_role_ids(term()) -> list().
+strip_role_ids(Roles) when is_list(Roles) ->
+ case all_role_ids_parsed(Roles) of
+ true -> Roles;
+ false -> snowflake_id:parse_list(Roles)
+ end;
+strip_role_ids(Roles) ->
+ snowflake_id:parse_list(Roles).
+
+-spec all_role_ids_parsed(term()) -> boolean().
+all_role_ids_parsed([Id | Rest]) when is_integer(Id), Id > 0 -> all_role_ids_parsed(Rest);
+all_role_ids_parsed([]) -> true;
+all_role_ids_parsed(_) -> false.
-spec copy_if_present(binary(), map(), map()) -> map().
copy_if_present(Key, Source, Current) ->
@@ -327,4 +384,263 @@ strip_member_preserves_communication_disabled_until_test() ->
ok = delete(GuildId)
end.
+strip_member_matches_reference_implementation_test() ->
+ lists:foreach(
+ fun(Member) ->
+ ?assertEqual(reference_strip_member(Member), strip_member(Member))
+ end,
+ strip_member_cases()
+ ).
+
+strip_member_cases() ->
+ Until = <<"2026-05-09T22:00:00.000Z">>,
+ User = #{<<"id">> => 5, <<"username">> => <<"dropped">>},
+ [
+ #{<<"user">> => User, <<"roles">> => [7, 9]},
+ #{<<"user">> => User, <<"roles">> => [<<"7">>, 9]},
+ #{<<"user">> => User, <<"roles">> => [<<"7">>, <<"9">>]},
+ #{<<"user">> => User, <<"roles">> => []},
+ #{<<"user">> => User},
+ #{<<"roles">> => [7]},
+ #{<<"user">> => User, <<"roles">> => [7], <<"communication_disabled_until">> => Until},
+ #{<<"user">> => User, <<"roles">> => [7], <<"nick">> => <<"dropped">>},
+ #{<<"user">> => #{}, <<"roles">> => undefined},
+ #{<<"user">> => not_a_map, <<"roles">> => null},
+ not_a_member
+ ].
+
+reference_strip_member(Member) when is_map(Member) ->
+ StrippedUser = strip_user(maps:get(<<"user">>, Member, #{})),
+ Roles = snowflake_id:parse_list(maps:get(<<"roles">>, Member, [])),
+ Base = #{<<"user">> => StrippedUser, <<"roles">> => Roles},
+ lists:foldl(
+ fun(Key, Current) -> copy_if_present(Key, Member, Current) end,
+ Base,
+ [<<"communication_disabled_until">>]
+ );
+reference_strip_member(_) ->
+ #{}.
+
+strip_member_rejects_unparsed_role_ids_test() ->
+ ?assertError({invalid_snowflake, 0}, strip_member(#{<<"roles">> => [0]})),
+ ?assertError({invalid_snowflake, -3}, strip_member(#{<<"roles">> => [-3]})),
+ ?assertError({invalid_snowflake, <<"0">>}, strip_member(#{<<"roles">> => [<<"0">>]})),
+ ?assertError({invalid_snowflake, bad}, strip_member(#{<<"roles">> => [7, bad]})),
+ ?assertError({invalid_snowflake_list, oops}, strip_member(#{<<"roles">> => oops})).
+
+strip_members_keeps_already_parsed_role_ids_test() ->
+ GuildId = 905,
+ UserId = 906,
+ Member = #{<<"user">> => #{<<"id">> => UserId}, <<"roles">> => [42, 43]},
+ Data = #{
+ <<"guild">> => #{<<"owner_id">> => <<"1">>},
+ <<"roles">> => [],
+ <<"members">> => #{UserId => Member},
+ <<"channels">> => []
+ },
+ ok = put_data(GuildId, Data),
+ try
+ {ok, #{} = MemberData} = get_member(GuildId, UserId),
+ ?assertEqual([42, 43], maps:get(<<"roles">>, MemberData)),
+ ?assertEqual(#{<<"id">> => UserId}, maps:get(<<"user">>, MemberData))
+ after
+ ok = delete(GuildId)
+ end.
+
+memo_reset() ->
+ erlang:erase(?STRIPPED_MEMBERS_MEMO).
+
+memo_member(Id, Roles) ->
+ #{
+ <<"user">> => #{<<"id">> => integer_to_binary(Id), <<"username">> => <<"dropped">>},
+ <<"roles">> => Roles,
+ <<"nick">> => <<"dropped">>
+ }.
+
+memo_members(Ids) ->
+ maps:from_list([{Id, memo_member(Id, [<<"42">>])} || Id <- Ids]).
+
+%% strip_members/1 as it read before the memo, over the pre-shortcut strip_member/1.
+reference_strip_members(Members) when is_map(Members) ->
+ maps:map(fun(_UserId, Member) -> reference_strip_member(Member) end, Members);
+reference_strip_members(Members) when is_list(Members) ->
+ lists:foldl(fun reference_strip_member_entry/2, #{}, Members);
+reference_strip_members(_) ->
+ #{}.
+
+reference_strip_member_entry(Member, Acc) when is_map(Member) ->
+ case get_member_user_id(Member) of
+ undefined -> Acc;
+ UserId -> Acc#{UserId => reference_strip_member(Member)}
+ end;
+reference_strip_member_entry(_, Acc) ->
+ Acc.
+
+assert_memo_matches_reference(Members) ->
+ Expected = reference_strip_members(Members),
+ ?assertEqual(Expected, memoised_strip_members(Members)),
+ ?assertEqual(Expected, memoised_strip_members(Members)).
+
+memo_members_cases() ->
+ [
+ #{},
+ memo_members([1]),
+ memo_members([1, 2, 3]),
+ #{7 => memo_member(7, [])},
+ [memo_member(4, [<<"42">>]), memo_member(5, [7, 9])],
+ [memo_member(6, []), not_a_member],
+ [],
+ not_a_member_map,
+ undefined
+ ].
+
+memoised_strip_members_matches_reference_test() ->
+ memo_reset(),
+ try
+ lists:foreach(fun assert_memo_matches_reference/1, memo_members_cases())
+ after
+ memo_reset()
+ end.
+
+memo_reuses_the_stripped_term_for_an_identical_members_term_test() ->
+ memo_reset(),
+ try
+ Members = memo_members([1, 2]),
+ First = memoised_strip_members(Members),
+ Second = memoised_strip_members(Members),
+ ?assertEqual(reference_strip_members(Members), First),
+ ?assert(erts_debug:same(First, Second)),
+ ?assertEqual({Members, First}, erlang:get(?STRIPPED_MEMBERS_MEMO))
+ after
+ memo_reset()
+ end.
+
+memo_recomputes_for_an_equal_but_distinct_members_term_test() ->
+ memo_reset(),
+ try
+ Members = memo_members([1, 2]),
+ Copy = binary_to_term(term_to_binary(Members)),
+ Stripped = memoised_strip_members(Members),
+ ?assertNot(erts_debug:same(Members, Copy)),
+ ?assertEqual(Stripped, memoised_strip_members(Copy)),
+ {Source, _Stripped} = erlang:get(?STRIPPED_MEMBERS_MEMO),
+ ?assert(erts_debug:same(Copy, Source))
+ after
+ memo_reset()
+ end.
+
+memo_changed_member_maps(Members) ->
+ [
+ Members#{2 => memo_member(2, [<<"7">>])},
+ Members#{3 => memo_member(3, [<<"42">>])},
+ maps:remove(1, Members),
+ maps:map(fun(Id, _Member) -> memo_member(Id, []) end, Members)
+ ].
+
+memo_follows_every_change_to_the_member_map_test() ->
+ memo_reset(),
+ try
+ Members = memo_members([1, 2]),
+ _Stripped = memoised_strip_members(Members),
+ lists:foreach(
+ fun assert_memo_matches_reference/1,
+ memo_changed_member_maps(Members)
+ )
+ after
+ memo_reset()
+ end.
+
+memo_never_suppresses_an_invalid_role_id_test() ->
+ memo_reset(),
+ try
+ Good = memo_members([1]),
+ Stripped = memoised_strip_members(Good),
+ Bad = #{2 => memo_member(2, [bad])},
+ ?assertError({invalid_snowflake, bad}, memoised_strip_members(Bad)),
+ ?assertEqual(undefined, erlang:get(?STRIPPED_MEMBERS_MEMO)),
+ ?assertError({invalid_snowflake, bad}, memoised_strip_members(Bad)),
+ ?assertEqual(Stripped, memoised_strip_members(Good)),
+ ?assertEqual({Good, Stripped}, erlang:get(?STRIPPED_MEMBERS_MEMO))
+ after
+ memo_reset()
+ end.
+
+memo_ignores_an_unrecognised_entry_test() ->
+ memo_reset(),
+ try
+ Members = memo_members([1]),
+ Expected = reference_strip_members(Members),
+ erlang:put(?STRIPPED_MEMBERS_MEMO, {Members, junk}),
+ ?assertEqual(Expected, memoised_strip_members(Members)),
+ erlang:put(?STRIPPED_MEMBERS_MEMO, stale_shape),
+ ?assertEqual(Expected, memoised_strip_members(Members)),
+ ?assertEqual({Members, Expected}, erlang:get(?STRIPPED_MEMBERS_MEMO))
+ after
+ memo_reset()
+ end.
+
+memo_disabled_matches_reference_and_releases_the_memo_test() ->
+ memo_reset(),
+ Members = memo_members([1, 2]),
+ Expected = reference_strip_members(Members),
+ ?assertEqual(Expected, memoised_strip_members(Members)),
+ application:set_env(fluxer_gateway, permission_cache_strip_memo_enabled, false),
+ try
+ ?assertEqual(Expected, memoised_strip_members(Members)),
+ ?assertEqual(undefined, erlang:get(?STRIPPED_MEMBERS_MEMO))
+ after
+ application:unset_env(fluxer_gateway, permission_cache_strip_memo_enabled),
+ memo_reset()
+ end.
+
+%% strip_data/1 with the memo off is the computation this module ran before the memo.
+unmemoised_strip_data(Data) ->
+ application:set_env(fluxer_gateway, permission_cache_strip_memo_enabled, false),
+ try
+ strip_data(Data)
+ after
+ application:unset_env(fluxer_gateway, permission_cache_strip_memo_enabled)
+ end.
+
+memo_test_data(OwnerId) ->
+ guild_data_index:normalize_data(#{
+ <<"guild">> => #{<<"owner_id">> => OwnerId},
+ <<"roles">> => [#{<<"id">> => <<"42">>, <<"permissions">> => <<"0">>}],
+ <<"members">> => [memo_member(1, [<<"42">>]), memo_member(2, [])],
+ <<"channels">> => [#{<<"id">> => <<"500">>, <<"permission_overwrites">> => []}]
+ }).
+
+strip_data_reuses_the_stripped_members_across_a_non_member_change_test() ->
+ memo_reset(),
+ Data = memo_test_data(<<"1">>),
+ Updated = Data#{<<"guild">> => #{<<"owner_id">> => <<"9">>}},
+ Expected = unmemoised_strip_data(Data),
+ ExpectedUpdated = unmemoised_strip_data(Updated),
+ memo_reset(),
+ try
+ ?assertEqual(Expected, strip_data(Data)),
+ ?assertEqual(ExpectedUpdated, strip_data(Updated)),
+ MembersA = maps:get(<<"members">>, strip_data(Data)),
+ MembersB = maps:get(<<"members">>, strip_data(Updated)),
+ ?assert(erts_debug:same(MembersA, MembersB))
+ after
+ memo_reset()
+ end.
+
+put_normalized_data_writes_the_same_snapshot_with_a_warm_memo_test() ->
+ GuildId = 907,
+ memo_reset(),
+ Data = memo_test_data(<<"1">>),
+ try
+ ok = put_normalized_data(GuildId, Data),
+ {ok, First} = get_snapshot(GuildId),
+ ok = put_normalized_data(GuildId, Data),
+ {ok, Second} = get_snapshot(GuildId),
+ ?assertEqual(First, Second),
+ ?assertEqual(#{id => GuildId, data => unmemoised_strip_data(Data)}, Second)
+ after
+ memo_reset(),
+ ok = delete(GuildId)
+ end.
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_presence.erl b/fluxer_gateway/src/guild/guild_presence.erl
index 07719adab..dd06a85c0 100644
--- a/fluxer_gateway/src/guild/guild_presence.erl
+++ b/fluxer_gateway/src/guild/guild_presence.erl
@@ -14,8 +14,13 @@
-type member() :: map().
-type user_id() :: integer().
--define(HEAVY_MEMBER_DATA_KEYS, [<<"members">>, members_normalized, <<"member_role_index">>]).
+%% members_sorted_ids trims with the member map it indexes: a snapshot that kept it would
+%% answer sorted_member_ids/2 with ids for members the snapshot no longer carries.
+-define(HEAVY_MEMBER_DATA_KEYS, [
+ <<"members">>, members_normalized, <<"member_role_index">>, members_sorted_ids
+]).
-define(PRESENCE_SNAPSHOT_TRIM_MEMBER_THRESHOLD_DEFAULT, 5000).
+-define(SESSIONS_PROJECTION_CACHE, presence_snapshot_sessions_cache).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
@@ -112,12 +117,17 @@ spawn_presence_broadcast(UserId, OldPresence, PresenceMap, OldState, NewState) -
PresenceMap
),
{Pid, NewState2} = guild_broadcaster:ensure(NewState1),
- NewSnap = build_broadcast_snapshot(NewState2),
+ {NewSnap, NewState3} = build_broadcast_snapshot_cached(NewState2),
guild_broadcaster:cast_presence(Pid, UserId, PresenceMap, #{}, NewSnap),
- NewState2.
+ NewState3.
-spec build_broadcast_snapshot(guild_state()) -> map().
build_broadcast_snapshot(State) ->
+ {Snapshot, _State} = build_broadcast_snapshot_cached(State),
+ Snapshot.
+
+-spec build_broadcast_snapshot_cached(guild_state()) -> {map(), guild_state()}.
+build_broadcast_snapshot_cached(State) ->
maybe_trim_snapshot(build_base_snapshot(State), State).
-spec build_base_snapshot(guild_state()) -> map().
@@ -139,11 +149,11 @@ build_base_snapshot(State) ->
Keys
).
--spec maybe_trim_snapshot(map(), guild_state()) -> map().
+-spec maybe_trim_snapshot(map(), guild_state()) -> {map(), guild_state()}.
maybe_trim_snapshot(Snapshot, State) ->
case should_trim_snapshot(State) of
- true -> trim_snapshot(Snapshot);
- false -> Snapshot
+ true -> trim_snapshot(Snapshot, State);
+ false -> {Snapshot, State}
end.
-spec should_trim_snapshot(guild_state()) -> boolean().
@@ -179,9 +189,9 @@ presence_snapshot_trim_member_threshold() ->
_ -> ?PRESENCE_SNAPSHOT_TRIM_MEMBER_THRESHOLD_DEFAULT
end.
--spec trim_snapshot(map()) -> map().
-trim_snapshot(Snapshot) ->
- trim_snapshot_sessions(trim_snapshot_data(Snapshot)).
+-spec trim_snapshot(map(), guild_state()) -> {map(), guild_state()}.
+trim_snapshot(Snapshot, State) ->
+ trim_snapshot_sessions(trim_snapshot_data(Snapshot), State).
-spec trim_snapshot_data(map()) -> map().
trim_snapshot_data(#{data := Data} = Snapshot) when is_map(Data) ->
@@ -189,15 +199,33 @@ trim_snapshot_data(#{data := Data} = Snapshot) when is_map(Data) ->
trim_snapshot_data(Snapshot) ->
Snapshot.
--spec trim_snapshot_sessions(map()) -> map().
-trim_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) ->
- Projected = maps:map(
+-spec trim_snapshot_sessions(map(), guild_state()) -> {map(), guild_state()}.
+trim_snapshot_sessions(#{sessions := Sessions} = Snapshot, State) when is_map(Sessions) ->
+ {Projected, NewState} = cached_projected_sessions(Sessions, State),
+ {Snapshot#{sessions => Projected}, NewState};
+trim_snapshot_sessions(Snapshot, State) ->
+ {Snapshot, State}.
+
+%% Keyed by the sessions map itself, so every writer of that map invalidates the
+%% projection without having to know this cache exists.
+-spec cached_projected_sessions(map(), guild_state()) -> {map(), guild_state()}.
+cached_projected_sessions(Sessions, State) ->
+ case maps:get(?SESSIONS_PROJECTION_CACHE, State, undefined) of
+ {Sessions, Projected} when is_map(Projected) -> {Projected, State};
+ _ -> store_projected_sessions(Sessions, State)
+ end.
+
+-spec store_projected_sessions(map(), guild_state()) -> {map(), guild_state()}.
+store_projected_sessions(Sessions, State) ->
+ Projected = project_sessions(Sessions),
+ {Projected, State#{?SESSIONS_PROJECTION_CACHE => {Sessions, Projected}}}.
+
+-spec project_sessions(map()) -> map().
+project_sessions(Sessions) ->
+ maps:map(
fun(_SessionId, SessionData) -> project_session(SessionData) end,
Sessions
- ),
- Snapshot#{sessions => Projected};
-trim_snapshot_sessions(Snapshot) ->
- Snapshot.
+ ).
-spec project_session(term()) -> term().
project_session(SessionData) when is_map(SessionData) ->
@@ -440,6 +468,7 @@ snapshot_trim_test_state(Tab) ->
<<"members">> => #{1 => #{<<"user">> => #{<<"id">> => <<"1">>}}},
members_normalized => #{1 => #{<<"user">> => #{<<"id">> => <<"1">>}}},
<<"member_role_index">> => #{1 => []},
+ members_sorted_ids => [1],
<<"channels">> => [],
<<"roles">> => []
},
@@ -463,6 +492,7 @@ build_broadcast_snapshot_trims_by_default_test() ->
?assertNot(maps:is_key(<<"members">>, SnapData)),
?assertNot(maps:is_key(members_normalized, SnapData)),
?assertNot(maps:is_key(<<"member_role_index">>, SnapData)),
+ ?assertNot(maps:is_key(members_sorted_ids, SnapData)),
?assertEqual(Tab, maps:get(members_ets, SnapData)),
?assert(maps:is_key(<<"channels">>, SnapData)),
SnapSession = maps:get(<<"s1">>, maps:get(sessions, Snap)),
@@ -499,4 +529,149 @@ build_broadcast_snapshot_no_trim_below_threshold_test() ->
ets:delete(Tab)
end.
+reference_project_session(SessionData) when is_map(SessionData) ->
+ maps:with([user_id, pid, viewable_channels], SessionData);
+reference_project_session(SessionData) ->
+ SessionData.
+
+reference_project_sessions(Sessions) ->
+ maps:map(
+ fun(_SessionId, SessionData) -> reference_project_session(SessionData) end,
+ Sessions
+ ).
+
+reference_trim_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) ->
+ Snapshot#{sessions => reference_project_sessions(Sessions)};
+reference_trim_snapshot_sessions(Snapshot) ->
+ Snapshot.
+
+reference_maybe_trim_snapshot(true, Snapshot) ->
+ reference_trim_snapshot_sessions(trim_snapshot_data(Snapshot));
+reference_maybe_trim_snapshot(false, Snapshot) ->
+ Snapshot.
+
+reference_broadcast_snapshot(State) ->
+ reference_maybe_trim_snapshot(should_trim_snapshot(State), build_base_snapshot(State)).
+
+cache_test_session(UserId, ViewableChannels) ->
+ #{
+ user_id => UserId,
+ pid => self(),
+ viewable_channels => ViewableChannels,
+ active_guilds => [42],
+ pending_connect => false
+ }.
+
+cache_test_state(Sessions) ->
+ #{
+ id => 42,
+ member_count => 10,
+ data => #{<<"channels">> => [], <<"members">> => #{}},
+ sessions => Sessions
+ }.
+
+assert_cached_snapshot_matches_reference(Sessions, State) ->
+ NextState = State#{sessions => Sessions},
+ {Snapshot, CachedState} = build_broadcast_snapshot_cached(NextState),
+ ?assertEqual(reference_broadcast_snapshot(NextState), Snapshot),
+ ?assertNot(maps:is_key(?SESSIONS_PROJECTION_CACHE, Snapshot)),
+ CachedState.
+
+session_projection_mutation_sequence() ->
+ A = cache_test_session(1, #{100 => true}),
+ B = cache_test_session(2, #{100 => true, 200 => true}),
+ S1 = #{<<"a">> => A},
+ S2 = S1#{<<"b">> => B},
+ S3 = S2#{<<"a">> => cache_test_session(1, #{100 => true, 300 => true})},
+ S4 = maps:remove(<<"b">>, S3),
+ S5 = S4#{<<"a">> => (maps:get(<<"a">>, S4))#{pending_connect => true}},
+ S6 = S5#{<<"c">> => not_a_map},
+ [#{}, S1, S1, S2, S3, S4, S5, S6, S1, #{}].
+
+cached_projection_matches_reference_across_mutations_test() ->
+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),
+ try
+ lists:foldl(
+ fun assert_cached_snapshot_matches_reference/2,
+ cache_test_state(#{}),
+ session_projection_mutation_sequence()
+ )
+ after
+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)
+ end.
+
+cached_projection_is_reused_when_sessions_unchanged_test() ->
+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),
+ try
+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},
+ {Snap1, State1} = build_broadcast_snapshot_cached(cache_test_state(Sessions)),
+ {Snap2, State2} = build_broadcast_snapshot_cached(State1),
+ Projected1 = maps:get(sessions, Snap1),
+ Projected2 = maps:get(sessions, Snap2),
+ ?assertEqual(reference_project_sessions(Sessions), Projected1),
+ ?assertEqual(Projected1, Projected2),
+ ?assert(erts_debug:same(Projected1, Projected2)),
+ ?assertEqual({Sessions, Projected1}, maps:get(?SESSIONS_PROJECTION_CACHE, State2))
+ after
+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)
+ end.
+
+cached_projection_ignores_foreign_cache_entry_test() ->
+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),
+ try
+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},
+ Stale = #{<<"z">> => cache_test_session(9, #{999 => true})},
+ Foreign = {Stale, reference_project_sessions(Stale)},
+ State = (cache_test_state(Sessions))#{?SESSIONS_PROJECTION_CACHE => Foreign},
+ {Snapshot, _NewState} = build_broadcast_snapshot_cached(State),
+ ?assertEqual(reference_broadcast_snapshot(State), Snapshot),
+ ?assertEqual(reference_project_sessions(Sessions), maps:get(sessions, Snapshot))
+ after
+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)
+ end.
+
+cached_projection_ignores_corrupt_cache_entry_test() ->
+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),
+ try
+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},
+ State = (cache_test_state(Sessions))#{?SESSIONS_PROJECTION_CACHE => {Sessions, junk}},
+ {Snapshot, _NewState} = build_broadcast_snapshot_cached(State),
+ ?assertEqual(reference_project_sessions(Sessions), maps:get(sessions, Snapshot))
+ after
+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)
+ end.
+
+cached_projection_not_stored_when_trim_disabled_test() ->
+ application:set_env(fluxer_gateway, presence_snapshot_trim_enabled, false),
+ try
+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},
+ State = cache_test_state(Sessions),
+ {Snapshot, NewState} = build_broadcast_snapshot_cached(State),
+ ?assertEqual(reference_broadcast_snapshot(State), Snapshot),
+ ?assertEqual(Sessions, maps:get(sessions, Snapshot)),
+ ?assertNot(maps:is_key(?SESSIONS_PROJECTION_CACHE, NewState))
+ after
+ application:unset_env(fluxer_gateway, presence_snapshot_trim_enabled)
+ end.
+
+handle_bus_presence_threads_projection_cache_test() ->
+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),
+ Sessions = #{<<"s1">> => cache_test_session(1, #{100 => true})},
+ State = (presence_test_state())#{member_count => 10, sessions => Sessions},
+ Payload = #{
+ <<"status">> => <<"online">>,
+ <<"mobile">> => true,
+ <<"afk">> => false,
+ <<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"Alpha">>}
+ },
+ try
+ {noreply, NewState} = handle_bus_presence(1, Payload, State),
+ ?assertEqual(
+ {Sessions, reference_project_sessions(Sessions)},
+ maps:get(?SESSIONS_PROJECTION_CACHE, NewState)
+ )
+ after
+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)
+ end.
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_public_online.erl b/fluxer_gateway/src/guild/guild_public_online.erl
index c118903d3..894975d6c 100644
--- a/fluxer_gateway/src/guild/guild_public_online.erl
+++ b/fluxer_gateway/src/guild/guild_public_online.erl
@@ -15,6 +15,8 @@
-type channel() :: map().
-type user_id() :: integer().
+-define(ALL_PERMS_VIEWER_KEY, public_online_all_perms_viewer).
+
-spec compute_count(guild_state()) -> non_neg_integer().
compute_count(State) ->
case resolve_data(State) of
@@ -57,7 +59,15 @@ classify_everyone_viewable(Channels, State) ->
case {OpenAcc, RestrictedAcc} of
{[], []} -> empty;
{[_ | _], _} -> {open, OpenAcc};
- {[], _} -> {restricted, sets:from_list(RestrictedAcc)}
+ {[], _} -> classify_restricted(RestrictedAcc)
+ end.
+
+-spec classify_restricted([integer()]) ->
+ {open, [integer()]} | {restricted, sets:set(integer())}.
+classify_restricted(RestrictedIds) ->
+ case application:get_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, false) of
+ true -> {open, RestrictedIds};
+ _ -> {restricted, sets:from_list(RestrictedIds)}
end.
-spec classify_channel_visibility(
@@ -200,9 +210,66 @@ build_state(Channels, Members, Roles, Presences) ->
<<"channels">> => Channels
},
member_presence => Tab,
+ member_list_engine => build_engine(Presences),
sessions => #{}
}.
+build_engine(Presences) ->
+ Online = length([S || {_UserId, S} <- Presences, S =/= <<"offline">>, S =/= <<"invisible">>]),
+ Ref = ets:new(test_member_list_engine, [set, public]),
+ ets:insert(Ref, [{total_count, length(Presences)}, {online_count, Online}]),
+ Ref.
+
+deny_view_overwrite(Id, Type) ->
+ #{
+ <<"id">> => integer_to_binary(Id),
+ <<"type">> => Type,
+ <<"allow">> => <<"0">>,
+ <<"deny">> => integer_to_binary(view_perm())
+ }.
+
+channel(ChannelId, Overwrites) ->
+ #{
+ <<"id">> => integer_to_binary(ChannelId),
+ <<"type">> => 0,
+ <<"permission_overwrites">> => Overwrites
+ }.
+
+role(RoleId) ->
+ #{<<"id">> => integer_to_binary(RoleId), <<"permissions">> => <<"0">>}.
+
+with_all_perms_viewer(Fun) ->
+ Previous = application:get_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY),
+ application:set_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, true),
+ try
+ Fun()
+ after
+ restore_all_perms_viewer(Previous)
+ end.
+
+restore_all_perms_viewer(undefined) ->
+ application:unset_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY);
+restore_all_perms_viewer({ok, Value}) ->
+ application:set_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, Value).
+
+verify_gate_state() ->
+ VerifiedRoleId = 4242,
+ Channels = [channel(400, [deny_view_overwrite(VerifiedRoleId, 0)])],
+ Members = [{1, [VerifiedRoleId]}, {2, [VerifiedRoleId]}, {3, [VerifiedRoleId]}, {4, []}],
+ Presences = [
+ {1, <<"online">>},
+ {2, <<"online">>},
+ {3, <<"idle">>},
+ {4, <<"online">>}
+ ],
+ build_state(Channels, Members, [role(VerifiedRoleId)], Presences).
+
+user_deny_state() ->
+ Channels = [channel(700, [deny_view_overwrite(42, 1)])],
+ Members = [{41, []}, {42, []}, {43, []}],
+ Presences = [{41, <<"online">>}, {42, <<"online">>}, {43, <<"online">>}],
+ build_state(Channels, Members, [], Presences).
+
returns_zero_when_no_channels_test() ->
State = build_state([], [], [], []),
?assertEqual(0, compute_count(State)).
@@ -303,4 +370,36 @@ channel_has_view_restricting_overrides_no_overwrites_test() ->
Channel = #{<<"permission_overwrites">> => []},
?assertNot(channel_has_view_restricting_overrides(Channel, view_perm())).
+verify_gate_collapses_count_without_flag_test() ->
+ ?assertEqual(1, compute_count(verify_gate_state())).
+
+verify_gate_reports_full_count_with_flag_test() ->
+ State = verify_gate_state(),
+ ?assertEqual(4, with_all_perms_viewer(fun() -> compute_count(State) end)).
+
+everyone_deny_channel_stays_excluded_with_flag_test() ->
+ GuildId = guild_id(),
+ Channels = [channel(500, [deny_view_overwrite(GuildId, 0)])],
+ State = build_state(Channels, [{1, []}, {2, []}], [], [{1, <<"online">>}, {2, <<"idle">>}]),
+ ?assertEqual(0, compute_count(State)),
+ ?assertEqual(0, with_all_perms_viewer(fun() -> compute_count(State) end)).
+
+no_channels_stays_zero_with_flag_test() ->
+ State = build_state([], [{1, []}], [], [{1, <<"online">>}]),
+ ?assertEqual(0, with_all_perms_viewer(fun() -> compute_count(State) end)).
+
+plain_open_channel_unchanged_by_flag_test() ->
+ Members = [{1, []}, {2, []}, {3, []}],
+ Presences = [{1, <<"online">>}, {2, <<"idle">>}, {3, <<"offline">>}],
+ State = build_state([channel(600, [])], Members, [], Presences),
+ ?assertEqual(2, compute_count(State)),
+ ?assertEqual(2, with_all_perms_viewer(fun() -> compute_count(State) end)).
+
+user_deny_collapses_count_without_flag_test() ->
+ ?assertEqual(2, compute_count(user_deny_state())).
+
+user_deny_reports_full_count_with_flag_test() ->
+ State = user_deny_state(),
+ ?assertEqual(3, with_all_perms_viewer(fun() -> compute_count(State) end)).
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_sessions.erl b/fluxer_gateway/src/guild/guild_sessions.erl
index 95a5035e2..63e3ea108 100644
--- a/fluxer_gateway/src/guild/guild_sessions.erl
+++ b/fluxer_gateway/src/guild/guild_sessions.erl
@@ -28,6 +28,8 @@
build_viewable_channel_map/1
]).
+-define(MAX_PERM_MEMO_ENTRIES, 8192).
+
-type guild_state() :: map().
-type session_id() :: binary().
-type user_id() :: integer().
@@ -36,6 +38,8 @@
-type session_data() :: map().
-type sessions_map() :: #{session_id() => session_data()}.
-type session_pair() :: {session_id(), session_data()}.
+-type perm_memo() :: #{user_id() => non_neg_integer()}.
+-type message_ctx() :: {channel_id(), binary(), session_id() | undefined, guild_state()}.
-export_type([
guild_state/0,
session_id/0,
@@ -59,7 +63,116 @@ handle_session_connect(Request, Pid, State) ->
-spec handle_session_down(reference(), guild_state()) ->
{noreply, guild_state()} | {stop, normal, guild_state()}.
handle_session_down(Ref, State) ->
- guild_sessions_connect:handle_session_down(Ref, State).
+ case pending_session_by_ref(Ref, State) of
+ {ok, SessionId, Session, Sessions} ->
+ handle_pending_ref_down(SessionId, Session, Ref, Sessions, State);
+ not_found ->
+ guild_sessions_connect:handle_session_down(Ref, State)
+ end.
+
+-spec handle_pending_ref_down(
+ session_id(), session_data(), reference(), sessions_map(), guild_state()
+) -> {noreply, guild_state()} | {stop, normal, guild_state()}.
+handle_pending_ref_down(SessionId, Session, Ref, Sessions, State) ->
+ case pending_session_owns_connected_tracking(Session, Sessions, State) of
+ true -> guild_sessions_connect:handle_session_down(Ref, State);
+ false -> handle_pending_session_down(SessionId, Ref, Sessions, State)
+ end.
+
+-spec pending_session_by_ref(reference(), guild_state()) ->
+ {ok, session_id(), session_data(), sessions_map()} | not_found.
+pending_session_by_ref(Ref, State) ->
+ Sessions = maps:get(sessions, State, #{}),
+ case session_id_by_ref(Ref, Sessions, State) of
+ SessionId when is_binary(SessionId) -> pending_session_entry(SessionId, Sessions);
+ _ -> not_found
+ end.
+
+-spec pending_session_entry(session_id(), sessions_map()) ->
+ {ok, session_id(), session_data(), sessions_map()} | not_found.
+pending_session_entry(SessionId, Sessions) ->
+ case maps:get(SessionId, Sessions, undefined) of
+ #{pending_connect := true} = Session -> {ok, SessionId, Session, Sessions};
+ _ -> not_found
+ end.
+
+-spec session_id_by_ref(reference(), sessions_map(), guild_state()) -> session_id() | undefined.
+session_id_by_ref(Ref, Sessions, State) ->
+ Refs = maps:get(guild_session_refs, State, #{}),
+ case maps:get(Ref, Refs, undefined) of
+ SessionId when is_binary(SessionId) -> SessionId;
+ _ -> session_id_by_ref_scan(Ref, Sessions)
+ end.
+
+-spec session_id_by_ref_scan(reference(), sessions_map()) -> session_id() | undefined.
+session_id_by_ref_scan(Ref, Sessions) ->
+ maps:fold(
+ fun(SessionId, Session, Found) ->
+ match_session_ref(Ref, SessionId, Session, Found)
+ end,
+ undefined,
+ Sessions
+ ).
+
+-spec match_session_ref(reference(), session_id(), session_data(), session_id() | undefined) ->
+ session_id() | undefined.
+match_session_ref(Ref, SessionId, #{mref := Ref}, _Found) -> SessionId;
+match_session_ref(_Ref, _SessionId, _Session, Found) -> Found.
+
+-spec pending_session_owns_connected_tracking(session_data(), sessions_map(), guild_state()) ->
+ boolean().
+pending_session_owns_connected_tracking(Session, Sessions, State) ->
+ UserId = maps:get(user_id, Session, undefined),
+ Counts = maps:get(user_session_counts, State, #{}),
+ TrackedCount = non_negative_count(maps:get(UserId, Counts, 0)),
+ TrackedCount > active_session_count(UserId, Sessions).
+
+-spec non_negative_count(term()) -> non_neg_integer().
+non_negative_count(Count) when is_integer(Count), Count >= 0 -> Count;
+non_negative_count(_) -> 0.
+
+-spec active_session_count(user_id() | undefined, sessions_map()) -> non_neg_integer().
+active_session_count(UserId, Sessions) ->
+ maps:fold(
+ fun(_SessionId, Session, Count) ->
+ count_active_session(UserId, Session, Count)
+ end,
+ 0,
+ Sessions
+ ).
+
+-spec count_active_session(user_id() | undefined, session_data(), non_neg_integer()) ->
+ non_neg_integer().
+count_active_session(UserId, #{user_id := UserId} = Session, Count) ->
+ case maps:get(pending_connect, Session, false) of
+ true -> Count;
+ false -> Count + 1
+ end;
+count_active_session(_UserId, _Session, Count) ->
+ Count.
+
+-spec handle_pending_session_down(session_id(), reference(), sessions_map(), guild_state()) ->
+ {noreply, guild_state()}.
+handle_pending_session_down(SessionId, Ref, Sessions, State) ->
+ NewSessions = maps:remove(SessionId, Sessions),
+ SessionRefs = maps:get(guild_session_refs, State, #{}),
+ State1 = State#{
+ sessions => NewSessions,
+ guild_session_refs => maps:remove(Ref, SessionRefs)
+ },
+ State2 = guild_sessions_connect_cleanup:cleanup_connect_admission_for_session(
+ SessionId, State1
+ ),
+ finish_pending_session_down(NewSessions, State2).
+
+-spec finish_pending_session_down(sessions_map(), guild_state()) -> {noreply, guild_state()}.
+finish_pending_session_down(NewSessions, State) ->
+ case map_size(NewSessions) of
+ 0 ->
+ {noreply, guild_sessions_connect_cleanup:maybe_mark_auto_stop_pending(State)};
+ _ ->
+ {noreply, guild_sessions_connect_cleanup:clear_auto_stop_pending(State)}
+ end.
-spec remove_session(session_id(), guild_state()) -> guild_state().
remove_session(SessionId, State) ->
@@ -121,21 +234,68 @@ filter_sessions_for_channel(Sessions, ChannelId, SessionIdOpt, State) ->
sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state()
) -> [session_pair()].
filter_sessions_for_message(Sessions, ChannelId, MessageId, SessionIdOpt, State) ->
- filter_active_sessions(Sessions, SessionIdOpt, fun(S, _Sid) ->
- session_can_view_channel(S, ChannelId, State) andalso
- session_can_access_message(S, ChannelId, MessageId, State)
- end).
+ filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State).
--spec session_can_access_message(map(), channel_id(), binary(), guild_state()) -> boolean().
-session_can_access_message(SessionData, ChannelId, MessageId, State) ->
- case maps:get(user_id, SessionData, undefined) of
+-spec filter_message_memo(
+ sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state()
+) -> [session_pair()].
+filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State) ->
+ Ctx = {ChannelId, MessageId, SessionIdOpt, State},
+ {Acc, _Memo} = maps:fold(
+ fun(Sid, S, In) -> collect_message_session(Sid, S, Ctx, In) end,
+ {[], #{}},
+ Sessions
+ ),
+ Acc.
+
+-spec collect_message_session(
+ session_id(), session_data(), message_ctx(), {[session_pair()], perm_memo()}
+) -> {[session_pair()], perm_memo()}.
+collect_message_session(Sid, S, Ctx, {Acc, Memo}) ->
+ {ChannelId, _MessageId, SessionIdOpt, State} = Ctx,
+ Visible =
+ not is_pending_or_excluded(Sid, S, SessionIdOpt) andalso
+ session_can_view_channel(S, ChannelId, State),
+ case Visible of
+ true -> memo_message_session(Sid, S, Ctx, Acc, Memo);
+ false -> {Acc, Memo}
+ end.
+
+-spec memo_message_session(
+ session_id(), session_data(), message_ctx(), [session_pair()], perm_memo()
+) -> {[session_pair()], perm_memo()}.
+memo_message_session(Sid, S, {ChannelId, MessageId, _SessionIdOpt, State}, Acc, Memo) ->
+ case maps:get(user_id, S, undefined) of
UserId when is_integer(UserId) ->
- Perms = guild_permissions:get_member_permissions(UserId, ChannelId, State),
- guild_permissions:can_access_message_by_permissions(Perms, MessageId, State);
+ {Perms, Memo1} = memo_member_permissions(UserId, ChannelId, State, Memo),
+ Ok = guild_permissions:can_access_message_by_permissions(Perms, MessageId, State),
+ {prepend_session(Ok, Sid, S, Acc), Memo1};
_ ->
- false
+ {Acc, Memo}
end.
+-spec memo_member_permissions(user_id(), channel_id(), guild_state(), perm_memo()) ->
+ {non_neg_integer(), perm_memo()}.
+memo_member_permissions(UserId, ChannelId, State, Memo) ->
+ case maps:get(UserId, Memo, undefined) of
+ Perms when is_integer(Perms) ->
+ {Perms, Memo};
+ _ ->
+ Computed = guild_permissions:get_member_permissions(UserId, ChannelId, State),
+ {Computed, store_perm_memo(UserId, Computed, Memo)}
+ end.
+
+-spec store_perm_memo(user_id(), non_neg_integer(), perm_memo()) -> perm_memo().
+store_perm_memo(UserId, Perms, Memo) when map_size(Memo) < ?MAX_PERM_MEMO_ENTRIES ->
+ Memo#{UserId => Perms};
+store_perm_memo(_UserId, _Perms, Memo) ->
+ Memo.
+
+-spec prepend_session(boolean(), session_id(), session_data(), [session_pair()]) ->
+ [session_pair()].
+prepend_session(true, Sid, S, Acc) -> [{Sid, S} | Acc];
+prepend_session(false, _Sid, _S, Acc) -> Acc.
+
-spec filter_sessions_for_manage_channels(
sessions_map(), channel_id(), session_id() | undefined, guild_state()
) -> [session_pair()].
@@ -341,4 +501,182 @@ filter_active_sessions_with_predicate_test() ->
?assertEqual(1, length(Result)),
[{<<"b">>, _}] = Result.
+store_perm_memo_test() ->
+ ?assertEqual(#{7 => 42}, store_perm_memo(7, 42, #{})),
+ ?assertEqual(#{7 => 42}, store_perm_memo(7, 42, #{7 => 42})).
+
+store_perm_memo_bound_test() ->
+ Full = maps:from_list([{I, 0} || I <- lists:seq(1, ?MAX_PERM_MEMO_ENTRIES)]),
+ ?assertEqual(Full, store_perm_memo(0, 1, Full)),
+ ?assertEqual(?MAX_PERM_MEMO_ENTRIES, map_size(store_perm_memo(0, 1, Full))).
+
+memo_member_permissions_hit_test() ->
+ Memo = #{9 => 123},
+ ?assertEqual({123, Memo}, memo_member_permissions(9, 5, #{}, Memo)).
+
+memo_member_permissions_miss_test() ->
+ ?assertEqual({0, #{9 => 0}}, memo_member_permissions(9, 5, #{}, #{})).
+
+-spec reference_session_can_access_message(map(), channel_id(), binary(), guild_state()) ->
+ boolean().
+reference_session_can_access_message(SessionData, ChannelId, MessageId, State) ->
+ case maps:get(user_id, SessionData, undefined) of
+ UserId when is_integer(UserId) ->
+ Perms = guild_permissions:get_member_permissions(UserId, ChannelId, State),
+ guild_permissions:can_access_message_by_permissions(Perms, MessageId, State);
+ _ ->
+ false
+ end.
+
+-spec reference_filter_message_direct(
+ sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state()
+) -> [session_pair()].
+reference_filter_message_direct(Sessions, ChannelId, MessageId, SessionIdOpt, State) ->
+ filter_active_sessions(Sessions, SessionIdOpt, fun(S, _Sid) ->
+ session_can_view_channel(S, ChannelId, State) andalso
+ reference_session_can_access_message(S, ChannelId, MessageId, State)
+ end).
+
+filter_message_memo_matches_direct_test() ->
+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},
+ Owner1 = #{user_id => 1, viewable_channels => #{5 => true}},
+ Owner2 = #{user_id => 1, viewable_channels => #{5 => true}},
+ Other = #{user_id => 2, viewable_channels => #{5 => true}},
+ Sessions = #{<<"a">> => Owner1, <<"b">> => Owner2, <<"c">> => Other},
+ Direct = reference_filter_message_direct(Sessions, 5, <<"1">>, undefined, State),
+ Memoized = filter_message_memo(Sessions, 5, <<"1">>, undefined, State),
+ ?assertEqual(Direct, Memoized),
+ ?assertEqual([<<"a">>, <<"b">>], lists:sort([Sid || {Sid, _} <- Memoized])).
+
+filter_message_memo_collects_visible_sessions_test() ->
+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},
+ Owner1 = #{user_id => 1, viewable_channels => #{5 => true}},
+ Owner2 = #{user_id => 1, viewable_channels => #{5 => true}},
+ Other = #{user_id => 2, viewable_channels => #{5 => true}},
+ Sessions = #{<<"a">> => Owner1, <<"b">> => Owner2, <<"c">> => Other},
+ Memoized = filter_message_memo(Sessions, 5, <<"1">>, undefined, State),
+ ?assertEqual([<<"a">>, <<"b">>], lists:sort([Sid || {Sid, _} <- Memoized])).
+
+filter_message_memo_skips_excluded_test() ->
+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},
+ S1 = #{user_id => 1, viewable_channels => #{5 => true}},
+ S2 = #{user_id => 1, viewable_channels => #{5 => true}, pending_connect => true},
+ Sessions = #{<<"a">> => S1, <<"b">> => S2},
+ ?assertEqual([], filter_message_memo(Sessions, 5, <<"1">>, <<"a">>, State)).
+
+filter_sessions_for_message_test() ->
+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},
+ Sessions = #{<<"a">> => #{user_id => 1, viewable_channels => #{5 => true}}},
+ ?assertEqual(
+ [{<<"a">>, maps:get(<<"a">>, Sessions)}],
+ filter_sessions_for_message(Sessions, 5, <<"1">>, undefined, State)
+ ).
+
+non_negative_count_test() ->
+ ?assertEqual(3, non_negative_count(3)),
+ ?assertEqual(0, non_negative_count(0)),
+ ?assertEqual(0, non_negative_count(-1)),
+ ?assertEqual(0, non_negative_count(undefined)).
+
+active_session_count_test() ->
+ Sessions = #{
+ <<"a">> => #{user_id => 1, pending_connect => true},
+ <<"b">> => #{user_id => 1, pending_connect => false},
+ <<"c">> => #{user_id => 1},
+ <<"d">> => #{user_id => 2}
+ },
+ ?assertEqual(2, active_session_count(1, Sessions)),
+ ?assertEqual(1, active_session_count(2, Sessions)),
+ ?assertEqual(0, active_session_count(3, Sessions)).
+
+session_id_by_ref_uses_index_test() ->
+ Ref = make_ref(),
+ Sessions = #{<<"a">> => #{mref => make_ref()}},
+ State = #{guild_session_refs => #{Ref => <<"a">>}},
+ ?assertEqual(<<"a">>, session_id_by_ref(Ref, Sessions, State)).
+
+session_id_by_ref_scan_fallback_test() ->
+ Ref = make_ref(),
+ Sessions = #{<<"a">> => #{mref => make_ref()}, <<"b">> => #{mref => Ref}},
+ ?assertEqual(<<"b">>, session_id_by_ref(Ref, Sessions, #{})),
+ ?assertEqual(undefined, session_id_by_ref(make_ref(), Sessions, #{})).
+
+pending_session_by_ref_found_test() ->
+ Ref = make_ref(),
+ Session = #{user_id => 1, mref => Ref, pending_connect => true},
+ Sessions = #{<<"a">> => Session},
+ State = #{sessions => Sessions, guild_session_refs => #{Ref => <<"a">>}},
+ ?assertEqual({ok, <<"a">>, Session, Sessions}, pending_session_by_ref(Ref, State)),
+ ?assertEqual(
+ {ok, <<"a">>, Session, Sessions},
+ pending_session_by_ref(Ref, #{sessions => Sessions})
+ ).
+
+pending_session_by_ref_not_pending_test() ->
+ Ref = make_ref(),
+ Session = #{user_id => 1, mref => Ref, pending_connect => false},
+ State = #{sessions => #{<<"a">> => Session}, guild_session_refs => #{Ref => <<"a">>}},
+ ?assertEqual(not_found, pending_session_by_ref(Ref, State)).
+
+pending_session_by_ref_unknown_ref_test() ->
+ Sessions = #{<<"a">> => #{user_id => 1, mref => make_ref(), pending_connect => true}},
+ ?assertEqual(not_found, pending_session_by_ref(make_ref(), #{sessions => Sessions})).
+
+pending_session_by_ref_stale_index_test() ->
+ Ref = make_ref(),
+ State = #{sessions => #{}, guild_session_refs => #{Ref => <<"gone">>}},
+ ?assertEqual(not_found, pending_session_by_ref(Ref, State)).
+
+pending_session_owns_connected_tracking_untracked_test() ->
+ Session = #{user_id => 1, pending_connect => true},
+ Sessions = #{<<"a">> => Session},
+ ?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, #{})).
+
+pending_session_owns_connected_tracking_other_session_owns_test() ->
+ Session = #{user_id => 1, pending_connect => true},
+ Active = #{user_id => 1, pending_connect => false},
+ Sessions = #{<<"a">> => Session, <<"b">> => Active},
+ State = #{user_session_counts => #{1 => 1}},
+ ?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, State)).
+
+pending_session_owns_connected_tracking_true_test() ->
+ Session = #{user_id => 1, pending_connect => true},
+ Sessions = #{<<"a">> => Session},
+ State = #{user_session_counts => #{1 => 1}},
+ ?assertEqual(true, pending_session_owns_connected_tracking(Session, Sessions, State)).
+
+handle_pending_session_down_keeps_tracking_test() ->
+ Ref = make_ref(),
+ Pending = #{session_id => <<"a">>, user_id => 1, mref => Ref, pending_connect => true},
+ Active = #{session_id => <<"b">>, user_id => 1, mref => make_ref()},
+ Sessions = #{<<"a">> => Pending, <<"b">> => Active},
+ State = #{
+ sessions => Sessions,
+ guild_session_refs => #{Ref => <<"a">>},
+ session_connect_pending => #{<<"a">> => #{}},
+ user_session_counts => #{1 => 1},
+ connected_user_ids => sets:from_list([1])
+ },
+ {noreply, NewState} = handle_pending_session_down(<<"a">>, Ref, Sessions, State),
+ ?assertEqual(#{<<"b">> => Active}, maps:get(sessions, NewState)),
+ ?assertEqual(#{}, maps:get(guild_session_refs, NewState)),
+ ?assertEqual(#{}, maps:get(session_connect_pending, NewState)),
+ ?assertEqual(#{1 => 1}, maps:get(user_session_counts, NewState)),
+ ?assertEqual([1], sets:to_list(maps:get(connected_user_ids, NewState))),
+ ?assertEqual(false, maps:is_key(auto_stop_pending, NewState)).
+
+handle_pending_session_down_last_session_test() ->
+ Ref = make_ref(),
+ Pending = #{session_id => <<"a">>, user_id => 1, mref => Ref, pending_connect => true},
+ Sessions = #{<<"a">> => Pending},
+ State = #{
+ sessions => Sessions,
+ guild_session_refs => #{Ref => <<"a">>},
+ disable_auto_stop_on_empty => true
+ },
+ {noreply, NewState} = handle_pending_session_down(<<"a">>, Ref, Sessions, State),
+ ?assertEqual(#{}, maps:get(sessions, NewState)),
+ ?assertEqual(#{}, maps:get(guild_session_refs, NewState)),
+ ?assertEqual(false, maps:is_key(auto_stop_pending, NewState)).
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_subscription_handler.erl b/fluxer_gateway/src/guild/guild_subscription_handler.erl
index 3e0274135..2062a719a 100644
--- a/fluxer_gateway/src/guild/guild_subscription_handler.erl
+++ b/fluxer_gateway/src/guild/guild_subscription_handler.erl
@@ -22,6 +22,8 @@
-define(LAZY_SUBSCRIBE_COALESCE_MS, 100).
-define(MAX_BUFFERED_LAZY_SUBSCRIBE_RANGES, 10).
+-define(ENGINES_KEY, channel_member_list_engines).
+-define(DISPATCHED_KEY, lazy_subscribe_dispatched).
-spec handle_call(term(), gen_server:from(), guild_state()) ->
{reply, term(), guild_state()}.
@@ -47,6 +49,73 @@ handle_info(flush_lazy_subscribe_buffer, State) ->
-spec buffer_lazy_subscribe(map(), guild_state()) -> guild_state().
buffer_lazy_subscribe(Request, State) ->
+ case dispatch_immediately(Request, State) of
+ true -> dispatch_lazy_subscribe_now(Request, State);
+ false -> enqueue_lazy_subscribe(Request, State)
+ end.
+
+-spec dispatch_immediately(map(), guild_state()) -> boolean().
+dispatch_immediately(Request, State) ->
+ maps:get(lazy_subscribe_timer, State, undefined) =:= undefined andalso
+ map_size(maps:get(lazy_subscribe_buffer, State, #{})) =:= 0 andalso
+ engine_already_built(Request, State).
+
+-spec engine_already_built(map(), guild_state()) -> boolean().
+engine_already_built(#{channel_id := ChannelId}, State) ->
+ case guild_member_list:calculate_list_id(ChannelId, State) of
+ ListId when is_binary(ListId) ->
+ maps:is_key(ListId, maps:get(?ENGINES_KEY, State, #{}));
+ _ ->
+ false
+ end;
+engine_already_built(_Request, _State) ->
+ false.
+
+%% A leading-edge dispatch still buffers its request, marked as dispatched, so a second
+%% subscribe for the same key inside the window merges into it exactly as it did before
+%% the leading edge existed. The mark only survives while nothing merges in, because every
+%% merge_lazy_subscribe_request/2 clause returns a map derived from the incoming request.
+-spec dispatch_lazy_subscribe_now(map(), guild_state()) -> guild_state().
+dispatch_lazy_subscribe_now(Request, State) ->
+ try process_lazy_subscribe(Request, State) of
+ NewState -> enqueue_lazy_subscribe(mark_lazy_subscribe_dispatched(Request), NewState)
+ catch
+ Class:Reason:Stack ->
+ log_lazy_subscribe_dispatch_error(Request, Class, Reason, Stack),
+ State
+ end.
+
+-spec mark_lazy_subscribe_dispatched(map()) -> map().
+mark_lazy_subscribe_dispatched(Request) ->
+ Request#{?DISPATCHED_KEY => true}.
+
+-spec log_lazy_subscribe_dispatch_error(map(), atom(), term(), list()) -> ok.
+log_lazy_subscribe_dispatch_error(Request, Class, Reason, Stack) ->
+ logger:warning(
+ "guild_lazy_subscribe_dispatch_failed: session_id=~p channel_id=~p error=~p:~p ~p",
+ [
+ maps:get(session_id, Request, undefined),
+ maps:get(channel_id, Request, undefined),
+ Class,
+ Reason,
+ Stack
+ ]
+ ).
+
+-spec arm_lazy_subscribe_timer(guild_state()) -> guild_state().
+arm_lazy_subscribe_timer(State) ->
+ case maps:get(lazy_subscribe_timer, State, undefined) of
+ undefined ->
+ Ref = erlang:send_after(
+ ?LAZY_SUBSCRIBE_COALESCE_MS, self(), flush_lazy_subscribe_buffer
+ ),
+ State#{lazy_subscribe_timer => Ref};
+ _ ->
+ State
+ end.
+
+-spec enqueue_lazy_subscribe(map(), guild_state()) -> guild_state().
+enqueue_lazy_subscribe(Request, State) ->
#{session_id := SessionId, channel_id := ChannelId} = Request,
BufferKey = {SessionId, ChannelId},
Buffer = maps:get(lazy_subscribe_buffer, State, #{}),
@@ -59,13 +128,8 @@ buffer_lazy_subscribe(Request, State) ->
TimerRef = maps:get(lazy_subscribe_timer, State, undefined),
NewState = State#{lazy_subscribe_buffer => NewBuffer, lazy_subscribe_order => NewOrder},
case TimerRef of
- undefined ->
- Ref = erlang:send_after(
- ?LAZY_SUBSCRIBE_COALESCE_MS, self(), flush_lazy_subscribe_buffer
- ),
- NewState#{lazy_subscribe_timer => Ref};
- _ ->
- NewState
+ undefined -> arm_lazy_subscribe_timer(NewState);
+ _ -> NewState
end.
-spec merge_lazy_subscribe_request(map() | undefined, map()) -> map().
@@ -139,6 +203,8 @@ ordered_lazy_subscribe_keys(State, Buffer) ->
guild_state().
process_buffered_lazy_subscribe(BufferKey, Buffer, State) ->
case maps:find(BufferKey, Buffer) of
+ {ok, #{?DISPATCHED_KEY := true}} ->
+ State;
{ok, Request} ->
process_lazy_subscribe(Request, State);
error ->
@@ -294,13 +360,9 @@ handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) -
FilteredMemberIds = filter_member_ids_for_subscription(
GuildId, SessionUserId, MemberIds, State
),
- OldSubscriptions = guild_subscriptions:get_user_ids_for_session(SessionId, MemberSubs),
- NewMemberSubs = guild_subscriptions:update_subscriptions(
+ {NewMemberSubs, Added, Removed} = guild_subscriptions:update_subscriptions_with_delta(
SessionId, FilteredMemberIds, MemberSubs
),
- NewSubscriptions = guild_subscriptions:get_user_ids_for_session(SessionId, NewMemberSubs),
- Added = sets:to_list(sets:subtract(NewSubscriptions, OldSubscriptions)),
- Removed = sets:to_list(sets:subtract(OldSubscriptions, NewSubscriptions)),
State1 = State#{member_subscriptions => NewMemberSubs},
State2 = handle_added_subscriptions(Added, SessionId, State1),
handle_removed_subscriptions(Removed, State2).
@@ -312,7 +374,7 @@ handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) -
filter_member_ids_for_subscription(_GuildId, undefined, _MemberIds, _State) ->
[];
filter_member_ids_for_subscription(_GuildId, SessionUserId, MemberIds, State) ->
- filter_member_ids_with_mutual_channels(SessionUserId, MemberIds, State).
+ guild_subscription_mutual_channels:filter_member_ids(SessionUserId, MemberIds, State).
-spec handle_added_subscriptions([user_id()], session_id(), guild_state()) -> guild_state().
handle_added_subscriptions(Added, SessionId, State) ->
@@ -333,58 +395,6 @@ handle_removed_subscriptions(Removed, State) ->
Removed
).
--spec filter_member_ids_with_mutual_channels(
- user_id(), [user_id()], guild_state()
-) -> [user_id()].
-filter_member_ids_with_mutual_channels(SessionUserId, MemberIds, State) ->
- SessionMap = get_session_channel_map(SessionUserId, State),
- lists:filtermap(
- fun(MemberId) ->
- has_mutual_channel(
- MemberId, SessionUserId, SessionMap, State
- )
- end,
- MemberIds
- ).
-
--spec get_session_channel_map(user_id(), guild_state()) -> map().
-get_session_channel_map(SessionUserId, State) ->
- case
- guild_visibility_channels:get_cached_viewable_channel_map(
- SessionUserId, State
- )
- of
- undefined ->
- guild_sessions:build_viewable_channel_map(
- guild_visibility:get_user_viewable_channels(
- SessionUserId, State
- )
- );
- M ->
- M
- end.
-
--spec has_mutual_channel(
- term(), user_id(), map(), guild_state()
-) -> {true, user_id()} | false.
-has_mutual_channel(MemberId, SessionUserId, _SessionMap, _State) when
- MemberId =:= SessionUserId; not is_integer(MemberId)
-->
- false;
-has_mutual_channel(MemberId, _SessionUserId, SessionMap, State) ->
- MemberChannels = guild_visibility:get_user_viewable_channels(
- MemberId, State
- ),
- HasMutual = has_shared_channel(MemberChannels, SessionMap),
- case HasMutual of
- true -> {true, MemberId};
- false -> false
- end.
-
--spec has_shared_channel([integer()], map()) -> boolean().
-has_shared_channel(MemberChannels, SessionMap) ->
- lists:any(fun(Ch) -> maps:is_key(Ch, SessionMap) end, MemberChannels).
-
-ifdef(TEST).
-spec disabled_operations_state(integer() | binary()) -> guild_state().
@@ -497,4 +507,138 @@ flush_lazy_subscribe_buffer_clears_state_test() ->
?assertEqual(error, maps:find(lazy_subscribe_order, NewState)),
?assertEqual(error, maps:find(lazy_subscribe_timer, NewState)).
+warm_list_state(ChannelId) ->
+ ListId = integer_to_binary(ChannelId),
+ #{
+ data => #{
+ <<"channel_index">> => #{
+ ChannelId => #{<<"id">> => ListId}
+ }
+ },
+ ?ENGINES_KEY => #{ListId => warm_list_state_engine_ref}
+ }.
+
+engine_already_built_detects_built_engine_test() ->
+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ ?assertEqual(true, engine_already_built(Request, warm_list_state(500))).
+
+engine_already_built_false_without_engine_test() ->
+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ State = maps:put(?ENGINES_KEY, #{}, warm_list_state(500)),
+ ?assertEqual(false, engine_already_built(Request, State)).
+
+engine_already_built_false_for_unknown_channel_test() ->
+ Request = #{session_id => <<"s1">>, channel_id => 999, ranges => [{0, 99}]},
+ ?assertEqual(false, engine_already_built(Request, warm_list_state(500))).
+
+engine_already_built_false_on_bare_state_test() ->
+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ ?assertEqual(false, engine_already_built(Request, #{})).
+
+dispatch_immediately_requires_idle_window_test() ->
+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ Warm = warm_list_state(500),
+ ?assertEqual(true, dispatch_immediately(Request, Warm)),
+ ?assertEqual(
+ false, dispatch_immediately(Request, Warm#{lazy_subscribe_timer => make_ref()})
+ ),
+ ?assertEqual(
+ false,
+ dispatch_immediately(Request, Warm#{
+ lazy_subscribe_buffer => #{{<<"s2">>, 501} => Request}
+ })
+ ).
+
+warm_subscribe_records_the_dispatched_request_test() ->
+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ NewState = buffer_lazy_subscribe(Request, warm_list_state(500)),
+ Buffer = maps:get(lazy_subscribe_buffer, NewState),
+ ?assertEqual(Request#{?DISPATCHED_KEY => true}, maps:get({<<"s1">>, 500}, Buffer)),
+ ?assertEqual([{<<"s1">>, 500}], maps:get(lazy_subscribe_order, NewState)),
+ ?assertNotEqual(undefined, maps:get(lazy_subscribe_timer, NewState, undefined)).
+
+subscribe_behind_a_warm_dispatch_is_coalesced_test() ->
+ Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)),
+ Request2 = #{session_id => <<"s2">>, channel_id => 500, ranges => [{0, 99}]},
+ State2 = buffer_lazy_subscribe(Request2, State1),
+ Buffer = maps:get(lazy_subscribe_buffer, State2),
+ ?assertEqual(Request2, maps:get({<<"s2">>, 500}, Buffer)),
+ ?assertEqual(Request1#{?DISPATCHED_KEY => true}, maps:get({<<"s1">>, 500}, Buffer)),
+ ?assertEqual(2, map_size(Buffer)).
+
+%% Reference oracle: what enqueue_lazy_subscribe/2 buffered for a repeated
+%% {session, channel} before the leading-edge dispatch existed.
+reference_coalesced_request(Request1, Request2) ->
+ merge_lazy_subscribe_request(Request1, Request2).
+
+repeated_warm_subscribe_merges_like_the_pre_leading_edge_path_test() ->
+ Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ Request2 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{100, 199}]},
+ State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)),
+ State2 = buffer_lazy_subscribe(Request2, State1),
+ Buffer = maps:get(lazy_subscribe_buffer, State2),
+ Merged = maps:get({<<"s1">>, 500}, Buffer),
+ ?assertEqual(1, map_size(Buffer)),
+ ?assertEqual(reference_coalesced_request(Request1, Request2), Merged),
+ ?assertNot(maps:is_key(?DISPATCHED_KEY, Merged)),
+ ?assertEqual([{<<"s1">>, 500}], maps:get(lazy_subscribe_order, State2)).
+
+repeated_warm_subscribe_with_empty_ranges_replaces_test() ->
+ Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ Request2 = #{session_id => <<"s1">>, channel_id => 500, ranges => []},
+ State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)),
+ State2 = buffer_lazy_subscribe(Request2, State1),
+ Merged = maps:get({<<"s1">>, 500}, maps:get(lazy_subscribe_buffer, State2)),
+ ?assertEqual(reference_coalesced_request(Request1, Request2), Merged),
+ ?assertNot(maps:is_key(?DISPATCHED_KEY, Merged)).
+
+dispatched_entry_state() ->
+ Key = {<<"s1">>, 500},
+ Dispatched = #{session_id => <<"s1">>, channel_id => 500, ?DISPATCHED_KEY => true},
+ State = #{
+ lazy_subscribe_buffer => #{Key => Dispatched},
+ lazy_subscribe_order => [Key],
+ lazy_subscribe_timer => make_ref()
+ },
+ {Key, Dispatched, State}.
+
+flush_skips_an_already_dispatched_entry_test() ->
+ {_Key, _Dispatched, State} = dispatched_entry_state(),
+ NewState = flush_lazy_subscribe_buffer(State),
+ ?assertEqual(error, maps:find(lazy_subscribe_buffer, NewState)),
+ ?assertEqual(error, maps:find(lazy_subscribe_order, NewState)),
+ ?assertEqual(error, maps:find(lazy_subscribe_timer, NewState)).
+
+flush_processes_the_same_entry_once_unmarked_test() ->
+ {Key, Dispatched, State} = dispatched_entry_state(),
+ Unmarked = maps:remove(?DISPATCHED_KEY, Dispatched),
+ ?assertError(
+ {badmatch, _},
+ flush_lazy_subscribe_buffer(State#{lazy_subscribe_buffer => #{Key => Unmarked}})
+ ).
+
+failed_warm_dispatch_is_not_re_enqueued_test() ->
+ Warm = warm_list_state(500),
+ Broken = #{session_id => <<"s1">>, channel_id => 500},
+ ?assertEqual(Warm, buffer_lazy_subscribe(Broken, Warm)).
+
+failed_warm_dispatch_does_not_escape_handle_call_test() ->
+ Warm = warm_list_state(500),
+ Broken = #{session_id => <<"s1">>, channel_id => 500},
+ From = {self(), make_ref()},
+ ?assertEqual({reply, ok, Warm}, handle_call({lazy_subscribe, Broken}, From, Warm)).
+
+cold_list_still_uses_the_coalesce_buffer_test() ->
+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},
+ State = maps:put(?ENGINES_KEY, #{}, warm_list_state(500)),
+ NewState = buffer_lazy_subscribe(Request, State),
+ Buffer = maps:get(lazy_subscribe_buffer, NewState),
+ ?assertEqual(Request, maps:get({<<"s1">>, 500}, Buffer)).
+
+arm_lazy_subscribe_timer_is_idempotent_test() ->
+ Ref = make_ref(),
+ State = #{lazy_subscribe_timer => Ref},
+ ?assertEqual(Ref, maps:get(lazy_subscribe_timer, arm_lazy_subscribe_timer(State))).
+
-endif.
diff --git a/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl
new file mode 100644
index 000000000..404bdf93f
--- /dev/null
+++ b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl
@@ -0,0 +1,287 @@
+%% SPDX-License-Identifier: AGPL-3.0-or-later
+
+-module(guild_subscription_mutual_channels).
+-typing([eqwalizer]).
+
+-export([filter_member_ids/3]).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+-endif.
+
+-type guild_state() :: map().
+-type user_id() :: integer().
+-type memo() :: #{exceptions := sets:set(user_id()), cache := #{term() => boolean()}}.
+-export_type([guild_state/0, user_id/0]).
+
+-spec filter_member_ids(user_id(), [user_id()], guild_state()) -> [user_id()].
+filter_member_ids(_SessionUserId, [], _State) ->
+ [];
+filter_member_ids(SessionUserId, MemberIds, State) ->
+ SessionMap = session_channel_map(SessionUserId, State),
+ {Kept, _Memo} = lists:foldl(
+ fun(MemberId, Acc) ->
+ keep_member_with_mutual_channel(MemberId, SessionUserId, SessionMap, State, Acc)
+ end,
+ {[], new_memo(State)},
+ MemberIds
+ ),
+ lists:reverse(Kept).
+
+-spec session_channel_map(user_id(), guild_state()) -> map().
+session_channel_map(SessionUserId, State) ->
+ case guild_visibility_channels:get_cached_viewable_channel_map(SessionUserId, State) of
+ undefined ->
+ guild_sessions:build_viewable_channel_map(
+ guild_visibility:get_user_viewable_channels(SessionUserId, State)
+ );
+ Map ->
+ Map
+ end.
+
+-spec keep_member_with_mutual_channel(
+ term(), user_id(), map(), guild_state(), {[user_id()], memo()}
+) -> {[user_id()], memo()}.
+keep_member_with_mutual_channel(MemberId, SessionUserId, _SessionMap, _State, Acc) when
+ MemberId =:= SessionUserId; not is_integer(MemberId)
+->
+ Acc;
+keep_member_with_mutual_channel(MemberId, _SessionUserId, SessionMap, State, {Kept, Memo}) ->
+ case memoised_has_mutual_channel(MemberId, SessionMap, State, Memo) of
+ {true, Memo1} -> {[MemberId | Kept], Memo1};
+ {false, Memo1} -> {Kept, Memo1}
+ end.
+
+%% get_user_viewable_channels/2 reads nothing from a member except its raw roles
+%% term, so two members holding the same term see the same channels and give the same
+%% answer against a fixed session map. The user-specific inputs are the guild owner,
+%% virtual channel access and user-type permission overwrites, and every user named
+%% by one of those bypasses the memo.
+-spec new_memo(guild_state()) -> memo().
+new_memo(State) ->
+ #{exceptions => exceptions(State), cache => #{}}.
+
+-spec memoised_has_mutual_channel(user_id(), map(), guild_state(), memo()) ->
+ {boolean(), memo()}.
+memoised_has_mutual_channel(MemberId, SessionMap, State, Memo) ->
+ #{exceptions := Exceptions, cache := Cache} = Memo,
+ case memo_key(MemberId, Exceptions, State) of
+ bypass ->
+ {has_mutual_channel(MemberId, SessionMap, State), Memo};
+ {ok, RawRoles} ->
+ cached_has_mutual_channel(MemberId, RawRoles, SessionMap, State, Cache, Memo)
+ end.
+
+-spec cached_has_mutual_channel(
+ user_id(), term(), map(), guild_state(), #{term() => boolean()}, memo()
+) -> {boolean(), memo()}.
+cached_has_mutual_channel(MemberId, RawRoles, SessionMap, State, Cache, Memo) ->
+ case maps:find(RawRoles, Cache) of
+ {ok, HasMutual} ->
+ {HasMutual, Memo};
+ error ->
+ HasMutual = has_mutual_channel(MemberId, SessionMap, State),
+ {HasMutual, Memo#{cache := Cache#{RawRoles => HasMutual}}}
+ end.
+
+-spec memo_key(user_id(), sets:set(user_id()), guild_state()) -> {ok, term()} | bypass.
+memo_key(MemberId, Exceptions, State) ->
+ case sets:is_element(MemberId, Exceptions) of
+ true -> bypass;
+ false -> member_roles_key(guild_permissions:find_member_by_user_id(MemberId, State))
+ end.
+
+-spec member_roles_key(map() | undefined) -> {ok, term()} | bypass.
+member_roles_key(Member) when is_map(Member) ->
+ {ok, maps:get(<<"roles">>, Member, [])};
+member_roles_key(_Member) ->
+ bypass.
+
+-spec exceptions(guild_state()) -> sets:set(user_id()).
+exceptions(State) ->
+ Data = map_utils:ensure_map(map_utils:get_safe(State, data, #{})),
+ Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})),
+ Owner = owner_ids(maps:get(<<"owner_id">>, Guild, undefined)),
+ Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))),
+ sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(Data)).
+
+%% parse_maybe/1, not parse_optional/1: an id this rejects can never match a user in
+%% a permission check either, so leaving it out of the exception set keeps the memo
+%% exact while keeping the set itself total.
+-spec owner_ids(term()) -> [user_id()].
+owner_ids(OwnerIdRaw) ->
+ case snowflake_id:parse_maybe(OwnerIdRaw) of
+ OwnerId when is_integer(OwnerId) -> [OwnerId];
+ _ -> []
+ end.
+
+%% Every overwrite target on every channel, read from both places a permission check
+%% can take them from, and role targets alongside user targets. Role ids can never
+%% collide with user ids, so the superset only costs a few members their memo entry.
+-spec overwrite_target_ids(map()) -> [user_id()].
+overwrite_target_ids(Data) ->
+ Cached = maps:fold(
+ fun(_ChannelId, Entries, Acc) -> cached_target_ids(Entries, Acc) end,
+ [],
+ map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{}))
+ ),
+ maps:fold(
+ fun(_ChannelId, Channel, Acc) -> channel_target_ids(Channel, Acc) end,
+ Cached,
+ guild_data_index:channel_index(Data)
+ ).
+
+-spec cached_target_ids(term(), [user_id()]) -> [user_id()].
+cached_target_ids(Entries, Acc) when is_list(Entries) ->
+ [Id || {Id, _Type, _Allow, _Deny} <- Entries, is_integer(Id)] ++ Acc;
+cached_target_ids(_Entries, Acc) ->
+ Acc.
+
+-spec channel_target_ids(term(), [user_id()]) -> [user_id()].
+channel_target_ids(Channel, Acc) when is_map(Channel) ->
+ lists:foldl(
+ fun overwrite_target_id/2,
+ Acc,
+ map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, []))
+ );
+channel_target_ids(_Channel, Acc) ->
+ Acc.
+
+-spec overwrite_target_id(term(), [user_id()]) -> [user_id()].
+overwrite_target_id(Overwrite, Acc) when is_map(Overwrite) ->
+ case snowflake_id:parse_maybe(maps:get(<<"id">>, Overwrite, undefined)) of
+ Id when is_integer(Id) -> [Id | Acc];
+ _ -> Acc
+ end;
+overwrite_target_id(_Overwrite, Acc) ->
+ Acc.
+
+-spec has_mutual_channel(user_id(), map(), guild_state()) -> boolean().
+has_mutual_channel(MemberId, SessionMap, State) ->
+ MemberChannels = guild_visibility:get_user_viewable_channels(MemberId, State),
+ has_shared_channel(MemberChannels, SessionMap).
+
+-spec has_shared_channel([integer()], map()) -> boolean().
+has_shared_channel(MemberChannels, SessionMap) ->
+ lists:any(fun(Ch) -> maps:is_key(Ch, SessionMap) end, MemberChannels).
+
+-ifdef(TEST).
+
+-define(GUILD_ID, 42).
+-define(VIEWER_ROLE, 1000).
+-define(OTHER_ROLE, 2000).
+
+test_role(RoleId) ->
+ #{<<"id">> => integer_to_binary(RoleId), <<"permissions">> => <<"0">>}.
+
+test_member(UserId, RoleIds) ->
+ #{
+ <<"user">> => #{<<"id">> => integer_to_binary(UserId)},
+ <<"roles">> => [integer_to_binary(RoleId) || RoleId <- RoleIds]
+ }.
+
+test_overwrite(TargetId, Type) ->
+ #{
+ <<"id">> => integer_to_binary(TargetId),
+ <<"type">> => Type,
+ <<"allow">> => integer_to_binary(constants:view_channel_permission()),
+ <<"deny">> => <<"0">>
+ }.
+
+test_channel(ChannelId, Overwrites) ->
+ #{
+ <<"id">> => integer_to_binary(ChannelId),
+ <<"type">> => 0,
+ <<"permission_overwrites">> => Overwrites
+ }.
+
+%% Session user 10 and members 20 and 21 reach channel 500 through the viewer role.
+%% Members 30, 31, 99 and 4242 all carry the same roles term and reach only channel
+%% 600 by role, but 99 holds virtual access to 500 and 4242 is a user-overwrite
+%% target on 500, so both must still come out true. Member 7 owns the guild.
+test_state() ->
+ Channels = [
+ test_channel(500, [test_overwrite(?VIEWER_ROLE, 0), test_overwrite(4242, 1)]),
+ test_channel(600, [test_overwrite(?OTHER_ROLE, 0)])
+ ],
+ Roles = [test_role(?GUILD_ID), test_role(?VIEWER_ROLE), test_role(?OTHER_ROLE)],
+ Members = maps:from_list(
+ [{7, test_member(7, [])}] ++
+ [{Id, test_member(Id, [?VIEWER_ROLE])} || Id <- [10, 20, 21]] ++
+ [{Id, test_member(Id, [?OTHER_ROLE])} || Id <- [30, 31, 99, 4242]]
+ ),
+ #{
+ id => ?GUILD_ID,
+ virtual_channel_access => #{99 => sets:from_list([500])},
+ data => #{
+ <<"guild">> => #{<<"owner_id">> => <<"7">>},
+ <<"roles">> => Roles,
+ <<"members">> => Members,
+ <<"channels">> => Channels,
+ <<"channel_index">> => guild_data_index:build_id_index(Channels)
+ }
+ }.
+
+%% filter_member_ids/3 as it read before the memo: every candidate materialises its
+%% own complete viewable channel list.
+reference_filter_member_ids(SessionUserId, MemberIds, State) ->
+ SessionMap = session_channel_map(SessionUserId, State),
+ lists:filtermap(
+ fun(MemberId) -> reference_keep(MemberId, SessionUserId, SessionMap, State) end,
+ MemberIds
+ ).
+
+reference_keep(MemberId, SessionUserId, _SessionMap, _State) when
+ MemberId =:= SessionUserId; not is_integer(MemberId)
+->
+ false;
+reference_keep(MemberId, _SessionUserId, SessionMap, State) ->
+ case has_mutual_channel(MemberId, SessionMap, State) of
+ true -> {true, MemberId};
+ false -> false
+ end.
+
+candidate_ids() ->
+ [20, 30, 21, 31, 10, 7, 99, 4242, 777, 20].
+
+exceptions_cover_owner_virtual_and_overwrite_targets_test() ->
+ Exceptions = exceptions(test_state()),
+ ?assertEqual([true, true, true], [
+ sets:is_element(Id, Exceptions)
+ || Id <- [7, 99, 4242]
+ ]),
+ ?assertEqual([false, false], [sets:is_element(Id, Exceptions) || Id <- [20, 30]]).
+
+filter_member_ids_matches_unmemoised_test() ->
+ State = test_state(),
+ Ids = candidate_ids(),
+ ?assertEqual([20, 21, 7, 99, 4242, 20], filter_member_ids(10, Ids, State)),
+ ?assertEqual(
+ [reference_filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]],
+ [filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]]
+ ).
+
+memo_holds_one_entry_per_roles_term_test() ->
+ State = test_state(),
+ SessionMap = session_channel_map(10, State),
+ {First, Memo1} = memoised_has_mutual_channel(20, SessionMap, State, new_memo(State)),
+ {Second, Memo2} = memoised_has_mutual_channel(21, SessionMap, State, Memo1),
+ ?assert(First),
+ ?assertEqual(First, Second),
+ ?assertEqual([1, 1], [map_size(maps:get(cache, M)) || M <- [Memo1, Memo2]]).
+
+exception_user_is_not_answered_from_its_role_set_test() ->
+ State = test_state(),
+ SessionMap = session_channel_map(10, State),
+ {false, Memo} = memoised_has_mutual_channel(30, SessionMap, State, new_memo(State)),
+ ?assertEqual(
+ [{true, Memo}, {true, Memo}, {true, Memo}],
+ [memoised_has_mutual_channel(Id, SessionMap, State, Memo) || Id <- [7, 99, 4242]]
+ ).
+
+non_member_candidate_is_dropped_test() ->
+ State = test_state(),
+ ?assertEqual([], filter_member_ids(10, [777], State)),
+ ?assertEqual([], reference_filter_member_ids(10, [777], State)).
+
+-endif.
diff --git a/fluxer_gateway/src/guild/guild_subscriptions.erl b/fluxer_gateway/src/guild/guild_subscriptions.erl
index 50c5b8850..2ca02b260 100644
--- a/fluxer_gateway/src/guild/guild_subscriptions.erl
+++ b/fluxer_gateway/src/guild/guild_subscriptions.erl
@@ -9,6 +9,7 @@
unsubscribe/3,
unsubscribe_session/2,
update_subscriptions/3,
+ update_subscriptions_with_delta/3,
get_subscribed_sessions/2,
is_subscribed/3,
get_user_ids_for_session/2
@@ -52,30 +53,44 @@ remove_session_subscriber(UserId, SessionId, Subscribers, State) ->
_ -> State#{UserId => NewSubscribers}
end.
+%% Every subscriber set in the map is non-empty: this module, guild_maintenance and
+%% guild_presence_sync all drop a user key once its last subscriber goes. Entries the
+%% session is absent from are therefore already correct and are left as they are
+%% instead of being rebuilt into a fresh map.
-spec unsubscribe_session(session_id(), subscription_state()) -> subscription_state().
unsubscribe_session(SessionId, State) ->
maps:fold(
fun(UserId, Subscribers, Acc) ->
- remove_session_from_subs(UserId, SessionId, Subscribers, Acc)
+ drop_session_subscriber(UserId, SessionId, Subscribers, Acc)
end,
- #{},
+ State,
State
).
--spec remove_session_from_subs(
+-spec drop_session_subscriber(
user_id(), session_id(), sets:set(session_id()), subscription_state()
) ->
subscription_state().
-remove_session_from_subs(UserId, SessionId, Subscribers, Acc) ->
- NewSubscribers = sets:del_element(SessionId, Subscribers),
- case sets:size(NewSubscribers) of
- 0 -> Acc;
- _ -> Acc#{UserId => NewSubscribers}
+drop_session_subscriber(UserId, SessionId, Subscribers, Acc) ->
+ case sets:is_element(SessionId, Subscribers) of
+ true -> remove_session_subscriber(UserId, SessionId, Subscribers, Acc);
+ false -> Acc
end.
-spec update_subscriptions(session_id(), [user_id()], subscription_state()) ->
subscription_state().
update_subscriptions(SessionId, NewMemberIds, State) ->
+ {NewState, _Added, _Removed} = update_subscriptions_with_delta(
+ SessionId, NewMemberIds, State
+ ),
+ NewState.
+
+%% After both folds the session subscribes to exactly NewMemberIds, so the added and
+%% removed sets are the ToAdd and ToRemove already computed here. Deriving them from
+%% the resulting map instead would cost two more full folds of the subscription map.
+-spec update_subscriptions_with_delta(session_id(), [user_id()], subscription_state()) ->
+ {subscription_state(), [user_id()], [user_id()]}.
+update_subscriptions_with_delta(SessionId, NewMemberIds, State) ->
CurrentlySubscribed = get_user_ids_for_session(SessionId, State),
NewMemberIdSet = sets:from_list(NewMemberIds),
ToRemove = sets:subtract(CurrentlySubscribed, NewMemberIdSet),
@@ -87,13 +102,14 @@ update_subscriptions(SessionId, NewMemberIds, State) ->
State,
ToRemove
),
- sets:fold(
+ State2 = sets:fold(
fun(UserId, AccState) ->
subscribe(SessionId, UserId, AccState)
end,
State1,
ToAdd
- ).
+ ),
+ {State2, sets:to_list(ToAdd), sets:to_list(ToRemove)}.
-spec get_subscribed_sessions(user_id(), subscription_state()) -> [session_id()].
get_subscribed_sessions(UserId, State) ->
@@ -193,4 +209,83 @@ get_user_ids_for_session_test() ->
UserIds = get_user_ids_for_session(<<"session1">>, State2),
?assertEqual([100, 200], lists:sort(sets:to_list(UserIds))).
+%% The subscription map as unsubscribe_session/2 built it before it stopped
+%% rebuilding untouched entries.
+reference_unsubscribe_session(SessionId, State) ->
+ maps:fold(
+ fun(UserId, Subscribers, Acc) ->
+ reference_drop_subscriber(UserId, sets:del_element(SessionId, Subscribers), Acc)
+ end,
+ #{},
+ State
+ ).
+
+reference_drop_subscriber(UserId, NewSubscribers, Acc) ->
+ case sets:size(NewSubscribers) of
+ 0 -> Acc;
+ _ -> Acc#{UserId => NewSubscribers}
+ end.
+
+%% The state and delta handle_update_member_subscriptions_local/4 derived before it
+%% stopped re-folding the map to rediscover what changed.
+reference_update_with_delta(SessionId, NewMemberIds, State) ->
+ Old = get_user_ids_for_session(SessionId, State),
+ NewState = update_subscriptions(SessionId, NewMemberIds, State),
+ New = get_user_ids_for_session(SessionId, NewState),
+ {
+ NewState,
+ sets:to_list(sets:subtract(New, Old)),
+ sets:to_list(sets:subtract(Old, New))
+ }.
+
+populate(Pairs) ->
+ lists:foldl(
+ fun({SessionId, UserId}, Acc) -> subscribe(SessionId, UserId, Acc) end,
+ init_state(),
+ Pairs
+ ).
+
+delta_fixture_states() ->
+ [
+ init_state(),
+ populate([{<<"s1">>, 100}]),
+ populate([{<<"s1">>, 100}, {<<"s1">>, 200}, {<<"s2">>, 200}, {<<"s2">>, 300}]),
+ populate([{<<"s2">>, N} || N <- lists:seq(1, 64)]),
+ populate(
+ [{<<"s1">>, N} || N <- lists:seq(1, 200)] ++
+ [{<<"s2">>, N} || N <- lists:seq(150, 400)]
+ )
+ ].
+
+delta_fixture_cases() ->
+ Inputs = [[], [100], [200, 300], [300, 200], [100, 100, 200], lists:seq(100, 500)],
+ [{State, Ids} || State <- delta_fixture_states(), Ids <- Inputs].
+
+unsubscribe_session_matches_rebuilt_map_test() ->
+ Cases = [
+ {State, SessionId}
+ || State <- delta_fixture_states(),
+ SessionId <- [<<"s1">>, <<"s2">>, <<"never_subscribed">>]
+ ],
+ ?assertEqual(
+ [reference_unsubscribe_session(S, M) || {M, S} <- Cases],
+ [unsubscribe_session(S, M) || {M, S} <- Cases]
+ ).
+
+update_subscriptions_with_delta_matches_refold_test() ->
+ Cases = delta_fixture_cases(),
+ ?assertEqual(
+ [reference_update_with_delta(<<"s1">>, Ids, M) || {M, Ids} <- Cases],
+ [update_subscriptions_with_delta(<<"s1">>, Ids, M) || {M, Ids} <- Cases]
+ ).
+
+update_subscriptions_with_delta_leaves_other_sessions_alone_test() ->
+ State0 = subscribe(<<"s2">>, 100, subscribe(<<"s1">>, 100, init_state())),
+ {NewState, Added, Removed} = update_subscriptions_with_delta(<<"s1">>, [200], State0),
+ ?assertEqual([200], Added),
+ ?assertEqual([100], Removed),
+ ?assert(is_subscribed(<<"s2">>, 100, NewState)),
+ ?assertNot(is_subscribed(<<"s1">>, 100, NewState)),
+ ?assert(is_subscribed(<<"s1">>, 200, NewState)).
+
-endif.
diff --git a/fluxer_gateway/src/guild/voice/guild_voice.erl b/fluxer_gateway/src/guild/voice/guild_voice.erl
index 75992f6c5..e080649a8 100644
--- a/fluxer_gateway/src/guild/voice/guild_voice.erl
+++ b/fluxer_gateway/src/guild/voice/guild_voice.erl
@@ -112,7 +112,7 @@ switch_voice_region(GuildId, ChannelId, GuildPid) ->
-spec handle_virtual_channel_access_for_move(integer(), integer(), map(), pid()) -> ok.
handle_virtual_channel_access_for_move(UserId, ChannelId, _ConnectionsToMove, GuildPid) ->
- case gen_server:call(GuildPid, {get_sessions}, 10000) of
+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) of
State when is_map(State) ->
maybe_grant_move_access(UserId, ChannelId, State, GuildPid);
_ ->
@@ -159,9 +159,9 @@ cleanup_virtual_access_on_disconnect(UserId, GuildPid) ->
-spec resolve_guild_id_from_pid(pid()) -> integer() | undefined.
resolve_guild_id_from_pid(GuildPid) ->
- try gen_server:call(GuildPid, {get_sessions}, 5000) of
- State when is_map(State) ->
- maps:get(id, State, undefined);
+ try guild_voice_server_state:guild_id_call(GuildPid, 5000) of
+ GuildId when is_integer(GuildId) ->
+ GuildId;
_ ->
undefined
catch
diff --git a/fluxer_gateway/src/guild/voice/guild_voice_move.erl b/fluxer_gateway/src/guild/voice/guild_voice_move.erl
index a4d620cbf..79b602e46 100644
--- a/fluxer_gateway/src/guild/voice/guild_voice_move.erl
+++ b/fluxer_gateway/src/guild/voice/guild_voice_move.erl
@@ -176,7 +176,7 @@ send_voice_server_update_for_move(
integer(), integer(), integer(), binary(), binary() | null, pid()
) -> ok.
do_send_voice_server_update(GuildId, ChannelId, UserId, SessionId, OldConnectionId, GuildPid) ->
- try gen_server:call(GuildPid, {get_sessions}, 10000) of
+ try guild_voice_server_state:guild_state_call(GuildPid, 10000) of
State when is_map(State) ->
request_and_broadcast_token(
GuildId, ChannelId, UserId, SessionId, OldConnectionId, State
diff --git a/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl b/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl
index 8789c59d4..a64145252 100644
--- a/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl
+++ b/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl
@@ -256,7 +256,7 @@ member_user_id(Member) ->
-spec fetch_state_and_send(integer(), integer(), map(), pid(), integer(), map()) -> ok.
fetch_state_and_send(GuildId, ChannelId, SessionInfo, GuildPid, UserId, Member) ->
- try gen_server:call(GuildPid, {get_sessions}, 10000) of
+ try guild_voice_server_state:guild_state_call(GuildPid, 10000) of
StateData when is_map(StateData) ->
request_and_broadcast(
GuildId, ChannelId, SessionInfo, GuildPid, UserId, Member, StateData
diff --git a/fluxer_gateway/src/guild/voice/guild_voice_region.erl b/fluxer_gateway/src/guild/voice/guild_voice_region.erl
index fc9b2ef9d..5f6deb9ad 100644
--- a/fluxer_gateway/src/guild/voice/guild_voice_region.erl
+++ b/fluxer_gateway/src/guild/voice/guild_voice_region.erl
@@ -40,7 +40,7 @@ switch_voice_region_channel_reply(Channel, State) ->
-spec switch_voice_region(integer(), integer(), pid()) -> ok.
switch_voice_region(GuildId, ChannelId, GuildPid) ->
- case gen_server:call(GuildPid, {get_sessions}, 10000) of
+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) of
State when is_map(State) ->
do_switch_voice_region(GuildId, ChannelId, GuildPid, State);
_ ->
@@ -53,7 +53,7 @@ do_switch_voice_region(GuildId, ChannelId, GuildPid, State) ->
UsersInChannel = collect_users_in_channel(VoiceStates, ChannelId),
lists:foreach(
fun(UserInfo) ->
- maybe_send_region_switch_update(GuildId, ChannelId, GuildPid, UserInfo)
+ maybe_send_region_switch_update(GuildId, ChannelId, GuildPid, State, UserInfo)
end,
UsersInChannel
).
@@ -70,17 +70,28 @@ collect_users_in_channel(VoiceStates, ChannelId) ->
).
-spec maybe_send_region_switch_update(
- integer(), integer(), pid(), {integer(), binary() | undefined, binary(), voice_state()}
+ integer(),
+ integer(),
+ pid(),
+ map(),
+ {integer(), binary() | undefined, binary(), voice_state()}
) -> ok.
maybe_send_region_switch_update(
- _GuildId, _ChannelId, _GuildPid, {_UserId, undefined, _ConnId, _VS}
+ _GuildId, _ChannelId, _GuildPid, _State, {_UserId, undefined, _ConnId, _VS}
) ->
ok;
maybe_send_region_switch_update(
- GuildId, ChannelId, GuildPid, {UserId, SessionId, ExistingConnectionId, VoiceState}
+ GuildId, ChannelId, GuildPid, State, {UserId, SessionId, ExistingConnectionId, VoiceState}
) ->
- send_voice_server_update_for_region_switch(
- GuildId, ChannelId, UserId, SessionId, ExistingConnectionId, VoiceState, GuildPid
+ request_and_broadcast_region_switch(
+ GuildId,
+ ChannelId,
+ UserId,
+ SessionId,
+ ExistingConnectionId,
+ VoiceState,
+ GuildPid,
+ State
).
-spec collect_user_in_channel(binary(), voice_state(), integer(), list()) -> list().
@@ -100,28 +111,6 @@ collect_user_voice_state(ConnectionId, VoiceState, Acc) ->
[{UserId, SessionId, ConnectionId, VoiceState} | Acc]
end.
--spec send_voice_server_update_for_region_switch(
- integer(), integer(), integer(), binary(), binary(), voice_state(), pid()
-) -> ok.
-send_voice_server_update_for_region_switch(
- GuildId, ChannelId, UserId, SessionId, ExistingConnectionId, ExistingVoiceState, GuildPid
-) ->
- case gen_server:call(GuildPid, {get_sessions}, 10000) of
- State when is_map(State) ->
- request_and_broadcast_region_switch(
- GuildId,
- ChannelId,
- UserId,
- SessionId,
- ExistingConnectionId,
- ExistingVoiceState,
- GuildPid,
- State
- );
- _ ->
- ok
- end.
-
-spec request_and_broadcast_region_switch(
integer(),
integer(),
@@ -297,6 +286,201 @@ collect_users_in_channel_test() ->
?assertEqual(<<"sess1">>, SessionId),
?assertEqual(<<"conn1">>, ConnectionId).
+switch_voice_region_reads_guild_state_once_test() ->
+ with_region_stubs(fun() ->
+ {Calls, _Dispatches, _Pending} = run_region_switch(fun switch_voice_region/3),
+ ?assertEqual(1, Calls)
+ end).
+
+switch_voice_region_matches_per_user_refetch_reference_test() ->
+ with_region_stubs(fun region_reference_equivalence_scenario/0).
+
+region_reference_equivalence_scenario() ->
+ {RefCalls, RefDispatches, RefPending} = run_region_switch(
+ fun reference_switch_voice_region/3
+ ),
+ {NewCalls, NewDispatches, NewPending} = run_region_switch(fun switch_voice_region/3),
+ ?assertEqual(3, RefCalls),
+ ?assertEqual(1, NewCalls),
+ ?assertEqual(2, length(RefDispatches)),
+ ?assertEqual(RefDispatches, NewDispatches),
+ ?assertEqual(RefPending, NewPending).
+
+switch_voice_region_skips_users_without_a_session_id_test() ->
+ with_region_stubs(fun region_missing_session_id_scenario/0).
+
+region_missing_session_id_scenario() ->
+ Anonymous = maps:remove(
+ <<"session_id">>, region_voice_state(<<"conn-c">>, 10, <<"sess1">>)
+ ),
+ State = region_state(),
+ Stateless = State#{voice_states => #{<<"conn-c">> => Anonymous}},
+ {Calls, Dispatches, Pending} = run_region_switch_with(
+ fun switch_voice_region/3, Stateless
+ ),
+ ?assertEqual(1, Calls),
+ ?assertEqual([], Dispatches),
+ ?assertEqual([], Pending).
+
+%% The pre-change implementation, kept as a live oracle: it refetched the guild state once per
+%% user in the channel instead of reusing the one the enumeration already fetched.
+reference_switch_voice_region(GuildId, ChannelId, GuildPid) ->
+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) of
+ State when is_map(State) ->
+ reference_do_switch_voice_region(GuildId, ChannelId, GuildPid, State);
+ _ ->
+ ok
+ end.
+
+reference_do_switch_voice_region(GuildId, ChannelId, GuildPid, State) ->
+ VoiceStates = voice_state_utils:voice_states(State),
+ UsersInChannel = collect_users_in_channel(VoiceStates, ChannelId),
+ lists:foreach(
+ fun(UserInfo) -> reference_send_update(GuildId, ChannelId, GuildPid, UserInfo) end,
+ UsersInChannel
+ ).
+
+reference_send_update(_GuildId, _ChannelId, _GuildPid, {_UserId, undefined, _ConnId, _VS}) ->
+ ok;
+reference_send_update(GuildId, ChannelId, GuildPid, {UserId, SessionId, ConnId, VoiceState}) ->
+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) of
+ State when is_map(State) ->
+ request_and_broadcast_region_switch(
+ GuildId, ChannelId, UserId, SessionId, ConnId, VoiceState, GuildPid, State
+ );
+ _ ->
+ ok
+ end.
+
+run_region_switch(Fun) ->
+ run_region_switch_with(Fun, region_state()).
+
+run_region_switch_with(Fun, State) ->
+ drain_region_dispatches(),
+ drain_region_pending(),
+ erlang:put(region_token_index, 0),
+ GuildPid = spawn_region_guild(State),
+ try
+ Fun(999, 200, GuildPid),
+ {region_guild_calls(GuildPid), drain_region_dispatches(), drain_region_pending()}
+ after
+ GuildPid ! stop
+ end.
+
+with_region_stubs(Fun) ->
+ meck:new(rpc_client, [passthrough, no_link, non_strict]),
+ meck:expect(rpc_client, call, fun(_Request) -> {ok, next_region_token()} end),
+ meck:new(guild_voice_server, [passthrough, no_link]),
+ meck:expect(guild_voice_server, resolve, fun(_GuildId, GuildPid) -> GuildPid end),
+ try
+ Fun()
+ after
+ meck:unload(guild_voice_server),
+ meck:unload(rpc_client)
+ end.
+
+next_region_token() ->
+ Index = erlang:get(region_token_index) + 1,
+ erlang:put(region_token_index, Index),
+ #{
+ <<"token">> => <<"tok-", (integer_to_binary(Index))/binary>>,
+ <<"endpoint">> => <<"wss://voice.example">>,
+ <<"connectionId">> => <<"new-conn-", (integer_to_binary(Index))/binary>>,
+ <<"regionId">> => <<"us-east">>,
+ <<"serverId">> => <<"voice-1">>
+ }.
+
+spawn_region_guild(State) ->
+ Owner = self(),
+ spawn(fun() -> region_guild_loop(State, Owner, 0) end).
+
+region_guild_loop(State, Owner, Calls) ->
+ receive
+ {'$gen_call', From, {get_voice_guild_state}} ->
+ gen_server:reply(From, State),
+ region_guild_loop(State, Owner, Calls + 1);
+ {'$gen_call', From, {store_pending_connection, ConnId, Meta}} ->
+ Owner ! {region_pending, ConnId, Meta},
+ gen_server:reply(From, ok),
+ region_guild_loop(State, Owner, Calls);
+ {'$gen_call', From, _Other} ->
+ gen_server:reply(From, ok),
+ region_guild_loop(State, Owner, Calls);
+ {region_calls, Caller} ->
+ Caller ! {region_calls, Calls},
+ region_guild_loop(State, Owner, Calls);
+ stop ->
+ ok
+ after 30000 ->
+ ok
+ end.
+
+region_guild_calls(GuildPid) ->
+ GuildPid ! {region_calls, self()},
+ receive
+ {region_calls, Calls} -> Calls
+ after 1000 ->
+ error(region_guild_unresponsive)
+ end.
+
+drain_region_dispatches() ->
+ receive
+ {'$gen_cast', {dispatch, voice_server_update, Payload}} ->
+ [Payload | drain_region_dispatches()]
+ after 100 ->
+ []
+ end.
+
+%% token_nonce and the timestamps are freshly generated per call by design, so they are
+%% excluded before the reference and the threaded run are compared.
+drain_region_pending() ->
+ receive
+ {region_pending, ConnId, Meta} ->
+ Stable = maps:without([token_nonce, created_at, expires_at], Meta),
+ [{ConnId, Stable} | drain_region_pending()]
+ after 100 ->
+ []
+ end.
+
+region_state() ->
+ #{
+ id => 999,
+ data => #{
+ <<"id">> => <<"999">>,
+ <<"guild">> => #{<<"owner_id">> => <<"10">>},
+ <<"channels">> => [#{<<"id">> => <<"200">>, <<"type">> => 2}],
+ <<"members">> => [region_member(10), region_member(11)]
+ },
+ sessions => #{
+ <<"sess1">> => #{user_id => 10, pid => self(), pending_connect => false},
+ <<"sess2">> => #{user_id => 11, pid => self(), pending_connect => false}
+ },
+ voice_states => #{
+ <<"conn-a">> => region_voice_state(<<"conn-a">>, 10, <<"sess1">>),
+ <<"conn-b">> => region_voice_state(<<"conn-b">>, 11, <<"sess2">>)
+ },
+ pending_voice_connections => #{}
+ }.
+
+region_member(UserId) ->
+ #{
+ <<"user">> => #{<<"id">> => integer_to_binary(UserId)},
+ <<"mute">> => false,
+ <<"deaf">> => false
+ }.
+
+region_voice_state(ConnId, UserId, SessionId) ->
+ #{
+ <<"user_id">> => integer_to_binary(UserId),
+ <<"guild_id">> => <<"999">>,
+ <<"channel_id">> => <<"200">>,
+ <<"connection_id">> => ConnId,
+ <<"session_id">> => SessionId,
+ <<"self_mute">> => false,
+ <<"self_deaf">> => true,
+ <<"member">> => region_member(UserId)
+ }.
+
build_pending_metadata_preserves_voice_routing_test() ->
Metadata = build_pending_metadata(
10,
diff --git a/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl b/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl
index d74c665db..4f77dc31f 100644
--- a/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl
+++ b/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl
@@ -13,7 +13,9 @@
repair_voice_state_from_guild_cache/2,
voice_state_rpc_entries/1,
pending_join_rpc_entries/1,
- fetch_guild_data/1
+ fetch_guild_data/1,
+ guild_state_call/2,
+ guild_id_call/2
]).
-export_type([
@@ -335,9 +337,36 @@ pending_join_channel_id(Metadata) when is_map(Metadata) ->
pending_join_channel_id(_) ->
undefined.
+-spec guild_state_call(pid(), timeout()) -> term().
+guild_state_call(GuildPid, Timeout) ->
+ case gen_server:call(GuildPid, {get_voice_guild_state}, Timeout) of
+ GuildState when is_map(GuildState) -> GuildState;
+ _ -> full_guild_state_call(GuildPid, Timeout)
+ end.
+
+-spec full_guild_state_call(pid(), timeout()) -> term().
+full_guild_state_call(GuildPid, Timeout) ->
+ gen_server:call(GuildPid, {get_sessions}, Timeout).
+
+-spec guild_id_call(pid(), timeout()) -> integer() | undefined.
+guild_id_call(GuildPid, Timeout) ->
+ case gen_server:call(GuildPid, {get_guild_id}, Timeout) of
+ GuildId when is_integer(GuildId) -> GuildId;
+ _ -> guild_state_id(full_guild_state_call(GuildPid, Timeout))
+ end.
+
+-spec guild_state_id(term()) -> integer() | undefined.
+guild_state_id(GuildState) when is_map(GuildState) ->
+ case maps:get(id, GuildState, undefined) of
+ GuildId when is_integer(GuildId) -> GuildId;
+ _ -> undefined
+ end;
+guild_state_id(_) ->
+ undefined.
+
-spec fetch_guild_data(pid()) -> map().
fetch_guild_data(GuildPid) ->
- try gen_server:call(GuildPid, {get_sessions}, ?GUILD_CALL_TIMEOUT) of
+ try guild_state_call(GuildPid, ?GUILD_CALL_TIMEOUT) of
GuildState when is_map(GuildState) -> GuildState;
_ -> #{}
catch
@@ -407,3 +436,89 @@ normalize_rpc_millisecond(Value) when is_binary(Value), byte_size(Value) > 0 ->
end;
normalize_rpc_millisecond(_) ->
0.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+guild_state_call_uses_projection_test() ->
+ with_fake_guild(fun new_guild_handler/1, fun(Pid) ->
+ ?assertEqual(projected_guild_state(), guild_state_call(Pid, 1000)),
+ ?assertEqual([{get_voice_guild_state}], drain_requests())
+ end).
+
+guild_state_call_retries_get_sessions_against_old_peer_test() ->
+ with_fake_guild(fun old_guild_handler/1, fun(Pid) ->
+ ?assertEqual(full_guild_state(), guild_state_call(Pid, 1000)),
+ ?assertEqual([{get_voice_guild_state}, {get_sessions}], drain_requests())
+ end).
+
+fetch_guild_data_retries_get_sessions_against_old_peer_test() ->
+ with_fake_guild(fun old_guild_handler/1, fun(Pid) ->
+ ?assertEqual(full_guild_state(), fetch_guild_data(Pid)),
+ ?assertEqual([{get_voice_guild_state}, {get_sessions}], drain_requests())
+ end).
+
+guild_id_call_uses_get_guild_id_test() ->
+ with_fake_guild(fun new_guild_handler/1, fun(Pid) ->
+ ?assertEqual(42, guild_id_call(Pid, 1000)),
+ ?assertEqual([{get_guild_id}], drain_requests())
+ end).
+
+guild_id_call_returns_real_id_against_old_peer_test() ->
+ with_fake_guild(fun old_guild_handler/1, fun(Pid) ->
+ ?assertEqual(42, guild_id_call(Pid, 1000)),
+ ?assertEqual([{get_guild_id}, {get_sessions}], drain_requests())
+ end).
+
+guild_id_call_returns_undefined_when_peer_has_no_id_test() ->
+ with_fake_guild(fun(_) -> ok end, fun(Pid) ->
+ ?assertEqual(undefined, guild_id_call(Pid, 1000)),
+ ?assertEqual([{get_guild_id}, {get_sessions}], drain_requests())
+ end).
+
+new_guild_handler({get_voice_guild_state}) -> projected_guild_state();
+new_guild_handler({get_guild_id}) -> 42;
+new_guild_handler({get_sessions}) -> full_guild_state();
+new_guild_handler(_) -> ok.
+
+old_guild_handler({get_sessions}) -> full_guild_state();
+old_guild_handler(_) -> ok.
+
+full_guild_state() ->
+ #{
+ id => 42,
+ sessions => #{},
+ voice_states => #{},
+ data => #{<<"id">> => <<"42">>, <<"members">> => #{}}
+ }.
+
+projected_guild_state() ->
+ #{id => 42, sessions => #{}, voice_states => #{}, data => #{<<"id">> => <<"42">>}}.
+
+with_fake_guild(Handler, Fun) ->
+ _ = drain_requests(),
+ Owner = self(),
+ Pid = spawn(fun() -> fake_guild_loop(Owner, Handler) end),
+ try
+ Fun(Pid)
+ after
+ Pid ! stop
+ end.
+
+fake_guild_loop(Owner, Handler) ->
+ receive
+ {'$gen_call', From, Request} ->
+ Owner ! {fake_guild_request, Request},
+ gen_server:reply(From, Handler(Request)),
+ fake_guild_loop(Owner, Handler);
+ stop ->
+ ok
+ end.
+
+drain_requests() ->
+ receive
+ {fake_guild_request, Request} -> [Request | drain_requests()]
+ after 0 -> []
+ end.
+
+-endif.
diff --git a/fluxer_gateway/src/presence/presence_broadcast.erl b/fluxer_gateway/src/presence/presence_broadcast.erl
index a5d1bf20f..b7988af38 100644
--- a/fluxer_gateway/src/presence/presence_broadcast.erl
+++ b/fluxer_gateway/src/presence/presence_broadcast.erl
@@ -30,6 +30,8 @@
-type state() :: map().
-type sessions() :: #{binary() => map()}.
+-define(CUSTOM_STATUS_EXPIRY_TIMER, custom_status_expiry_timer).
+
-spec publish_global_if_needed({reply, term(), state()} | {noreply, state()}) ->
{reply, term(), state()} | {noreply, state()}.
publish_global_if_needed({reply, Reply, NewState}) ->
@@ -40,7 +42,8 @@ publish_global_if_needed({noreply, NewState}) ->
{noreply, FinalState}.
-spec publish_global_presence(sessions(), state()) -> state().
-publish_global_presence(_Sessions, State) ->
+publish_global_presence(_Sessions, State0) ->
+ State = refresh_custom_status_expiry_timer(State0),
{Payload, CurrentExternal, ExternalStatus} = build_presence_external(State),
LastPublished = maps:get(last_published_presence, State, undefined),
case presence_changed(LastPublished, CurrentExternal) of
@@ -55,11 +58,33 @@ publish_global_presence(_Sessions, State) ->
end.
-spec force_publish_global_presence(state()) -> state().
-force_publish_global_presence(State) ->
+force_publish_global_presence(State0) ->
+ State = refresh_custom_status_expiry_timer(State0),
{Payload, CurrentExternal, ExternalStatus} = build_presence_external(State),
NewState = publish_presence_payload(State, Payload, CurrentExternal, ExternalStatus),
presence_update:maybe_update_push_eligibility(NewState).
+-spec refresh_custom_status_expiry_timer(state()) -> state().
+refresh_custom_status_expiry_timer(State) ->
+ ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State, undefined)),
+ arm_expiry_timer(
+ custom_status_expiry:next_wakeup_ms(maps:get(custom_status, State, null)), State
+ ).
+
+-spec arm_expiry_timer({ok, pos_integer()} | none, state()) -> state().
+arm_expiry_timer(none, State) ->
+ maps:remove(?CUSTOM_STATUS_EXPIRY_TIMER, State);
+arm_expiry_timer({ok, DelayMs}, State) ->
+ Ref = erlang:send_after(DelayMs, self(), custom_status_expiry:reconcile_message()),
+ State#{?CUSTOM_STATUS_EXPIRY_TIMER => Ref}.
+
+-spec cancel_expiry_timer(term()) -> ok.
+cancel_expiry_timer(Ref) when is_reference(Ref) ->
+ _ = erlang:cancel_timer(Ref, [{async, true}, {info, false}]),
+ ok;
+cancel_expiry_timer(_Ref) ->
+ ok.
+
-spec dispatch_global_presence(user_id(), map(), state()) -> {noreply, state()}.
dispatch_global_presence(TargetId, Payload, State) ->
UserId = maps:get(user_id, State),
@@ -240,3 +265,97 @@ dispatch_global_user_update(TargetId, Payload, State) ->
dispatch_to_sessions(Payload, State) ->
SessionPids = presence_connect:collect_session_pids(State),
gateway_dispatch_relay:dispatch_many(SessionPids, presence_update, Payload).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+expiry_timer_not_armed_while_disabled_test() ->
+ application:unset_env(fluxer_gateway, custom_status_expiry_enabled),
+ State = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),
+ ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, State)).
+
+expiry_timer_not_armed_without_expires_at_test() ->
+ with_expiry_enabled(fun() ->
+ ?assertNot(
+ maps:is_key(
+ ?CUSTOM_STATUS_EXPIRY_TIMER,
+ refresh_custom_status_expiry_timer(#{custom_status => null})
+ )
+ ),
+ ?assertNot(
+ maps:is_key(
+ ?CUSTOM_STATUS_EXPIRY_TIMER,
+ refresh_custom_status_expiry_timer(#{
+ custom_status => #{<<"text">> => <<"hi">>}
+ })
+ )
+ )
+ end).
+
+expiry_timer_armed_for_a_future_expiry_test() ->
+ with_expiry_enabled(fun() ->
+ State = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),
+ ?assert(is_reference(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))),
+ ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))
+ end).
+
+expiry_timer_not_armed_for_an_already_expired_status_test() ->
+ with_expiry_enabled(fun() ->
+ Expired = #{<<"expires_at">> => <<"2020-01-01T00:00:00.000Z">>},
+ State = refresh_custom_status_expiry_timer(#{custom_status => Expired}),
+ ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, State))
+ end).
+
+expiry_timer_replaces_the_previous_one_test() ->
+ with_expiry_enabled(fun() ->
+ First = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),
+ Second = refresh_custom_status_expiry_timer(First),
+ ?assertNotEqual(
+ maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, First),
+ maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, Second)
+ ),
+ ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, Second))
+ end).
+
+expiry_timer_is_dropped_when_the_status_is_cleared_test() ->
+ with_expiry_enabled(fun() ->
+ Armed = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),
+ Cleared = refresh_custom_status_expiry_timer(Armed#{custom_status => null}),
+ ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, Cleared))
+ end).
+
+expiry_timer_delivers_a_reconcile_cast_test() ->
+ State = arm_expiry_timer({ok, 1}, #{}),
+ ?assert(is_reference(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))),
+ Expected = custom_status_expiry:reconcile_message(),
+ receive
+ Expected -> ok
+ after 1000 -> ?assert(false)
+ end.
+
+cancel_expiry_timer_ignores_non_references_test() ->
+ ?assertEqual(ok, cancel_expiry_timer(undefined)),
+ ?assertEqual(ok, cancel_expiry_timer(not_a_ref)).
+
+future_custom_status() ->
+ ExpiresAt = calendar:system_time_to_rfc3339(
+ erlang:system_time(millisecond) + 3600000, [{unit, millisecond}, {offset, "Z"}]
+ ),
+ #{<<"text">> => <<"brb">>, <<"expires_at">> => list_to_binary(ExpiresAt)}.
+
+with_expiry_enabled(Fun) ->
+ Key = custom_status_expiry_enabled,
+ Previous = application:get_env(fluxer_gateway, Key),
+ application:set_env(fluxer_gateway, Key, true),
+ try
+ Fun()
+ after
+ restore_expiry_env(Key, Previous)
+ end.
+
+restore_expiry_env(Key, undefined) ->
+ application:unset_env(fluxer_gateway, Key);
+restore_expiry_env(Key, {ok, Value}) ->
+ application:set_env(fluxer_gateway, Key, Value).
+
+-endif.
diff --git a/fluxer_gateway/src/presence/presence_payload.erl b/fluxer_gateway/src/presence/presence_payload.erl
index 44c2ed6c2..a1ea69bf3 100644
--- a/fluxer_gateway/src/presence/presence_payload.erl
+++ b/fluxer_gateway/src/presence/presence_payload.erl
@@ -32,9 +32,12 @@ ensure_status_binary(Status) when is_binary(Status) -> Status;
ensure_status_binary(_) -> <<"offline">>.
-spec custom_status_for(binary(), custom_status()) -> custom_status().
-custom_status_for(<<"offline">>, _CustomStatus) -> null;
-custom_status_for(<<"invisible">>, _CustomStatus) -> null;
-custom_status_for(_StatusBin, CustomStatus) -> normalize_custom_status(CustomStatus).
+custom_status_for(<<"offline">>, _CustomStatus) ->
+ null;
+custom_status_for(<<"invisible">>, _CustomStatus) ->
+ null;
+custom_status_for(_StatusBin, CustomStatus) ->
+ custom_status_expiry:clear_if_expired(normalize_custom_status(CustomStatus)).
-spec normalize_custom_status(term()) -> custom_status().
normalize_custom_status(null) -> null;
@@ -81,16 +84,88 @@ normalize_custom_status_test() ->
?assertEqual(null, normalize_custom_status(123)).
build_invisible_atom_normalized_to_offline_test() ->
- User = #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>},
CustomStatus = #{<<"text">> => <<"hello">>},
- Result = build(User, invisible, false, false, CustomStatus),
+ Result = build(test_user(), invisible, false, false, CustomStatus),
?assertEqual(<<"offline">>, maps:get(<<"status">>, Result)),
?assertEqual(null, maps:get(<<"custom_status">>, Result)).
build_invisible_binary_normalized_to_offline_test() ->
- User = #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>},
CustomStatus = #{<<"text">> => <<"hello">>},
- Result = build(User, <<"invisible">>, false, false, CustomStatus),
+ Result = build(test_user(), <<"invisible">>, false, false, CustomStatus),
?assertEqual(<<"offline">>, maps:get(<<"status">>, Result)),
?assertEqual(null, maps:get(<<"custom_status">>, Result)).
+
+build_clears_expired_custom_status_test() ->
+ with_expiry_enabled(fun() ->
+ Result = build(test_user(), online, false, false, expired_custom_status()),
+ ?assertEqual(<<"online">>, maps:get(<<"status">>, Result)),
+ ?assertEqual(null, maps:get(<<"custom_status">>, Result))
+ end).
+
+build_coalesces_an_expired_status_with_no_status_test() ->
+ with_expiry_enabled(fun() ->
+ ?assertEqual(
+ build(test_user(), online, false, false, null),
+ build(test_user(), online, false, false, expired_custom_status())
+ )
+ end).
+
+build_keeps_unexpired_custom_status_test() ->
+ with_expiry_enabled(fun() ->
+ Live = future_custom_status(),
+ Result = build(test_user(), dnd, false, false, Live),
+ ?assertEqual(Live, maps:get(<<"custom_status">>, Result))
+ end).
+
+build_keeps_custom_status_without_expires_at_test() ->
+ with_expiry_enabled(fun() ->
+ Live = #{<<"text">> => <<"forever">>},
+ Result = build(test_user(), idle, false, false, Live),
+ ?assertEqual(Live, maps:get(<<"custom_status">>, Result))
+ end).
+
+build_keeps_malformed_expires_at_test() ->
+ with_expiry_enabled(fun() ->
+ Live = #{<<"text">> => <<"hi">>, <<"expires_at">> => <<"not-a-date">>},
+ Result = build(test_user(), online, false, false, Live),
+ ?assertEqual(Live, maps:get(<<"custom_status">>, Result))
+ end).
+
+build_leaves_expired_status_alone_while_disabled_test() ->
+ application:unset_env(fluxer_gateway, custom_status_expiry_enabled),
+ Expired = expired_custom_status(),
+ Result = build(test_user(), online, false, false, Expired),
+ ?assertEqual(Expired, maps:get(<<"custom_status">>, Result)).
+
+test_user() ->
+ #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>}.
+
+expired_custom_status() ->
+ #{
+ <<"emoji_animated">> => false,
+ <<"emoji_name">> => null,
+ <<"expires_at">> => <<"2026-05-13T13:02:27.497Z">>,
+ <<"text">> => <<"brb">>
+ }.
+
+future_custom_status() ->
+ ExpiresAt = calendar:system_time_to_rfc3339(
+ erlang:system_time(millisecond) + 3600000, [{unit, millisecond}, {offset, "Z"}]
+ ),
+ #{<<"text">> => <<"brb">>, <<"expires_at">> => list_to_binary(ExpiresAt)}.
+
+with_expiry_enabled(Fun) ->
+ Key = custom_status_expiry_enabled,
+ Previous = application:get_env(fluxer_gateway, Key),
+ application:set_env(fluxer_gateway, Key, true),
+ try
+ Fun()
+ after
+ restore_expiry_env(Key, Previous)
+ end.
+
+restore_expiry_env(Key, undefined) ->
+ application:unset_env(fluxer_gateway, Key);
+restore_expiry_env(Key, {ok, Value}) ->
+ application:set_env(fluxer_gateway, Key, Value).
-endif.
diff --git a/fluxer_gateway/src/push/push.erl b/fluxer_gateway/src/push/push.erl
index f721926dc..d7508c7d1 100644
--- a/fluxer_gateway/src/push/push.erl
+++ b/fluxer_gateway/src/push/push.erl
@@ -25,6 +25,36 @@
-define(EVICT_INTERVAL_MS, 60000).
-define(DEFAULT_MAX_ENTRIES, 500000).
+-define(PUSH_COUNTER_TABLE, push_worker_counter).
+-define(CNT_WORKER_POOL, push_loss_worker_pool).
+-define(CNT_WARM_INFLIGHT, push_blocked_ids_warm_inflight).
+-define(CNT_FETCH_ATTEMPTS, push_blocked_ids_fetch_attempts).
+-define(CNT_FETCH_FAILURES, push_blocked_ids_fetch_failures).
+-define(CNT_SUPPRESSED, push_blocked_ids_suppressed).
+-define(CNT_BUDGET_EXHAUSTED, push_blocked_ids_budget_exhausted).
+-define(CNT_WARM_DROPPED, push_blocked_ids_warm_dropped).
+-define(CNT_DISPATCH_DROPPED, push_loss_dispatch_dropped).
+-define(CNT_DISPATCH_DROPPED_USERS, push_loss_dispatch_dropped_users).
+-define(CNT_CLEAR_DROPPED, push_loss_clear_dropped).
+-define(CNT_QUEUE_FULL, push_loss_queue_full).
+-define(CNT_INVALID_JOB, push_loss_invalid_job).
+-define(CNT_ENQUEUE_TIMEOUT, push_loss_enqueue_timeout).
+-define(CNT_ENQUEUE_FAILED, push_loss_enqueue_failed).
+-define(CNT_JOB_CRASHED, push_loss_job_crashed).
+-define(CNT_WORKER_DIED, push_loss_worker_died).
+-define(CNT_DISPATCHER_RESTARTS, push_dispatcher_restarts).
+-define(CNT_RESTART_DISCARDED, push_dispatcher_restart_discarded).
+-define(CNT_QUEUE_ENQUEUED, push_dispatcher_queue_enqueued).
+-define(CNT_QUEUE_DEQUEUED, push_dispatcher_queue_dequeued).
+
+-define(MAX_WARM_INFLIGHT, 4).
+-define(MAX_FETCH_RPCS, 8).
+-define(DEFAULT_FETCH_USERS, 2000).
+-define(MAX_FETCH_USERS, 5000).
+-define(DEFAULT_FETCH_CHUNK, 500).
+-define(MIN_FETCH_CHUNK, 50).
+-define(MAX_FETCH_CHUNK, 1000).
+
-type state() :: #{
badge_counts_ttl_seconds := non_neg_integer(),
max_entries := non_neg_integer()
@@ -61,8 +91,7 @@ init([]) ->
-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}.
handle_call(get_cache_stats, _From, State) ->
- Stats = push_ets_cache:cache_stats(),
- {reply, {ok, Stats}, State};
+ {reply, {ok, cache_stats_with_counters()}, State};
handle_call(_Request, _From, State) ->
{reply, ok, State}.
@@ -96,6 +125,7 @@ handle_cast(_Msg, State) ->
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info(evict_caches, State) ->
+ unstick_warm_gate(),
MaxEntries = maps:get(max_entries, State),
push_ets_cache:evict_tables(#{
user_guild_settings => MaxEntries,
@@ -199,9 +229,7 @@ is_push_noop() ->
-spec clear_channel_notifications(integer(), integer(), integer()) -> ok.
clear_channel_notifications(UserId, ChannelId, MessageId) ->
- case
- is_push_active() andalso persistent_term:get(push_clear_notifications_enabled, false)
- of
+ case is_push_active() andalso clear_notifications_enabled() of
true ->
cast_to_push_owner(
UserId, {clear_channel_notifications, UserId, ChannelId, MessageId}
@@ -210,6 +238,13 @@ clear_channel_notifications(UserId, ChannelId, MessageId) ->
ok
end.
+-spec clear_notifications_enabled() -> boolean().
+clear_notifications_enabled() ->
+ case persistent_term:get(push_clear_notifications_enabled, undefined) of
+ Value when is_boolean(Value) -> Value;
+ _ -> env_boolean(push_clear_notifications_enabled, false)
+ end.
+
-spec get_cache_stats() -> {ok, map()}.
get_cache_stats() ->
gen_server:call(?MODULE, get_cache_stats, 5000).
@@ -299,13 +334,13 @@ do_handle_message_create_context(Context, State) ->
MessageData,
GuildDefaultNotifications,
UserRolesMap,
- ConnectedUsers
+ ConnectedUsers,
+ maps:get(large_guild_metadata, Context, undefined)
),
logger:debug("Push: eligibility result", #{
message_id => MessageId,
channel_id => ChannelId,
- eligible_count => length(EligibleUsers),
- eligible_user_ids => EligibleUsers
+ eligible_count => length(EligibleUsers)
}),
dispatch_if_eligible(
EligibleUsers,
@@ -320,7 +355,15 @@ do_handle_message_create_context(Context, State) ->
).
-spec filter_eligible_users(
- [integer()], integer(), integer(), integer(), map(), integer(), map(), map()
+ [integer()],
+ integer(),
+ integer(),
+ integer(),
+ map(),
+ integer(),
+ map(),
+ map(),
+ map() | undefined
) -> [integer()].
filter_eligible_users(
UserIds,
@@ -330,11 +373,13 @@ filter_eligible_users(
MessageData,
GuildDefaultNotifications,
UserRolesMap,
- ConnectedUsers
+ ConnectedUsers,
+ SuppliedMetadata
) ->
- LargeGuildMetadata = large_guild_metadata(GuildId),
- push_eligibility:prefetch_user_guild_settings(UserIds, AuthorId, GuildId),
- lists:filter(
+ LargeGuildMetadata = resolve_large_guild_metadata(GuildId, SuppliedMetadata),
+ Candidates = drop_blocked_recipients(UserIds, AuthorId),
+ push_eligibility:prefetch_user_guild_settings(Candidates, AuthorId, GuildId),
+ EligibleUsers = lists:filter(
fun(UserId) ->
push_eligibility:is_eligible_for_push(
UserId,
@@ -348,15 +393,249 @@ filter_eligible_users(
LargeGuildMetadata
)
end,
- UserIds
- ).
+ Candidates
+ ),
+ warm_blocked_ids(EligibleUsers),
+ EligibleUsers.
--spec large_guild_metadata(integer()) -> map() | undefined.
-large_guild_metadata(0) ->
+-spec resolve_large_guild_metadata(integer(), map() | undefined) -> map() | undefined.
+resolve_large_guild_metadata(0, _SuppliedMetadata) ->
undefined;
-large_guild_metadata(GuildId) ->
+resolve_large_guild_metadata(GuildId, SuppliedMetadata) ->
+ supplied_or_local_metadata(GuildId, SuppliedMetadata).
+
+-spec supplied_or_local_metadata(integer(), map() | undefined) -> map() | undefined.
+supplied_or_local_metadata(GuildId, undefined) ->
+ large_guild_metadata_local(GuildId);
+supplied_or_local_metadata(_GuildId, SuppliedMetadata) when is_map(SuppliedMetadata) ->
+ SuppliedMetadata;
+supplied_or_local_metadata(GuildId, _SuppliedMetadata) ->
+ large_guild_metadata_local(GuildId).
+
+-spec large_guild_metadata_local(integer()) -> map() | undefined.
+large_guild_metadata_local(GuildId) ->
push_eligibility_checks:get_guild_large_metadata(GuildId).
+-spec drop_blocked_recipients([integer()], integer()) -> [integer()].
+drop_blocked_recipients(UserIds, AuthorId) ->
+ Kept = lists:filter(
+ fun(UserId) -> not push_eligibility:is_user_blocked(UserId, AuthorId) end,
+ UserIds
+ ),
+ count_suppressed(length(UserIds) - length(Kept)),
+ Kept.
+
+-spec count_suppressed(non_neg_integer()) -> ok.
+count_suppressed(0) ->
+ ok;
+count_suppressed(Suppressed) ->
+ bump_counter(?CNT_SUPPRESSED, Suppressed).
+
+-spec warm_blocked_ids([integer()]) -> ok.
+warm_blocked_ids([]) ->
+ ok;
+warm_blocked_ids(EligibleUsers) ->
+ case missing_blocked_ids(EligibleUsers) of
+ [] -> ok;
+ Missing -> spawn_blocked_ids_warm(Missing)
+ end.
+
+-spec missing_blocked_ids([integer()]) -> [integer()].
+missing_blocked_ids(EligibleUsers) ->
+ lists:usort(lists:filter(fun is_blocked_ids_cache_miss/1, EligibleUsers)).
+
+-spec is_blocked_ids_cache_miss(integer()) -> boolean().
+is_blocked_ids_cache_miss(UserId) ->
+ push_ets_cache:get_blocked_ids(UserId) =:= undefined.
+
+-spec spawn_blocked_ids_warm([integer()]) -> ok.
+spawn_blocked_ids_warm(Missing) ->
+ case claim_warm_slot() of
+ ok ->
+ _ = spawn(fun() -> run_blocked_ids_warm(Missing) end),
+ ok;
+ full ->
+ bump_counter(?CNT_WARM_DROPPED)
+ end.
+
+-spec run_blocked_ids_warm([integer()]) -> ok.
+run_blocked_ids_warm(Missing) ->
+ try
+ warm_within_budget(Missing)
+ catch
+ throw:Reason -> warm_crashed(throw, Reason);
+ error:Reason -> warm_crashed(error, Reason);
+ exit:Reason -> warm_crashed(exit, Reason)
+ after
+ release_warm_slot()
+ end.
+
+-spec warm_crashed(throw | error | exit, term()) -> ok.
+warm_crashed(Class, Reason) ->
+ bump_counter(?CNT_FETCH_FAILURES),
+ logger:debug("Push: blocked id warm crashed", #{class => Class, reason => Reason}),
+ ok.
+
+-spec warm_within_budget([integer()]) -> ok.
+warm_within_budget(Missing) ->
+ {Budget, ChunkSize} = blocked_ids_fetch_budget(),
+ Budgeted = lists:sublist(Missing, Budget),
+ count_budget_exhausted(length(Missing) - length(Budgeted)),
+ fetch_blocked_ids_chunks(chunk_user_ids(Budgeted, ChunkSize, [])).
+
+-spec blocked_ids_fetch_budget() -> {pos_integer(), pos_integer()}.
+blocked_ids_fetch_budget() ->
+ ChunkSize = blocked_ids_fetch_chunk(),
+ MaxUsers = blocked_ids_fetch_max_users(),
+ {min(MaxUsers, ChunkSize * ?MAX_FETCH_RPCS), ChunkSize}.
+
+-spec count_budget_exhausted(non_neg_integer()) -> ok.
+count_budget_exhausted(0) ->
+ ok;
+count_budget_exhausted(Skipped) ->
+ bump_counter(?CNT_BUDGET_EXHAUSTED, Skipped).
+
+-spec fetch_blocked_ids_chunks([[integer()]]) -> ok.
+fetch_blocked_ids_chunks([]) ->
+ ok;
+fetch_blocked_ids_chunks([Chunk | Rest]) ->
+ case fetch_blocked_ids_chunk(Chunk) of
+ ok -> fetch_blocked_ids_chunks(Rest);
+ error -> ok
+ end.
+
+-spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].
+chunk_user_ids([], _ChunkSize, Acc) ->
+ lists:reverse(Acc);
+chunk_user_ids(UserIds, ChunkSize, Acc) ->
+ Chunk = lists:sublist(UserIds, ChunkSize),
+ chunk_user_ids(drop_prefix(UserIds, ChunkSize), ChunkSize, [Chunk | Acc]).
+
+-spec drop_prefix([integer()], non_neg_integer()) -> [integer()].
+drop_prefix(UserIds, 0) ->
+ UserIds;
+drop_prefix([], _N) ->
+ [];
+drop_prefix([_UserId | Rest], N) ->
+ drop_prefix(Rest, N - 1).
+
+-spec fetch_blocked_ids_chunk([integer()]) -> ok | error.
+fetch_blocked_ids_chunk([]) ->
+ ok;
+fetch_blocked_ids_chunk(UserIds) ->
+ bump_counter(?CNT_FETCH_ATTEMPTS),
+ Request = #{
+ <<"type">> => <<"get_user_blocked_ids">>,
+ <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]
+ },
+ case rpc_client:call(Request) of
+ {ok, Data} when is_map(Data) ->
+ cache_blocked_ids_response(UserIds, Data);
+ {ok, _Data} ->
+ fetch_blocked_ids_failed(malformed_response, length(UserIds));
+ {error, Reason} ->
+ fetch_blocked_ids_failed(Reason, length(UserIds))
+ end.
+
+-spec fetch_blocked_ids_failed(term(), non_neg_integer()) -> error.
+fetch_blocked_ids_failed(Reason, UserCount) ->
+ bump_counter(?CNT_FETCH_FAILURES),
+ logger:debug("Push: blocked id fetch failed", #{
+ reason => Reason, user_count => UserCount
+ }),
+ error.
+
+-spec cache_blocked_ids_response([integer()], map()) -> ok.
+cache_blocked_ids_response(UserIds, Data) ->
+ lists:foreach(
+ fun(UserId) -> cache_blocked_ids_entry(UserId, Data) end,
+ UserIds
+ ),
+ ok.
+
+-spec cache_blocked_ids_entry(integer(), map()) -> ok.
+cache_blocked_ids_entry(UserId, Data) ->
+ Raw = maps:get(integer_to_binary(UserId), Data, []),
+ push_ets_cache:put_blocked_ids_fetched(UserId, blocked_ids_from_response(Raw)).
+
+-spec blocked_ids_from_response(term()) -> [integer()].
+blocked_ids_from_response(Values) when is_list(Values) ->
+ lists:filtermap(fun snowflake_id:filter/1, Values);
+blocked_ids_from_response(_Values) ->
+ [].
+
+-spec blocked_ids_fetch_max_users() -> pos_integer().
+blocked_ids_fetch_max_users() ->
+ Value = app_pos_integer(push_blocked_ids_fetch_max_users, ?DEFAULT_FETCH_USERS),
+ min(Value, ?MAX_FETCH_USERS).
+
+-spec blocked_ids_fetch_chunk() -> pos_integer().
+blocked_ids_fetch_chunk() ->
+ Value = app_pos_integer(push_blocked_ids_fetch_chunk, ?DEFAULT_FETCH_CHUNK),
+ min(max(Value, ?MIN_FETCH_CHUNK), ?MAX_FETCH_CHUNK).
+
+-spec app_pos_integer(atom(), pos_integer()) -> pos_integer().
+app_pos_integer(Key, Default) ->
+ case application:get_env(fluxer_gateway, Key, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> Default
+ end.
+
+-spec blocked_ids_counters() -> map().
+blocked_ids_counters() ->
+ #{
+ blocked_ids_fetch_attempts => read_counter(?CNT_FETCH_ATTEMPTS),
+ blocked_ids_fetch_failures => read_counter(?CNT_FETCH_FAILURES),
+ blocked_ids_suppressed => read_counter(?CNT_SUPPRESSED),
+ blocked_ids_budget_exhausted => read_counter(?CNT_BUDGET_EXHAUSTED),
+ blocked_ids_warm_dropped => read_counter(?CNT_WARM_DROPPED),
+ blocked_ids_warm_inflight => read_counter(?CNT_WARM_INFLIGHT)
+ }.
+
+-spec claim_warm_slot() -> ok | full.
+claim_warm_slot() ->
+ try ets:update_counter(?PUSH_COUNTER_TABLE, ?CNT_WARM_INFLIGHT, {2, 1}) of
+ Value when is_integer(Value), Value =< ?MAX_WARM_INFLIGHT ->
+ ok;
+ _Value ->
+ release_warm_slot(),
+ full
+ catch
+ error:badarg -> claim_first_warm_slot()
+ end.
+
+-spec claim_first_warm_slot() -> ok | full.
+claim_first_warm_slot() ->
+ try ets:insert_new(?PUSH_COUNTER_TABLE, {?CNT_WARM_INFLIGHT, 1}) of
+ true -> ok;
+ false -> full
+ catch
+ error:badarg -> full
+ end.
+
+-spec release_warm_slot() -> ok.
+release_warm_slot() ->
+ try ets:update_counter(?PUSH_COUNTER_TABLE, ?CNT_WARM_INFLIGHT, {2, -1, 0, 0}) of
+ _Value -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+-spec unstick_warm_gate() -> ok.
+unstick_warm_gate() ->
+ case read_counter(?CNT_WARM_INFLIGHT) of
+ Value when is_integer(Value), Value >= ?MAX_WARM_INFLIGHT -> reset_warm_gate();
+ _Value -> ok
+ end.
+
+-spec reset_warm_gate() -> ok.
+reset_warm_gate() ->
+ try ets:insert(?PUSH_COUNTER_TABLE, {?CNT_WARM_INFLIGHT, 0}) of
+ _Value -> ok
+ catch
+ error:badarg -> ok
+ end.
+
-spec dispatch_if_eligible(
[integer()],
map(),
@@ -408,15 +687,30 @@ dispatch_if_eligible(
ok ->
ok;
dropped ->
- logger:debug("Push: dispatcher saturated, dropping notification job", #{
- message_id => MessageId,
- channel_id => ChannelId,
- guild_id => GuildId,
- eligible_count => length(EligibleUsers)
- }),
+ EligibleCount = length(EligibleUsers),
+ count_dispatch_dropped(EligibleCount),
+ log_dispatch_drop(
+ loss_logging_enabled(), MessageId, ChannelId, GuildId, EligibleCount
+ ),
ok
end.
+-spec log_dispatch_drop(boolean(), integer(), integer(), integer(), non_neg_integer()) -> ok.
+log_dispatch_drop(true, MessageId, ChannelId, GuildId, EligibleCount) ->
+ logger:error("Push: dispatcher saturated, dropping notification job", #{
+ message_id => MessageId,
+ channel_id => ChannelId,
+ guild_id => GuildId,
+ eligible_count => EligibleCount
+ });
+log_dispatch_drop(false, MessageId, ChannelId, GuildId, EligibleCount) ->
+ logger:debug("Push: dispatcher saturated, dropping notification job", #{
+ message_id => MessageId,
+ channel_id => ChannelId,
+ guild_id => GuildId,
+ eligible_count => EligibleCount
+ }).
+
-spec handle_sync_user_blocked_ids(integer(), term(), state()) -> {noreply, state()}.
handle_sync_user_blocked_ids(UserId, BlockedIds, State) ->
case push_normalize:integer_list(BlockedIds) of
@@ -449,22 +743,138 @@ handle_clear_channel_notifications(UserId, ChannelId, MessageId, State) ->
ok ->
ok;
dropped ->
- logger:debug("Push: dispatcher saturated, dropping clear notification job", #{
- user_id => UserId, channel_id => ChannelId, message_id => MessageId
- })
+ count_clear_dropped(),
+ log_clear_drop(loss_logging_enabled(), UserId, ChannelId, MessageId)
end,
{noreply, State}.
+-spec log_clear_drop(boolean(), integer(), integer(), integer()) -> ok.
+log_clear_drop(true, UserId, ChannelId, MessageId) ->
+ logger:warning("Push: dispatcher saturated, dropping clear notification job", #{
+ user_id => UserId, channel_id => ChannelId, message_id => MessageId
+ });
+log_clear_drop(false, UserId, ChannelId, MessageId) ->
+ logger:debug("Push: dispatcher saturated, dropping clear notification job", #{
+ user_id => UserId, channel_id => ChannelId, message_id => MessageId
+ }).
+
-spec log_message_worker_drop(ok | dropped, map()) -> ok.
log_message_worker_drop(ok, _Params) ->
ok;
log_message_worker_drop(dropped, Params) ->
+ bump_counter(?CNT_WORKER_POOL),
MessageData = maps:get(message_data, Params, #{}),
+ log_worker_pool_drop(
+ loss_logging_enabled(),
+ maps:get(<<"id">>, MessageData, undefined),
+ maps:get(<<"channel_id">>, MessageData, undefined)
+ ).
+
+-spec log_worker_pool_drop(boolean(), term(), term()) -> ok.
+log_worker_pool_drop(true, MessageId, ChannelId) ->
+ logger:error("Push: worker pool saturated, dropping message create", #{
+ message_id => MessageId, channel_id => ChannelId
+ });
+log_worker_pool_drop(false, MessageId, ChannelId) ->
logger:debug("Push: worker pool saturated, dropping message create", #{
- message_id => maps:get(<<"id">>, MessageData, undefined),
- channel_id => maps:get(<<"channel_id">>, MessageData, undefined)
+ message_id => MessageId, channel_id => ChannelId
}).
+-spec cache_stats_with_counters() -> map().
+cache_stats_with_counters() ->
+ Base = maps:merge(push_ets_cache:cache_stats(), blocked_ids_counters()),
+ maps:merge(Base, push_loss_counters()).
+
+-spec push_loss_counters() -> map().
+push_loss_counters() ->
+ #{
+ counters => counter_table_status(),
+ worker_pool_dropped => read_counter(?CNT_WORKER_POOL),
+ dispatch_dropped => read_counter(?CNT_DISPATCH_DROPPED),
+ dispatch_dropped_users => read_counter(?CNT_DISPATCH_DROPPED_USERS),
+ clear_dispatch_dropped => read_counter(?CNT_CLEAR_DROPPED),
+ dispatcher_queue_full => read_counter(?CNT_QUEUE_FULL),
+ dispatcher_invalid_job => read_counter(?CNT_INVALID_JOB),
+ dispatcher_enqueue_timeout => read_counter(?CNT_ENQUEUE_TIMEOUT),
+ dispatcher_enqueue_failed => read_counter(?CNT_ENQUEUE_FAILED),
+ dispatcher_job_crashed => read_counter(?CNT_JOB_CRASHED),
+ dispatcher_worker_died => read_counter(?CNT_WORKER_DIED),
+ dispatcher_restarts => read_counter(?CNT_DISPATCHER_RESTARTS),
+ dispatcher_restart_discarded => read_counter(?CNT_RESTART_DISCARDED),
+ dispatcher_queue_backlog => dispatcher_queue_backlog()
+ }.
+
+-spec count_dispatch_dropped(non_neg_integer()) -> ok.
+count_dispatch_dropped(EligibleCount) ->
+ bump_counter(?CNT_DISPATCH_DROPPED),
+ bump_counter(?CNT_DISPATCH_DROPPED_USERS, EligibleCount).
+
+-spec count_clear_dropped() -> ok.
+count_clear_dropped() ->
+ bump_counter(?CNT_CLEAR_DROPPED).
+
+-spec counter_table_status() -> live | unavailable.
+counter_table_status() ->
+ case ets:info(?PUSH_COUNTER_TABLE, size) of
+ Size when is_integer(Size) -> live;
+ _ -> unavailable
+ end.
+
+-spec dispatcher_queue_backlog() -> non_neg_integer() | unavailable.
+dispatcher_queue_backlog() ->
+ backlog(read_counter(?CNT_QUEUE_ENQUEUED), read_counter(?CNT_QUEUE_DEQUEUED)).
+
+-spec backlog(non_neg_integer() | unavailable, non_neg_integer() | unavailable) ->
+ non_neg_integer() | unavailable.
+backlog(Enqueued, Dequeued) when is_integer(Enqueued), is_integer(Dequeued) ->
+ max(0, Enqueued - Dequeued);
+backlog(Enqueued, unavailable) when is_integer(Enqueued) ->
+ Enqueued;
+backlog(_Enqueued, _Dequeued) ->
+ unavailable.
+
+-spec loss_logging_enabled() -> boolean().
+loss_logging_enabled() ->
+ application:get_env(fluxer_gateway, push_loss_logging, false) =:= true.
+
+-spec read_counter(atom()) -> non_neg_integer() | unavailable.
+read_counter(Key) ->
+ try ets:lookup(?PUSH_COUNTER_TABLE, Key) of
+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;
+ _ -> unavailable
+ catch
+ error:badarg -> unavailable
+ end.
+
+-spec bump_counter(atom()) -> ok.
+bump_counter(Key) ->
+ bump_counter(Key, 1).
+
+-spec bump_counter(atom(), non_neg_integer()) -> ok.
+bump_counter(Key, Increment) ->
+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of
+ _Value -> ok
+ catch
+ error:badarg -> insert_missing_counter(Key, Increment)
+ end.
+
+-spec insert_missing_counter(atom(), non_neg_integer()) -> ok.
+insert_missing_counter(Key, Increment) ->
+ try ets:insert_new(?PUSH_COUNTER_TABLE, {Key, Increment}) of
+ true -> ok;
+ false -> retry_bump_counter(Key, Increment)
+ catch
+ error:badarg -> ok
+ end.
+
+-spec retry_bump_counter(atom(), non_neg_integer()) -> ok.
+retry_bump_counter(Key, Increment) ->
+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of
+ _Value -> ok
+ catch
+ error:badarg -> ok
+ end.
+
-spec local_cache_mutation(fun(() -> ok)) -> ok.
local_cache_mutation(Fun) ->
case whereis(?MODULE) of
@@ -521,6 +931,13 @@ env_boolean(Key) ->
_ -> false
end.
+-spec env_boolean(atom(), boolean()) -> boolean().
+env_boolean(Key, Default) ->
+ case fluxer_gateway_env:get(Key) of
+ Value when is_boolean(Value) -> Value;
+ _ -> Default
+ end.
+
-spec env_non_neg_integer(atom(), non_neg_integer()) -> non_neg_integer().
env_non_neg_integer(Key, Default) ->
case fluxer_gateway_env:get(Key) of
@@ -580,12 +997,13 @@ filter_eligible_users_fetches_large_metadata_once_test() ->
fun(UserId) -> push_ets_cache:put_user_guild_settings(UserId, 42, #{}) end,
[1, 2, 3]
),
+ lists:foreach(fun(UserId) -> push_ets_cache:put_blocked_ids(UserId, []) end, [1, 2, 3]),
Self = self(),
ok = meck:new(push_eligibility_checks, [passthrough, no_link]),
try
ok = meck:expect(push_eligibility_checks, get_guild_large_metadata, fun(42) ->
Self ! metadata_lookup,
- #{member_count => 300, features => []}
+ #{member_count => 3000, features => []}
end),
MessageData = #{
<<"channel_type">> => 0,
@@ -593,7 +1011,9 @@ filter_eligible_users_fetches_large_metadata_once_test() ->
},
?assertEqual(
[1],
- filter_eligible_users([1, 2, 3], 999, 42, 10, MessageData, 0, #{}, #{})
+ filter_eligible_users(
+ [1, 2, 3], 999, 42, 10, MessageData, 0, #{}, #{}, undefined
+ )
),
?assertEqual(1, drain_metadata_lookup_count(0))
after
@@ -604,6 +1024,212 @@ filter_eligible_users_fetches_large_metadata_once_test() ->
)
end.
+blocked_ids_fetch_defaults_are_bounded_test() ->
+ ?assertEqual(?DEFAULT_FETCH_USERS, blocked_ids_fetch_max_users()),
+ ?assertEqual(?DEFAULT_FETCH_CHUNK, blocked_ids_fetch_chunk()).
+
+blocked_ids_from_response_skips_unparsable_entries_test() ->
+ ?assertEqual([123, 456], blocked_ids_from_response([<<"123">>, <<"abc">>, 456, null])),
+ ?assertEqual([], blocked_ids_from_response(<<"not_a_list">>)).
+
+chunk_user_ids_splits_into_bounded_batches_test() ->
+ ?assertEqual([], chunk_user_ids([], 2, [])),
+ ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])),
+ ?assertEqual([[1, 2, 3]], chunk_user_ids([1, 2, 3], 500, [])).
+
+blocked_ids_fetch_budget_is_capped_test() ->
+ assert_budget_capped(),
+ with_fetch_env(1, 1000000, fun assert_budget_capped/0),
+ with_fetch_env(1000000, 1000000, fun assert_budget_capped/0),
+ assert_budget_capped().
+
+assert_budget_capped() ->
+ {Budget, ChunkSize} = blocked_ids_fetch_budget(),
+ ?assert(ChunkSize >= ?MIN_FETCH_CHUNK),
+ ?assert(ChunkSize =< ?MAX_FETCH_CHUNK),
+ ?assert(Budget =< ?MAX_FETCH_USERS),
+ ?assert(length(chunk_user_ids(lists:seq(1, Budget), ChunkSize, [])) =< ?MAX_FETCH_RPCS).
+
+synced_blocked_recipients_are_dropped_test() ->
+ push_ets_cache:init(),
+ ok = push_ets_cache:put_blocked_ids(5020, [999]),
+ ?assertEqual([], filter_dm_recipients([5020], 999)).
+
+fetched_blocked_recipients_are_dropped_test() ->
+ push_ets_cache:init(),
+ ok = push_ets_cache:put_blocked_ids_fetched(5031, [999]),
+ ?assertEqual([], filter_dm_recipients([5031], 999)).
+
+block_suppression_drops_blocked_recipient_test() ->
+ push_ets_cache:init(),
+ ok = push_ets_cache:put_blocked_ids(5002, [999]),
+ ok = push_ets_cache:put_blocked_ids(5003, []),
+ ?assertEqual([5003], filter_dm_recipients([5002, 5003], 999)).
+
+blocked_ids_fetch_stops_after_a_failing_chunk_test() ->
+ ?assertEqual(ok, fetch_blocked_ids_chunks([])),
+ ?assertEqual(ok, fetch_blocked_ids_chunks([[], []])).
+
+cached_recipients_need_no_blocked_ids_warm_test() ->
+ push_ets_cache:init(),
+ ok = push_ets_cache:put_blocked_ids_fetched(5004, []),
+ ?assertEqual([], missing_blocked_ids([5004])),
+ ?assertEqual(ok, warm_blocked_ids([5004])).
+
+blocked_ids_counters_are_unavailable_without_the_shared_table_test() ->
+ delete_counter_table(),
+ ?assertEqual(unavailable, read_counter(?CNT_SUPPRESSED)),
+ ?assertEqual(full, claim_warm_slot()),
+ ?assertEqual(ok, release_warm_slot()),
+ ?assertEqual(ok, unstick_warm_gate()).
+
+blocked_ids_warm_gate_bounds_concurrency_test() ->
+ with_counter_table(fun() ->
+ Claims = [claim_warm_slot() || _ <- lists:seq(1, ?MAX_WARM_INFLIGHT + 2)],
+ ?assertEqual(?MAX_WARM_INFLIGHT, length([ok || ok <- Claims])),
+ ?assertEqual(?MAX_WARM_INFLIGHT, read_counter(?CNT_WARM_INFLIGHT)),
+ lists:foreach(
+ fun(_) -> release_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT + 2)
+ ),
+ ?assertEqual(0, read_counter(?CNT_WARM_INFLIGHT))
+ end).
+
+blocked_ids_warm_gate_unsticks_in_the_eviction_sweep_test() ->
+ with_counter_table(fun() ->
+ lists:foreach(fun(_) -> claim_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT)),
+ ?assertEqual(full, claim_warm_slot()),
+ unstick_warm_gate(),
+ ?assertEqual(0, read_counter(?CNT_WARM_INFLIGHT)),
+ ?assertEqual(ok, claim_warm_slot())
+ end).
+
+blocked_ids_warm_is_dropped_when_the_gate_is_full_test() ->
+ push_ets_cache:init(),
+ with_counter_table(fun() ->
+ lists:foreach(fun(_) -> claim_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT)),
+ ?assertEqual(ok, warm_blocked_ids([5099])),
+ ?assertEqual(1, read_counter(?CNT_WARM_DROPPED))
+ end).
+
+blocked_ids_counters_are_exposed_in_cache_stats_test() ->
+ push_ets_cache:init(),
+ with_counter_table(fun() ->
+ ok = push_ets_cache:put_blocked_ids(5010, [999]),
+ ?assertEqual([], filter_dm_recipients([5010], 999)),
+ ?assertEqual(1, read_counter(?CNT_SUPPRESSED)),
+ assert_cache_stats_expose_blocked_ids()
+ end).
+
+assert_cache_stats_expose_blocked_ids() ->
+ Stats = maps:merge(push_ets_cache:cache_stats(), blocked_ids_counters()),
+ lists:foreach(
+ fun(Key) -> ?assertEqual(true, maps:is_key(Key, Stats)) end,
+ [
+ blocked_ids_size,
+ blocked_ids_fetch_attempts,
+ blocked_ids_fetch_failures,
+ blocked_ids_suppressed,
+ blocked_ids_budget_exhausted,
+ blocked_ids_warm_dropped,
+ blocked_ids_warm_inflight
+ ]
+ ).
+
+push_loss_counters_expose_every_counter_push_writes_test() ->
+ with_counter_table(fun() ->
+ ok = log_message_worker_drop(dropped, #{}),
+ ok = count_dispatch_dropped(12),
+ ok = count_clear_dropped(),
+ Stats = push_loss_counters(),
+ ?assertEqual(live, maps:get(counters, Stats)),
+ ?assertEqual(1, maps:get(worker_pool_dropped, Stats)),
+ ?assertEqual(1, maps:get(dispatch_dropped, Stats)),
+ ?assertEqual(12, maps:get(dispatch_dropped_users, Stats)),
+ ?assertEqual(1, maps:get(clear_dispatch_dropped, Stats))
+ end).
+
+push_loss_counters_are_unavailable_without_the_shared_table_test() ->
+ delete_counter_table(),
+ Stats = push_loss_counters(),
+ ?assertEqual(unavailable, maps:get(counters, Stats)),
+ ?assertEqual(unavailable, maps:get(worker_pool_dropped, Stats)),
+ ?assertEqual(unavailable, maps:get(dispatch_dropped, Stats)),
+ ?assertEqual(unavailable, maps:get(dispatcher_enqueue_timeout, Stats)),
+ ?assertEqual(unavailable, maps:get(dispatcher_queue_backlog, Stats)).
+
+push_loss_counters_keep_a_genuine_zero_distinct_from_absent_test() ->
+ with_counter_table(fun() ->
+ ok = count_dispatch_dropped(0),
+ Stats = push_loss_counters(),
+ ?assertEqual(1, maps:get(dispatch_dropped, Stats)),
+ ?assertEqual(0, maps:get(dispatch_dropped_users, Stats)),
+ ?assertEqual(unavailable, maps:get(clear_dispatch_dropped, Stats))
+ end).
+
+dispatcher_queue_backlog_survives_an_untrappable_dispatcher_kill_test() ->
+ with_counter_table(fun() ->
+ ?assertEqual(unavailable, dispatcher_queue_backlog()),
+ ok = bump_counter(?CNT_QUEUE_ENQUEUED, 9),
+ ?assertEqual(9, dispatcher_queue_backlog()),
+ ok = bump_counter(?CNT_QUEUE_DEQUEUED, 4),
+ ?assertEqual(5, dispatcher_queue_backlog())
+ end).
+
+cache_stats_with_counters_carries_the_loss_surface_test() ->
+ push_ets_cache:init(),
+ with_counter_table(fun() ->
+ Stats = cache_stats_with_counters(),
+ lists:foreach(
+ fun(Key) -> ?assertEqual(true, maps:is_key(Key, Stats)) end,
+ [
+ blocked_ids_size,
+ blocked_ids_suppressed,
+ counters,
+ worker_pool_dropped,
+ dispatch_dropped,
+ dispatch_dropped_users,
+ clear_dispatch_dropped,
+ dispatcher_queue_full,
+ dispatcher_enqueue_timeout,
+ dispatcher_job_crashed,
+ dispatcher_worker_died,
+ dispatcher_restarts,
+ dispatcher_restart_discarded,
+ dispatcher_queue_backlog
+ ]
+ )
+ end).
+
+with_counter_table(Fun) ->
+ delete_counter_table(),
+ _ = ets:new(?PUSH_COUNTER_TABLE, [named_table, public, set, {write_concurrency, true}]),
+ try
+ Fun()
+ after
+ delete_counter_table()
+ end.
+
+delete_counter_table() ->
+ try ets:delete(?PUSH_COUNTER_TABLE) of
+ _ -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+filter_dm_recipients(UserIds, AuthorId) ->
+ MessageData = #{<<"channel_type">> => 1},
+ filter_eligible_users(UserIds, AuthorId, 0, 10, MessageData, 0, #{}, #{}, undefined).
+
+with_fetch_env(ChunkSize, MaxUsers, Fun) ->
+ application:set_env(fluxer_gateway, push_blocked_ids_fetch_chunk, ChunkSize),
+ application:set_env(fluxer_gateway, push_blocked_ids_fetch_max_users, MaxUsers),
+ try
+ Fun()
+ after
+ application:unset_env(fluxer_gateway, push_blocked_ids_fetch_chunk),
+ application:unset_env(fluxer_gateway, push_blocked_ids_fetch_max_users)
+ end.
+
filter_eligible_users_batches_missing_settings_lookups_test() ->
push_ets_cache:init(),
Self = self(),
@@ -615,7 +1241,7 @@ filter_eligible_users_batches_missing_settings_lookups_test() ->
end),
?assertEqual(
[1, 2, 3],
- filter_eligible_users([1, 2, 3], 999, 43, 10, #{}, 0, #{}, #{})
+ filter_eligible_users([1, 2, 3], 999, 43, 10, #{}, 0, #{}, #{}, undefined)
),
?assertEqual(1, drain_settings_request_count(0))
after
diff --git a/fluxer_gateway/src/push/push_dispatcher.erl b/fluxer_gateway/src/push/push_dispatcher.erl
index 73ca0158e..93e9755c2 100644
--- a/fluxer_gateway/src/push/push_dispatcher.erl
+++ b/fluxer_gateway/src/push/push_dispatcher.erl
@@ -17,6 +17,19 @@
-define(DEFAULT_MAX_QUEUE, 10000).
-define(ENQUEUE_TIMEOUT_MS, 1000).
+-define(PUSH_COUNTER_TABLE, push_worker_counter).
+-define(CNT_QUEUE_FULL, push_loss_queue_full).
+-define(CNT_INVALID_JOB, push_loss_invalid_job).
+-define(CNT_ENQUEUE_TIMEOUT, push_loss_enqueue_timeout).
+-define(CNT_ENQUEUE_FAILED, push_loss_enqueue_failed).
+-define(CNT_JOB_CRASHED, push_loss_job_crashed).
+-define(CNT_WORKER_DIED, push_loss_worker_died).
+-define(CNT_WORKER_POOL, push_loss_worker_pool).
+-define(CNT_RESTARTS, push_dispatcher_restarts).
+-define(CNT_RESTART_DISCARDED, push_dispatcher_restart_discarded).
+-define(CNT_QUEUE_ENQUEUED, push_dispatcher_queue_enqueued).
+-define(CNT_QUEUE_DEQUEUED, push_dispatcher_queue_dequeued).
+
-type push_job() ::
#{
type := message_create,
@@ -44,7 +57,27 @@
inflight := non_neg_integer(),
workers := #{reference() => true},
max_inflight := pos_integer(),
- max_queue := pos_integer()
+ max_queue := pos_integer(),
+ started_at => integer()
+}.
+
+-type counter_value() :: non_neg_integer() | unavailable.
+
+-type stats() :: #{
+ queued := non_neg_integer(),
+ inflight := non_neg_integer(),
+ counters := live | unavailable,
+ dispatcher_uptime_seconds := non_neg_integer() | undefined,
+ dispatcher_restarts := counter_value(),
+ restart_discarded_jobs := counter_value(),
+ queue_backlog_lost := counter_value(),
+ queue_full_dropped := counter_value(),
+ invalid_job_dropped := counter_value(),
+ enqueue_timeout := counter_value(),
+ enqueue_failed := counter_value(),
+ job_crashed := counter_value(),
+ worker_died := counter_value(),
+ worker_pool_dropped := counter_value()
}.
-spec start_link() -> {ok, pid()} | {error, term()} | ignore.
@@ -135,25 +168,57 @@ enqueue_clear_notifications(UserId, ChannelId, MessageId, BadgeCountsTtlSeconds)
),
safe_enqueue(Job).
--spec stats() -> #{queued := non_neg_integer(), inflight := non_neg_integer()} | #{}.
+-spec stats() -> stats() | #{}.
stats() ->
- try
- gen_server:call(?MODULE, stats, 1000)
+ Enqueued = read_counter(?CNT_QUEUE_ENQUEUED),
+ try gen_server:call(?MODULE, stats, 1000) of
+ #{queued := Queued, inflight := Inflight} = Reply ->
+ StartedAt = maps:get(started_at, Reply, undefined),
+ stats_with_loss(Queued, Inflight, StartedAt, Enqueued);
+ _ ->
+ #{}
catch
exit:_ -> #{};
error:_ -> #{}
end.
+-spec stats_with_loss(non_neg_integer(), non_neg_integer(), term(), term()) -> stats().
+stats_with_loss(Queued, Inflight, StartedAt, Enqueued) ->
+ #{
+ queued => Queued,
+ inflight => Inflight,
+ counters => counter_table_status(),
+ dispatcher_uptime_seconds => uptime_seconds(StartedAt),
+ dispatcher_restarts => read_counter(?CNT_RESTARTS),
+ restart_discarded_jobs => read_counter(?CNT_RESTART_DISCARDED),
+ queue_backlog_lost => queue_backlog_lost(Enqueued, Queued),
+ queue_full_dropped => read_counter(?CNT_QUEUE_FULL),
+ invalid_job_dropped => read_counter(?CNT_INVALID_JOB),
+ enqueue_timeout => read_counter(?CNT_ENQUEUE_TIMEOUT),
+ enqueue_failed => read_counter(?CNT_ENQUEUE_FAILED),
+ job_crashed => read_counter(?CNT_JOB_CRASHED),
+ worker_died => read_counter(?CNT_WORKER_DIED),
+ worker_pool_dropped => read_counter(?CNT_WORKER_POOL)
+ }.
+
+-spec uptime_seconds(term()) -> non_neg_integer() | undefined.
+uptime_seconds(StartedAt) when is_integer(StartedAt) ->
+ max(0, erlang:monotonic_time(second) - StartedAt);
+uptime_seconds(_StartedAt) ->
+ undefined.
+
-spec init([]) -> {ok, state()}.
init([]) ->
erlang:process_flag(fullsweep_after, 10),
+ bump_counter(?CNT_RESTARTS),
{ok, #{
queue => queue:new(),
queued => 0,
inflight => 0,
workers => #{},
max_inflight => budget_aware_max_inflight(),
- max_queue => get_int_or_default(push_dispatcher_max_queue, ?DEFAULT_MAX_QUEUE)
+ max_queue => get_int_or_default(push_dispatcher_max_queue, ?DEFAULT_MAX_QUEUE),
+ started_at => erlang:monotonic_time(second)
}}.
-spec budget_aware_max_inflight() -> pos_integer().
@@ -167,7 +232,12 @@ budget_aware_max_inflight() ->
-spec handle_call(term(), gen_server:from(), state()) ->
{reply, term(), state()}.
handle_call(stats, _From, #{queued := Queued, inflight := Inflight} = State) ->
- {reply, #{queued => Queued, inflight => Inflight}, State};
+ Reply = #{
+ queued => Queued,
+ inflight => Inflight,
+ started_at => maps:get(started_at, State, undefined)
+ },
+ {reply, Reply, State};
handle_call({enqueue, Job}, _From, State) ->
{Result, State1} = handle_enqueue(Job, State),
{reply, Result, State1};
@@ -183,11 +253,12 @@ handle_cast(_Msg, State) ->
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info(
- {'DOWN', Ref, process, _Pid, _Reason},
+ {'DOWN', Ref, process, _Pid, Reason},
#{workers := Workers, inflight := Inflight} = State
) ->
case maps:is_key(Ref, Workers) of
true ->
+ count_worker_down(Reason),
RemainingWorkers = maps:remove(Ref, Workers),
DecrementedInflight = max(0, Inflight - 1),
drain_queue(State#{
@@ -201,8 +272,15 @@ handle_info(_Info, State) ->
{noreply, State}.
-spec terminate(term(), state()) -> ok.
-terminate(_Reason, _State) ->
- ok.
+terminate(_Reason, State) ->
+ bump_counter(?CNT_RESTART_DISCARDED, maps:get(queued, State, 0)).
+
+-spec count_worker_down(term()) -> ok.
+count_worker_down(normal) ->
+ ok;
+count_worker_down(_Reason) ->
+ bump_counter(?CNT_JOB_CRASHED),
+ bump_counter(?CNT_WORKER_DIED).
-spec code_change(term(), state(), term()) -> {ok, state()}.
code_change(_OldVsn, State, _Extra) ->
@@ -239,22 +317,40 @@ maybe_enqueue_or_start(Job, #{inflight := Inflight, max_inflight := MaxInflight}
maybe_enqueue(Job, #{queued := Queued, max_queue := MaxQueue, queue := Queue0} = State) ->
case Queued < MaxQueue of
true ->
+ bump_counter(?CNT_QUEUE_ENQUEUED),
Queue1 = queue:in(Job, Queue0),
{ok, State#{queue := Queue1, queued := Queued + 1}};
false ->
+ bump_counter(?CNT_QUEUE_FULL),
DropCount = bump_drop_count(),
- logger:warning(
- "Push: queue full, dropping job",
- #{
- message_id => maps:get(message_id, Job, undefined),
- queued => Queued,
- max_queue => MaxQueue,
- total_dropped => DropCount
- }
- ),
+ log_queue_full_drop(loss_logging_enabled(), Job, Queued, MaxQueue, DropCount),
{dropped, State}
end.
+-spec log_queue_full_drop(
+ boolean(), push_job(), non_neg_integer(), pos_integer(), non_neg_integer()
+) -> ok.
+log_queue_full_drop(true, Job, Queued, MaxQueue, _DropCount) ->
+ logger:error(
+ "Push: queue full, dropping job",
+ #{
+ message_id => maps:get(message_id, Job, undefined),
+ queued => Queued,
+ max_queue => MaxQueue,
+ total_dropped => read_counter(?CNT_QUEUE_FULL)
+ }
+ );
+log_queue_full_drop(false, Job, Queued, MaxQueue, DropCount) ->
+ logger:warning(
+ "Push: queue full, dropping job",
+ #{
+ message_id => maps:get(message_id, Job, undefined),
+ queued => Queued,
+ max_queue => MaxQueue,
+ total_dropped => DropCount
+ }
+ ).
+
-spec bump_drop_count() -> non_neg_integer().
bump_drop_count() ->
Current =
@@ -266,6 +362,68 @@ bump_drop_count() ->
erlang:put(push_dispatcher_drop_count, Updated),
Updated.
+-spec loss_logging_enabled() -> boolean().
+loss_logging_enabled() ->
+ application:get_env(fluxer_gateway, push_loss_logging, false) =:= true.
+
+-spec counter_table_status() -> live | unavailable.
+counter_table_status() ->
+ case ets:info(?PUSH_COUNTER_TABLE, size) of
+ Size when is_integer(Size) -> live;
+ _ -> unavailable
+ end.
+
+-spec queue_backlog_lost(term(), non_neg_integer()) -> counter_value().
+queue_backlog_lost(Enqueued, Queued) when is_integer(Enqueued) ->
+ max(0, Enqueued - dequeued_total() - Queued);
+queue_backlog_lost(_Enqueued, _Queued) ->
+ unavailable.
+
+-spec dequeued_total() -> non_neg_integer().
+dequeued_total() ->
+ case read_counter(?CNT_QUEUE_DEQUEUED) of
+ Dequeued when is_integer(Dequeued) -> Dequeued;
+ unavailable -> 0
+ end.
+
+-spec read_counter(atom()) -> counter_value().
+read_counter(Key) ->
+ try ets:lookup(?PUSH_COUNTER_TABLE, Key) of
+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;
+ _ -> unavailable
+ catch
+ error:badarg -> unavailable
+ end.
+
+-spec bump_counter(atom()) -> ok.
+bump_counter(Key) ->
+ bump_counter(Key, 1).
+
+-spec bump_counter(atom(), non_neg_integer()) -> ok.
+bump_counter(Key, Increment) ->
+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of
+ _Value -> ok
+ catch
+ error:badarg -> insert_missing_counter(Key, Increment)
+ end.
+
+-spec insert_missing_counter(atom(), non_neg_integer()) -> ok.
+insert_missing_counter(Key, Increment) ->
+ try ets:insert_new(?PUSH_COUNTER_TABLE, {Key, Increment}) of
+ true -> ok;
+ false -> retry_bump_counter(Key, Increment)
+ catch
+ error:badarg -> ok
+ end.
+
+-spec retry_bump_counter(atom(), non_neg_integer()) -> ok.
+retry_bump_counter(Key, Increment) ->
+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of
+ _Value -> ok
+ catch
+ error:badarg -> ok
+ end.
+
-spec start_job(push_job(), state()) -> state().
start_job(Job, #{workers := Workers, inflight := Inflight} = State) ->
{_Pid, Ref} =
@@ -340,6 +498,7 @@ log_enqueue_send_notifications(UserIds, GuildId, ChannelId, MessageId) ->
state()
) -> {noreply, state()}.
drain_available_queue({{value, Job}, Queue1}, Queued, State) ->
+ bump_counter(?CNT_QUEUE_DEQUEUED),
State1 = State#{queue := Queue1, queued := max(0, Queued - 1)},
State2 = start_job(Job, State1),
drain_queue(State2);
@@ -353,14 +512,29 @@ run_job(#{message_id := MessageId} = Job) ->
logger:debug("Push: worker completed", #{message_id => MessageId}),
ok
catch
- Class:Reason ->
- logger:debug(
- "Push: worker crashed",
- #{message_id => MessageId, class => Class, reason => Reason}
- ),
+ Class:Reason:Stacktrace ->
+ bump_counter(?CNT_JOB_CRASHED),
+ log_job_crash(loss_logging_enabled(), MessageId, Class, Reason, Stacktrace),
ok
end.
+-spec log_job_crash(boolean(), integer(), atom(), term(), list()) -> ok.
+log_job_crash(true, MessageId, Class, Reason, Stacktrace) ->
+ logger:error(
+ "Push: worker crashed",
+ #{
+ message_id => MessageId,
+ class => Class,
+ reason => Reason,
+ stacktrace => Stacktrace
+ }
+ );
+log_job_crash(false, MessageId, Class, Reason, _Stacktrace) ->
+ logger:debug(
+ "Push: worker crashed",
+ #{message_id => MessageId, class => Class, reason => Reason}
+ ).
+
-spec run_typed_job(message_create | clear_channel, push_job()) -> ok.
run_typed_job(message_create, #{
user_ids := UserIds,
@@ -412,20 +586,39 @@ get_int_or_default(Key, Default) ->
-spec safe_enqueue(push_job()) -> ok | dropped.
safe_enqueue(Job) ->
try gen_server:call(?MODULE, {enqueue, Job}, ?ENQUEUE_TIMEOUT_MS) of
- ok -> ok;
- dropped -> dropped;
- _ -> dropped
+ ok ->
+ ok;
+ dropped ->
+ dropped;
+ _ ->
+ bump_counter(?CNT_ENQUEUE_FAILED),
+ dropped
catch
- throw:_Reason -> dropped;
- error:_Reason -> dropped;
- exit:_Reason -> dropped
+ throw:_Reason ->
+ bump_counter(?CNT_ENQUEUE_FAILED),
+ dropped;
+ error:_Reason ->
+ bump_counter(?CNT_ENQUEUE_FAILED),
+ dropped;
+ exit:Reason ->
+ count_enqueue_exit(Reason),
+ dropped
end.
+-spec count_enqueue_exit(term()) -> ok.
+count_enqueue_exit({timeout, _Call}) ->
+ bump_counter(?CNT_ENQUEUE_TIMEOUT);
+count_enqueue_exit(_Reason) ->
+ bump_counter(?CNT_ENQUEUE_FAILED).
+
-spec handle_enqueue(term(), state()) -> {ok | dropped, state()}.
handle_enqueue(Job0, State) ->
case push_job(Job0) of
- {ok, Job} -> maybe_enqueue_or_start(Job, State);
- error -> {dropped, State}
+ {ok, Job} ->
+ maybe_enqueue_or_start(Job, State);
+ error ->
+ bump_counter(?CNT_INVALID_JOB),
+ {dropped, State}
end.
-spec push_job(term()) -> {ok, push_job()} | error.
@@ -547,3 +740,135 @@ optional_binary(Value) when is_binary(Value) ->
{ok, Value};
optional_binary(_) ->
error.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+absent_counter_table_reads_unavailable_test() ->
+ delete_counter_table(),
+ ?assertEqual(unavailable, counter_table_status()),
+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)),
+ ?assertEqual(ok, bump_counter(?CNT_QUEUE_FULL)),
+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)).
+
+absent_counter_key_is_distinct_from_zero_test() ->
+ with_counter_table(fun() ->
+ ?assertEqual(live, counter_table_status()),
+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)),
+ ok = bump_counter(?CNT_RESTART_DISCARDED, 0),
+ ?assertEqual(0, read_counter(?CNT_RESTART_DISCARDED))
+ end).
+
+bump_counter_creates_then_increments_key_test() ->
+ with_counter_table(fun() ->
+ ok = bump_counter(?CNT_QUEUE_FULL),
+ ?assertEqual(1, read_counter(?CNT_QUEUE_FULL)),
+ ok = bump_counter(?CNT_QUEUE_FULL, 4),
+ ?assertEqual(5, read_counter(?CNT_QUEUE_FULL))
+ end).
+
+enqueue_exit_separates_timeout_from_failure_test() ->
+ with_counter_table(fun() ->
+ ok = count_enqueue_exit({timeout, {gen_server, call, []}}),
+ ok = count_enqueue_exit({noproc, {gen_server, call, []}}),
+ ?assertEqual(1, read_counter(?CNT_ENQUEUE_TIMEOUT)),
+ ?assertEqual(1, read_counter(?CNT_ENQUEUE_FAILED))
+ end).
+
+worker_down_counts_only_abnormal_exits_test() ->
+ with_counter_table(fun() ->
+ ok = count_worker_down(normal),
+ ?assertEqual(unavailable, read_counter(?CNT_WORKER_DIED)),
+ ok = count_worker_down(killed),
+ ok = count_worker_down({shutdown, restarting}),
+ ?assertEqual(2, read_counter(?CNT_WORKER_DIED))
+ end).
+
+terminate_counts_discarded_queue_depth_test() ->
+ with_counter_table(fun() ->
+ ?assertEqual(ok, terminate(shutdown, dispatcher_state(7, 10))),
+ ?assertEqual(7, read_counter(?CNT_RESTART_DISCARDED))
+ end).
+
+invalid_job_enqueue_is_counted_test() ->
+ with_counter_table(fun() ->
+ State = dispatcher_state(0, 10),
+ ?assertEqual({dropped, State}, handle_enqueue(#{type => invalid}, State)),
+ ?assertEqual(1, read_counter(?CNT_INVALID_JOB))
+ end).
+
+uptime_seconds_reports_undefined_without_start_time_test() ->
+ ?assertEqual(undefined, uptime_seconds(undefined)),
+ ?assertEqual(0, uptime_seconds(erlang:monotonic_time(second) + 5)),
+ ?assert(is_integer(uptime_seconds(erlang:monotonic_time(second) - 3))).
+
+abnormal_worker_down_also_counts_a_crashed_job_test() ->
+ with_counter_table(fun() ->
+ ok = count_worker_down(normal),
+ ?assertEqual(unavailable, read_counter(?CNT_JOB_CRASHED)),
+ ok = count_worker_down(killed),
+ ?assertEqual(1, read_counter(?CNT_JOB_CRASHED)),
+ ?assertEqual(1, read_counter(?CNT_WORKER_DIED))
+ end).
+
+enqueue_records_queue_growth_outside_the_process_test() ->
+ with_counter_table(fun() ->
+ {ok, State1} = maybe_enqueue(clear_job(), dispatcher_state(0, 10)),
+ ?assertEqual(1, maps:get(queued, State1)),
+ ?assertEqual(1, read_counter(?CNT_QUEUE_ENQUEUED)),
+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_DEQUEUED))
+ end).
+
+queue_full_drop_does_not_record_queue_growth_test() ->
+ with_counter_table(fun() ->
+ {dropped, _State1} = maybe_enqueue(clear_job(), dispatcher_state(3, 3)),
+ ?assertEqual(1, read_counter(?CNT_QUEUE_FULL)),
+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_ENQUEUED))
+ end).
+
+queue_backlog_lost_survives_an_untrappable_kill_test() ->
+ with_counter_table(fun() ->
+ ?assertEqual(unavailable, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 0)),
+ ok = bump_counter(?CNT_QUEUE_ENQUEUED, 7),
+ ok = bump_counter(?CNT_QUEUE_DEQUEUED, 2),
+ ?assertEqual(0, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 5)),
+ ?assertEqual(5, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 0)),
+ ?assertEqual(0, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 900))
+ end).
+
+dispatcher_state(Queued, MaxQueue) ->
+ #{
+ queue => queue:new(),
+ queued => Queued,
+ inflight => 0,
+ workers => #{},
+ max_inflight => 1,
+ max_queue => MaxQueue
+ }.
+
+clear_job() ->
+ #{
+ type => clear_channel,
+ user_id => 1,
+ channel_id => 2,
+ message_id => 3,
+ badge_counts_ttl_seconds => 0
+ }.
+
+with_counter_table(Fun) ->
+ delete_counter_table(),
+ _ = ets:new(?PUSH_COUNTER_TABLE, [named_table, public, set, {write_concurrency, true}]),
+ try
+ Fun()
+ after
+ delete_counter_table()
+ end.
+
+delete_counter_table() ->
+ try ets:delete(?PUSH_COUNTER_TABLE) of
+ _ -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+-endif.
diff --git a/fluxer_gateway/src/push/push_eligibility_checks.erl b/fluxer_gateway/src/push/push_eligibility_checks.erl
index 2d4ec072f..c13d6c20b 100644
--- a/fluxer_gateway/src/push/push_eligibility_checks.erl
+++ b/fluxer_gateway/src/push/push_eligibility_checks.erl
@@ -16,11 +16,12 @@
-export([override_for_large_guild/2]).
-export([enforce_only_mentions/1]).
-export([is_large_guild/2]).
+-export([large_guild_threshold/0]).
-export([has_large_guild_override/1]).
-export([get_guild_large_metadata/1]).
-export([check_temp_muted/1]).
--define(LARGE_GUILD_THRESHOLD, 250).
+-define(LARGE_GUILD_THRESHOLD, 2500).
-define(LARGE_GUILD_OVERRIDE_FEATURE, <<"LARGE_GUILD_OVERRIDE">>).
-define(MESSAGE_NOTIFICATIONS_NULL, -1).
-define(MESSAGE_NOTIFICATIONS_ALL, 0).
@@ -215,10 +216,14 @@ enforce_only_mentions(CurrentLevel) -> CurrentLevel.
-spec is_large_guild(integer() | term(), list()) -> boolean().
is_large_guild(Count, Features) when is_integer(Count) ->
- Count > ?LARGE_GUILD_THRESHOLD orelse has_large_guild_override(Features);
+ Count > large_guild_threshold() orelse has_large_guild_override(Features);
is_large_guild(_, Features) ->
has_large_guild_override(Features).
+-spec large_guild_threshold() -> pos_integer().
+large_guild_threshold() ->
+ ?LARGE_GUILD_THRESHOLD.
+
-spec has_large_guild_override(list() | term()) -> boolean().
has_large_guild_override(Features) when is_list(Features) ->
lists:member(?LARGE_GUILD_OVERRIDE_FEATURE, Features);
@@ -380,11 +385,27 @@ enforce_only_mentions_test() ->
?assertEqual(2, enforce_only_mentions(2)).
is_large_guild_test() ->
- ?assertEqual(true, is_large_guild(300, [])),
+ ?assertEqual(true, is_large_guild(3000, [])),
+ ?assertEqual(false, is_large_guild(300, [])),
+ ?assertEqual(true, is_large_guild(?LARGE_GUILD_THRESHOLD + 1, [])),
+ ?assertEqual(false, is_large_guild(?LARGE_GUILD_THRESHOLD, [])),
?assertEqual(false, is_large_guild(100, [])),
?assertEqual(true, is_large_guild(100, [<<"LARGE_GUILD_OVERRIDE">>])),
+ ?assertEqual(true, is_large_guild(300, [<<"LARGE_GUILD_OVERRIDE">>])),
?assertEqual(true, is_large_guild(undefined, [<<"LARGE_GUILD_OVERRIDE">>])).
+large_guild_threshold_is_the_production_value_test() ->
+ ?assertEqual(2500, ?LARGE_GUILD_THRESHOLD),
+ ?assertEqual(?LARGE_GUILD_THRESHOLD, large_guild_threshold()).
+
+mid_sized_guilds_keep_all_notifications_test() ->
+ MessageData = #{<<"channel_type">> => 0},
+ Metadata = #{member_count => 1000, features => []},
+ ?assertEqual(
+ true,
+ check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, Metadata)
+ ).
+
has_large_guild_override_test() ->
?assertEqual(true, has_large_guild_override([<<"LARGE_GUILD_OVERRIDE">>])),
?assertEqual(false, has_large_guild_override([<<"OTHER">>])),
@@ -392,7 +413,7 @@ has_large_guild_override_test() ->
cached_large_guild_metadata_overrides_all_notifications_test() ->
MessageData = #{<<"channel_type">> => 0},
- LargeMetadata = #{member_count => 300, features => []},
+ LargeMetadata = #{member_count => 3000, features => []},
?assertEqual(
false,
check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, LargeMetadata)
@@ -403,7 +424,7 @@ cached_large_guild_metadata_keeps_mentions_allowed_test() ->
<<"channel_type">> => 0,
<<"mentions">> => [#{<<"id">> => <<"100">>}]
},
- LargeMetadata = #{member_count => 300, features => []},
+ LargeMetadata = #{member_count => 3000, features => []},
?assertEqual(
true,
check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, LargeMetadata)
diff --git a/fluxer_gateway/src/push/push_ets_cache.erl b/fluxer_gateway/src/push/push_ets_cache.erl
index e36272826..9a45b4f7c 100644
--- a/fluxer_gateway/src/push/push_ets_cache.erl
+++ b/fluxer_gateway/src/push/push_ets_cache.erl
@@ -14,6 +14,7 @@
delete_subscriptions/1,
get_blocked_ids/1,
put_blocked_ids/2,
+ put_blocked_ids_fetched/2,
get_badge_count/1,
put_badge_count/3,
delete_badge_count/1,
@@ -31,6 +32,10 @@
-define(MAX_TABLE_ENTRIES, 500000).
-define(EVICT_BATCH, 4096).
+-define(MAX_EVICT_RESEEKS, 8).
+-define(DEFAULT_BLOCKED_IDS_TTL, 300).
+-define(MIN_BLOCKED_IDS_TTL, 30).
+-define(MAX_BLOCKED_IDS_TTL, 3600).
-define(ETS_OPTS, [
named_table, public, set, {read_concurrency, true}, {write_concurrency, true}
@@ -125,17 +130,54 @@ get_blocked_ids(UserId) ->
-spec get_blocked_ids_ets(integer()) -> [integer()] | undefined.
get_blocked_ids_ets(UserId) ->
try ets:lookup(?BLOCKED_IDS, UserId) of
- [{UserId, BlockedIds}] -> BlockedIds;
- _ -> undefined
+ [{UserId, BlockedIds}] ->
+ BlockedIds;
+ [{UserId, BlockedIds, infinity}] ->
+ BlockedIds;
+ [{UserId, BlockedIds, ExpiresAt}] when is_integer(ExpiresAt) ->
+ live_fetched_blocked_ids(BlockedIds, ExpiresAt);
+ _ ->
+ undefined
catch
error:badarg -> undefined
end.
+-spec live_fetched_blocked_ids([integer()], integer()) -> [integer()] | undefined.
+live_fetched_blocked_ids(BlockedIds, ExpiresAt) ->
+ case erlang:system_time(second) < ExpiresAt of
+ true -> BlockedIds;
+ false -> undefined
+ end.
+
-spec put_blocked_ids(integer(), [integer()]) -> ok.
put_blocked_ids(UserId, BlockedIds) ->
+ insert_blocked_ids(UserId, BlockedIds, infinity).
+
+-spec put_blocked_ids_fetched(integer(), [integer()]) -> ok.
+put_blocked_ids_fetched(UserId, BlockedIds) ->
+ ExpiresAt = erlang:system_time(second) + blocked_ids_ttl_seconds(),
+ insert_blocked_ids(UserId, BlockedIds, ExpiresAt).
+
+-spec insert_blocked_ids(integer(), [integer()], infinity | integer()) -> ok.
+insert_blocked_ids(UserId, BlockedIds, ExpiresAt) ->
guard_table_size(?BLOCKED_IDS),
- ets:insert(?BLOCKED_IDS, {UserId, BlockedIds}),
- ok.
+ try ets:insert(?BLOCKED_IDS, {UserId, BlockedIds, ExpiresAt}) of
+ _ -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+-spec blocked_ids_ttl_seconds() -> pos_integer().
+blocked_ids_ttl_seconds() ->
+ Value = app_pos_integer(push_blocked_ids_cache_ttl_seconds, ?DEFAULT_BLOCKED_IDS_TTL),
+ min(max(Value, ?MIN_BLOCKED_IDS_TTL), ?MAX_BLOCKED_IDS_TTL).
+
+-spec app_pos_integer(atom(), pos_integer()) -> pos_integer().
+app_pos_integer(Key, Default) ->
+ case application:get_env(fluxer_gateway, Key, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> Default
+ end.
-spec get_badge_count(integer()) -> {non_neg_integer(), integer()} | undefined.
get_badge_count(UserId) ->
@@ -233,12 +275,24 @@ cache_stats() ->
-spec evict_tables(map()) -> ok.
evict_tables(MaxEntries) ->
+ _ = expire_blocked_ids(),
evict_table(?USER_GUILD_SETTINGS, maps:get(user_guild_settings, MaxEntries, undefined)),
evict_table(?SUBSCRIPTIONS, maps:get(subscriptions, MaxEntries, undefined)),
evict_table(?BLOCKED_IDS, maps:get(blocked_ids, MaxEntries, undefined)),
evict_table(?BADGE_COUNTS, maps:get(badge_counts, MaxEntries, undefined)),
ok.
+-spec expire_blocked_ids() -> non_neg_integer().
+expire_blocked_ids() ->
+ Now = erlang:system_time(second),
+ MatchSpec = [{{'_', '_', '$1'}, [{is_integer, '$1'}, {'=<', '$1', Now}], [true]}],
+ try ets:select_delete(?BLOCKED_IDS, MatchSpec) of
+ Deleted when is_integer(Deleted) -> Deleted;
+ _ -> 0
+ catch
+ error:badarg -> 0
+ end.
+
-spec guard_table_size(atom()) -> ok.
guard_table_size(Table) ->
case table_size(Table) >= ?MAX_TABLE_ENTRIES of
@@ -279,20 +333,52 @@ evict_table(Table, MaxEntries) ->
case Size > MaxEntries of
true ->
ToDelete = Size - MaxEntries,
- evict_n(Table, ets:first(Table), ToDelete);
+ evict_scan(Table, ToDelete, ?MAX_EVICT_RESEEKS);
false ->
ok
end.
--spec evict_n(atom(), term(), non_neg_integer()) -> ok.
-evict_n(_Table, '$end_of_table', _N) ->
+-spec evict_scan(atom(), non_neg_integer(), non_neg_integer()) -> ok.
+evict_scan(_Table, 0, _Reseeks) ->
ok;
-evict_n(_Table, _Key, 0) ->
+evict_scan(_Table, _N, 0) ->
ok;
+evict_scan(Table, N, Reseeks) ->
+ case evict_n(Table, first_key(Table), N) of
+ {done, _Remaining} -> ok;
+ {interrupted, Remaining} -> evict_scan(Table, Remaining, Reseeks - 1)
+ end.
+
+-spec evict_n(atom(), term(), non_neg_integer()) ->
+ {done, non_neg_integer()} | {interrupted, non_neg_integer()}.
+evict_n(_Table, '$end_of_table', N) ->
+ {done, N};
+evict_n(_Table, _Key, 0) ->
+ {done, 0};
evict_n(Table, Key, N) ->
- Next = ets:next(Table, Key),
- ets:delete(Table, Key),
- evict_n(Table, Next, N - 1).
+ case next_key(Table, Key) of
+ {ok, Next} ->
+ safe_delete(Table, Key),
+ evict_n(Table, Next, N - 1);
+ {error, badarg} ->
+ {interrupted, N}
+ end.
+
+-spec first_key(atom()) -> term().
+first_key(Table) ->
+ try ets:first(Table) of
+ Key -> Key
+ catch
+ error:badarg -> '$end_of_table'
+ end.
+
+-spec next_key(atom(), term()) -> {ok, term()} | {error, badarg}.
+next_key(Table, Key) ->
+ try ets:next(Table, Key) of
+ Next -> {ok, Next}
+ catch
+ error:badarg -> {error, badarg}
+ end.
-spec rebalance_table(atom(), fun((term()) -> integer() | undefined)) -> non_neg_integer().
rebalance_table(Table, UserIdFun) ->
@@ -350,3 +436,67 @@ user_id_from_key(UserId) when is_integer(UserId) ->
UserId;
user_id_from_key(_) ->
undefined.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+synced_blocked_ids_are_stored_without_an_expiry_test() ->
+ init(),
+ try
+ ok = put_blocked_ids(4001, [4002]),
+ ?assertEqual([{4001, [4002], infinity}], ets:lookup(?BLOCKED_IDS, 4001))
+ after
+ safe_delete(?BLOCKED_IDS, 4001)
+ end.
+
+synced_blocked_ids_never_expire_test() ->
+ init(),
+ try
+ ok = put_blocked_ids(4002, [4003]),
+ ?assertEqual([4003], get_blocked_ids(4002)),
+ ok = evict_tables(#{}),
+ ?assertEqual([4003], get_blocked_ids(4002))
+ after
+ safe_delete(?BLOCKED_IDS, 4002)
+ end.
+
+fetched_blocked_ids_expire_and_are_reclaimed_test() ->
+ init(),
+ try
+ ok = put_blocked_ids_fetched(4004, [4005]),
+ ?assertEqual([4005], get_blocked_ids(4004)),
+ Stale = erlang:system_time(second) - 1,
+ true = ets:insert(?BLOCKED_IDS, {4004, [4005], Stale}),
+ ?assertEqual(undefined, get_blocked_ids(4004)),
+ ok = evict_tables(#{}),
+ ?assertEqual([], ets:lookup(?BLOCKED_IDS, 4004))
+ after
+ safe_delete(?BLOCKED_IDS, 4004)
+ end.
+
+legacy_untimed_blocked_ids_are_still_honoured_test() ->
+ init(),
+ try
+ true = ets:insert(?BLOCKED_IDS, {4008, [4009]}),
+ ?assertEqual([4009], get_blocked_ids(4008)),
+ ok = evict_tables(#{}),
+ ?assertEqual([4009], get_blocked_ids(4008))
+ after
+ safe_delete(?BLOCKED_IDS, 4008)
+ end.
+
+blocked_ids_ttl_is_clamped_test() ->
+ ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, blocked_ids_ttl_seconds()),
+ ?assertEqual(?MIN_BLOCKED_IDS_TTL, with_ttl_env(1, fun blocked_ids_ttl_seconds/0)),
+ ?assertEqual(?MAX_BLOCKED_IDS_TTL, with_ttl_env(999999, fun blocked_ids_ttl_seconds/0)),
+ ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, with_ttl_env(0, fun blocked_ids_ttl_seconds/0)).
+
+with_ttl_env(Value, Fun) ->
+ application:set_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds, Value),
+ try
+ Fun()
+ after
+ application:unset_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds)
+ end.
+
+-endif.
diff --git a/fluxer_gateway/src/push/push_fcm_payload.erl b/fluxer_gateway/src/push/push_fcm_payload.erl
index 5296254f8..9796c03b0 100644
--- a/fluxer_gateway/src/push/push_fcm_payload.erl
+++ b/fluxer_gateway/src/push/push_fcm_payload.erl
@@ -40,9 +40,9 @@ build_clear_message(DeviceToken, Payload) ->
<<"token">> => DeviceToken,
<<"data">> => Data,
<<"android">> => #{
- <<"priority">> => <<"HIGH">>,
- <<"ttl">> => <<"300s">>,
- <<"collapse_key">> => Tag
+ <<"priority">> => <<"NORMAL">>,
+ <<"ttl">> => <<"3600s">>,
+ <<"collapse_key">> => <<"clear:", Tag/binary>>
},
<<"fcm_options">> => #{
<<"analytics_label">> => <<"notification_clear">>
@@ -63,9 +63,8 @@ build_notification_message(DeviceToken, Payload) ->
<<"image">>, ImageUrl, #{<<"title">> => Title, <<"body">> => Body}
),
Data = build_notification_data(Payload, Title, Body, Tag, ImageUrl),
- Group = resolve_notification_group(maps:get(<<"data">>, Payload, #{}), Tag),
AndroidNotification = build_android_notification(Tag, ImageUrl),
- wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification, Group).
+ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification).
-spec build_android_notification(binary(), binary() | undefined) -> map().
build_android_notification(Tag, ImageUrl) ->
@@ -75,8 +74,8 @@ build_android_notification(Tag, ImageUrl) ->
<<"click_action">> => <<"FLUXER_MESSAGE">>
}).
--spec wrap_notification_message(binary(), map(), map(), map(), binary()) -> map().
-wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification, Group) ->
+-spec wrap_notification_message(binary(), map(), map(), map()) -> map().
+wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification) ->
#{
<<"message">> => #{
<<"token">> => DeviceToken,
@@ -85,7 +84,6 @@ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotificati
<<"android">> => #{
<<"priority">> => <<"HIGH">>,
<<"ttl">> => <<"86400s">>,
- <<"collapse_key">> => Group,
<<"notification">> => AndroidNotification
},
<<"fcm_options">> => #{
@@ -94,41 +92,6 @@ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotificati
}
}.
--spec resolve_notification_group(map(), binary()) -> binary().
-resolve_notification_group(Data, MessageTag) ->
- case
- push_utils:normalize_binary(
- maps:get(<<"notification_tag">>, Data, undefined), undefined
- )
- of
- Group when is_binary(Group), byte_size(Group) > 0 ->
- Group;
- _ ->
- resolve_channel_notification_group(Data, MessageTag)
- end.
-
--spec resolve_channel_notification_group(map(), binary()) -> binary().
-resolve_channel_notification_group(Data, MessageTag) ->
- case
- push_utils:normalize_binary(
- maps:get(<<"channel_id">>, Data, undefined), undefined
- )
- of
- ChannelId when is_binary(ChannelId), byte_size(ChannelId) > 0 ->
- <<"channel:", ChannelId/binary>>;
- _ ->
- derive_channel_tag_from_message_tag(MessageTag)
- end.
-
--spec derive_channel_tag_from_message_tag(binary()) -> binary().
-derive_channel_tag_from_message_tag(MessageTag) ->
- case binary:split(MessageTag, <<":">>, [global]) of
- [<<"channel">>, ChannelId, _MessageId] when byte_size(ChannelId) > 0 ->
- <<"channel:", ChannelId/binary>>;
- _ ->
- MessageTag
- end.
-
-spec build_notification_data(map(), binary(), binary(), binary(), binary() | undefined) ->
map().
build_notification_data(Payload, Title, Body, Tag, ImageUrl) ->
@@ -349,7 +312,7 @@ build_message_includes_android_chat_notification_fields_test() ->
?assertEqual(<<"fluxer_default_push">>, maps:get(<<"channel_id">>, AndroidNotification)),
?assertEqual(<<"channel:123:456">>, maps:get(<<"tag">>, AndroidNotification)),
Android = maps:get(<<"android">>, Message),
- ?assertEqual(<<"channel:123">>, maps:get(<<"collapse_key">>, Android)),
+ ?assertEqual(false, maps:is_key(<<"collapse_key">>, Android)),
?assertEqual(
<<"https://cdn.example/image.png">>, maps:get(<<"image">>, AndroidNotification)
),
@@ -370,9 +333,9 @@ build_clear_message_is_data_only_and_collapsible_test() ->
#{<<"message">> := Message} = build_message(<<"device-token">>, Payload),
?assertEqual(false, maps:is_key(<<"notification">>, Message)),
Android = maps:get(<<"android">>, Message),
- ?assertEqual(<<"HIGH">>, maps:get(<<"priority">>, Android)),
- ?assertEqual(<<"300s">>, maps:get(<<"ttl">>, Android)),
- ?assertEqual(<<"channel:123">>, maps:get(<<"collapse_key">>, Android)),
+ ?assertEqual(<<"NORMAL">>, maps:get(<<"priority">>, Android)),
+ ?assertEqual(<<"3600s">>, maps:get(<<"ttl">>, Android)),
+ ?assertEqual(<<"clear:channel:123">>, maps:get(<<"collapse_key">>, Android)),
Data = maps:get(<<"data">>, Message),
?assertEqual(<<"notification_clear">>, maps:get(<<"type">>, Data)),
?assertEqual(<<"clear_channel">>, maps:get(<<"action">>, Data)),
diff --git a/fluxer_gateway/src/push/push_message_params.erl b/fluxer_gateway/src/push/push_message_params.erl
index a328dcad7..846b40ee3 100644
--- a/fluxer_gateway/src/push/push_message_params.erl
+++ b/fluxer_gateway/src/push/push_message_params.erl
@@ -7,6 +7,8 @@
-export_type([context/0]).
+-define(MAX_GUILD_FEATURES, 64).
+
-type context() :: #{
message_data := map(),
user_ids := [pos_integer()],
@@ -20,7 +22,8 @@
role_names := map(),
user_roles := map(),
connected_users := map(),
- markdown_context := map()
+ markdown_context := map(),
+ large_guild_metadata := map() | undefined
}.
-spec context(map()) -> {ok, context()} | {error, term()}.
@@ -58,9 +61,34 @@ context_from_message_data(Params, MessageData) ->
connected_users => maps:get(connected_users, Params, #{}),
markdown_context => markdown_context(
MessageData, GuildId, RoleNames, maps:get(markdown_context, Params, undefined)
- )
+ ),
+ large_guild_metadata => large_guild_metadata(Params)
}).
+-spec large_guild_metadata(map()) -> map() | undefined.
+large_guild_metadata(Params) ->
+ MemberCount = maps:get(guild_member_count, Params, undefined),
+ Features = maps:get(guild_features, Params, undefined),
+ build_large_guild_metadata(MemberCount, Features).
+
+-spec build_large_guild_metadata(term(), term()) -> map() | undefined.
+build_large_guild_metadata(MemberCount, Features) when
+ is_integer(MemberCount), MemberCount >= 0, is_list(Features)
+->
+ #{member_count => MemberCount, features => bounded_features(Features, [], 0)};
+build_large_guild_metadata(_MemberCount, _Features) ->
+ undefined.
+
+-spec bounded_features(term(), [binary()], non_neg_integer()) -> [binary()].
+bounded_features(_Features, Acc, ?MAX_GUILD_FEATURES) ->
+ lists:reverse(Acc);
+bounded_features([Feature | Rest], Acc, Count) when is_binary(Feature) ->
+ bounded_features(Rest, [Feature | Acc], Count + 1);
+bounded_features([_Feature | Rest], Acc, Count) ->
+ bounded_features(Rest, Acc, Count);
+bounded_features(_Features, Acc, _Count) ->
+ lists:reverse(Acc).
+
-spec owner_key(map()) -> term().
owner_key(Params) ->
case user_ids(maps:get(user_ids, Params, [])) of
diff --git a/fluxer_gateway/src/push/push_notification.erl b/fluxer_gateway/src/push/push_notification.erl
index 2bd103511..71fa239ec 100644
--- a/fluxer_gateway/src/push/push_notification.erl
+++ b/fluxer_gateway/src/push/push_notification.erl
@@ -133,7 +133,7 @@ assemble_payload(
ContentPreview,
AuthorAvatarUrl
) ->
- Data = build_data(Ctx),
+ Data = build_data(Ctx, AuthorAvatarUrl),
Notification = build_notification_body(Ctx, Title, ContentPreview, AuthorAvatarUrl, Data),
maps:merge(
#{
@@ -149,19 +149,23 @@ assemble_payload(
ImageFields
).
--spec build_data(push_ctx()) -> map().
-build_data(#{
- channel_id := ChannelId,
- message_id := MessageId,
- guild_id := GuildId,
- navigate_url := NavigateUrl,
- badge_value := BadgeValue,
- target_user_id := TargetUserId,
- image_url := ImageUrl,
- image_fields := ImageFields
-}) ->
+-spec build_data(push_ctx(), binary()) -> map().
+build_data(
+ #{
+ channel_id := ChannelId,
+ message_id := MessageId,
+ guild_id := GuildId,
+ navigate_url := NavigateUrl,
+ badge_value := BadgeValue,
+ target_user_id := TargetUserId,
+ image_url := ImageUrl,
+ image_fields := ImageFields
+ },
+ AuthorAvatarUrl
+) ->
BaseData = #{
<<"channel_id">> => integer_to_binary(ChannelId),
+ <<"author_avatar_url">> => AuthorAvatarUrl,
<<"message_id">> => integer_to_binary(MessageId),
<<"notification_tag">> => build_channel_tag(ChannelId),
<<"guild_id">> =>
diff --git a/fluxer_gateway/src/push/push_notification_format.erl b/fluxer_gateway/src/push/push_notification_format.erl
index 1fbcb8b3d..a9d2c6097 100644
--- a/fluxer_gateway/src/push/push_notification_format.erl
+++ b/fluxer_gateway/src/push/push_notification_format.erl
@@ -326,9 +326,18 @@ first_nonempty_binary([Value | Rest]) ->
-spec truncate_preview(binary()) -> binary().
truncate_preview(Content) when byte_size(Content) > 100 ->
- binary:part(Content, 0, 100);
+ valid_utf8_prefix(binary:part(Content, 0, 100));
truncate_preview(Content) ->
- Content.
+ valid_utf8_prefix(Content).
+
+-spec valid_utf8_prefix(binary()) -> binary().
+valid_utf8_prefix(Content) ->
+ case unicode:characters_to_binary(Content, utf8, utf8) of
+ Valid when is_binary(Valid) -> Valid;
+ {incomplete, Valid, _Rest} when is_binary(Valid) -> Valid;
+ {error, Valid, _Rest} when is_binary(Valid) -> Valid;
+ _ -> <<>>
+ end.
-spec build_content_fallback_preview(map()) -> binary().
build_content_fallback_preview(MessageData) ->
@@ -568,6 +577,23 @@ lowercase_binary(Value) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
+truncate_preview_never_splits_a_utf8_character_test() ->
+ Emoji = <<"\xF0\x9F\x98\x80">>,
+ Content = <<(binary:copy(<<"a">>, 98))/binary, Emoji/binary>>,
+ ?assert(byte_size(Content) > 100),
+ Truncated = truncate_preview(Content),
+ ?assert(byte_size(Truncated) =< 100),
+ ?assertMatch(Bin when is_binary(Bin), unicode:characters_to_binary(Truncated, utf8, utf8)),
+ ?assertEqual(binary:copy(<<"a">>, 98), Truncated).
+
+truncate_preview_is_unchanged_for_valid_ascii_test() ->
+ Content = binary:copy(<<"a">>, 150),
+ ?assertEqual(binary:copy(<<"a">>, 100), truncate_preview(Content)).
+
+truncate_preview_keeps_short_valid_content_identical_test() ->
+ Content = <<"hello \xC3\xA9 world">>,
+ ?assertEqual(Content, truncate_preview(Content)).
+
build_url_dm_test() ->
?assertEqual(<<"/channels/@me/456/789">>, build_url(0, 456, 789)).
diff --git a/fluxer_gateway/src/push/push_sender.erl b/fluxer_gateway/src/push/push_sender.erl
index 3726a801a..7d7c05bcd 100644
--- a/fluxer_gateway/src/push/push_sender.erl
+++ b/fluxer_gateway/src/push/push_sender.erl
@@ -13,6 +13,11 @@
-export_type([send_context/0]).
+-define(DEFAULT_BADGE_FETCH_BATCH, 2000).
+-define(BADGE_FETCH_MAX_CONSECUTIVE_FAILURES, 3).
+-define(DEFAULT_BADGE_FETCH_BUDGET_MS, 120000).
+-define(PUSH_COUNTERS, push_worker_counter).
+
-type send_context() :: #{
message_data := map(),
guild_id := integer(),
@@ -257,16 +262,128 @@ check_badge_cache(UserId, TTL, Now, Acc, MissingAcc) ->
-spec fetch_badge_counts([integer()], map(), integer()) -> map().
fetch_badge_counts(UserIds, Counts, CachedAt) ->
+ Batches = chunk_badge_user_ids(UserIds, badge_fetch_batch_size(), []),
+ {Counted, FailedBatches, DefaultedUsers, _Consecutive} =
+ fetch_badge_count_batches(Batches, {Counts, 0, 0, 0}, CachedAt),
+ report_badge_fetch_failures(FailedBatches, DefaultedUsers),
+ Counted.
+
+-type badge_batch_acc() :: {map(), non_neg_integer(), non_neg_integer(), non_neg_integer()}.
+
+-spec fetch_badge_count_batches([[integer()]], badge_batch_acc(), integer()) ->
+ badge_batch_acc().
+fetch_badge_count_batches(Batches, Acc, CachedAt) ->
+ Deadline = erlang:monotonic_time(millisecond) + badge_fetch_budget_ms(),
+ fetch_badge_count_batches(Batches, Acc, CachedAt, Deadline).
+
+-spec fetch_badge_count_batches([[integer()]], badge_batch_acc(), integer(), integer()) ->
+ badge_batch_acc().
+fetch_badge_count_batches([], Acc, _CachedAt, _Deadline) ->
+ Acc;
+fetch_badge_count_batches(Remaining, Acc, CachedAt, Deadline) ->
+ case badge_fetch_exhausted(Acc, Deadline) of
+ true -> abandon_badge_batches(Remaining, Acc);
+ false -> fetch_next_badge_batch(Remaining, Acc, CachedAt, Deadline)
+ end.
+
+-spec badge_fetch_exhausted(badge_batch_acc(), integer()) -> boolean().
+badge_fetch_exhausted({_Counts, _FailedBatches, _DefaultedUsers, Consecutive}, Deadline) ->
+ Consecutive >= ?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES orelse
+ erlang:monotonic_time(millisecond) >= Deadline.
+
+-spec abandon_badge_batches([[integer()]], badge_batch_acc()) -> badge_batch_acc().
+abandon_badge_batches(Remaining, {Counts, FailedBatches, DefaultedUsers, Consecutive}) ->
+ {Counts, FailedBatches + length(Remaining),
+ DefaultedUsers + badge_batched_user_count(Remaining, 0), Consecutive}.
+
+-spec badge_batched_user_count([[integer()]], non_neg_integer()) -> non_neg_integer().
+badge_batched_user_count([], Acc) ->
+ Acc;
+badge_batched_user_count([Batch | Rest], Acc) ->
+ badge_batched_user_count(Rest, Acc + length(Batch)).
+
+-spec fetch_next_badge_batch([[integer()]], badge_batch_acc(), integer(), integer()) ->
+ badge_batch_acc().
+fetch_next_badge_batch(
+ [Batch | Rest], {Counts, FailedBatches, DefaultedUsers, Consecutive}, CachedAt, Deadline
+) ->
Request = #{
<<"type">> => <<"get_badge_counts">>,
- <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]
+ <<"user_ids">> => [integer_to_binary(UserId) || UserId <- Batch]
},
case rpc_client:call(Request) of
{ok, Data} ->
BadgeData = maps:get(<<"badge_counts">>, Data, #{}),
- merge_badge_data(UserIds, BadgeData, Counts, CachedAt);
+ Merged = merge_badge_data(Batch, BadgeData, Counts, CachedAt),
+ fetch_badge_count_batches(
+ Rest, {Merged, FailedBatches, DefaultedUsers, 0}, CachedAt, Deadline
+ );
{error, _Reason} ->
- Counts
+ fetch_badge_count_batches(
+ Rest,
+ {Counts, FailedBatches + 1, DefaultedUsers + length(Batch), Consecutive + 1},
+ CachedAt,
+ Deadline
+ )
+ end.
+
+-spec report_badge_fetch_failures(non_neg_integer(), non_neg_integer()) -> ok.
+report_badge_fetch_failures(0, _DefaultedUsers) ->
+ ok;
+report_badge_fetch_failures(FailedBatches, DefaultedUsers) ->
+ bump_counter(badge_fetch_calls_failed, FailedBatches),
+ bump_counter(badge_fetch_users_defaulted, DefaultedUsers),
+ logger:warning("Push: badge count batches failed; users default to zero badge", #{
+ failed_batches => FailedBatches, users_defaulted => DefaultedUsers
+ }).
+
+-spec bump_counter(atom(), integer()) -> ok.
+bump_counter(Key, Delta) ->
+ try
+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),
+ ok
+ catch
+ error:badarg -> seed_and_bump_counter(Key, Delta)
+ end.
+
+-spec seed_and_bump_counter(atom(), integer()) -> ok.
+seed_and_bump_counter(Key, Delta) ->
+ try
+ _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}),
+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),
+ ok
+ catch
+ error:badarg -> ok
+ end.
+
+-spec chunk_badge_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].
+chunk_badge_user_ids([], _BatchSize, Acc) ->
+ lists:reverse(Acc);
+chunk_badge_user_ids(UserIds, BatchSize, Acc) ->
+ {Batch, Rest} = take_badge_user_id_batch(UserIds, BatchSize, []),
+ chunk_badge_user_ids(Rest, BatchSize, [Batch | Acc]).
+
+-spec take_badge_user_id_batch([integer()], non_neg_integer(), [integer()]) ->
+ {[integer()], [integer()]}.
+take_badge_user_id_batch(Rest, 0, Acc) ->
+ {lists:reverse(Acc), Rest};
+take_badge_user_id_batch([], _Remaining, Acc) ->
+ {lists:reverse(Acc), []};
+take_badge_user_id_batch([UserId | Rest], Remaining, Acc) ->
+ take_badge_user_id_batch(Rest, Remaining - 1, [UserId | Acc]).
+
+-spec badge_fetch_batch_size() -> pos_integer().
+badge_fetch_batch_size() ->
+ case application:get_env(fluxer_gateway, push_badge_fetch_batch_size, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> ?DEFAULT_BADGE_FETCH_BATCH
+ end.
+
+-spec badge_fetch_budget_ms() -> pos_integer().
+badge_fetch_budget_ms() ->
+ case application:get_env(fluxer_gateway, push_badge_fetch_budget_ms, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> ?DEFAULT_BADGE_FETCH_BUDGET_MS
end.
-spec merge_badge_data([integer()], map(), map(), integer()) -> map().
@@ -285,3 +402,119 @@ merge_badge_data(UserIds, BadgeData, Counts, CachedAt) ->
-spec normalize_badge_count(integer() | term()) -> non_neg_integer().
normalize_badge_count(Value) when is_integer(Value), Value >= 0 -> Value;
normalize_badge_count(_) -> 0.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+chunk_badge_user_ids_uses_bounded_batches_test() ->
+ ?assertEqual([[1, 2], [3, 4], [5]], chunk_badge_user_ids([1, 2, 3, 4, 5], 2, [])),
+ ?assertEqual([], chunk_badge_user_ids([], 2, [])).
+
+fetch_badge_counts_in_batches_keeps_successful_batches_test() ->
+ ok = meck:new(rpc_client, [passthrough, no_link]),
+ ok = meck:new(push_ets_cache, [passthrough, no_link]),
+ application:set_env(fluxer_gateway, push_badge_fetch_batch_size, 2),
+ try
+ ok = meck:expect(push_ets_cache, put_badge_count, fun(_UserId, _Count, _At) -> ok end),
+ ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) ->
+ case Ids of
+ [<<"1">>, <<"2">>] ->
+ {ok, #{<<"badge_counts">> => #{<<"1">> => 3, <<"2">> => 4}}};
+ _ ->
+ {error, unavailable}
+ end
+ end),
+ ?assertEqual(#{1 => 3, 2 => 4}, fetch_badge_counts([1, 2, 3, 4], #{}, 0))
+ after
+ application:unset_env(fluxer_gateway, push_badge_fetch_batch_size),
+ meck:unload(push_ets_cache),
+ meck:unload(rpc_client)
+ end.
+
+fetch_badge_count_batches_stops_after_consecutive_failures_test() ->
+ ok = meck:new(rpc_client, [passthrough, no_link]),
+ try
+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),
+ Batches = [[N] || N <- lists:seq(1, 10)],
+ ?assertEqual(
+ {#{}, 10, 10, ?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES},
+ fetch_badge_count_batches(Batches, {#{}, 0, 0, 0}, 0)
+ ),
+ ?assertEqual(?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES, length(meck:history(rpc_client)))
+ after
+ meck:unload(rpc_client)
+ end.
+
+fetch_badge_count_batches_stops_at_the_wall_clock_budget_test() ->
+ ok = meck:new(rpc_client, [passthrough, no_link]),
+ application:set_env(fluxer_gateway, push_badge_fetch_budget_ms, 1),
+ try
+ ok = meck:expect(rpc_client, call, fun(_Req) ->
+ {ok, #{<<"badge_counts">> => #{}}}
+ end),
+ Batches = [[N] || N <- lists:seq(1, 10)],
+ Deadline = erlang:monotonic_time(millisecond) - 1,
+ ?assertEqual(
+ {#{}, 10, 10, 0},
+ fetch_badge_count_batches(Batches, {#{}, 0, 0, 0}, 0, Deadline)
+ ),
+ ?assertEqual(0, length(meck:history(rpc_client)))
+ after
+ application:unset_env(fluxer_gateway, push_badge_fetch_budget_ms),
+ meck:unload(rpc_client)
+ end.
+
+badge_fetch_defaults_missing_users_to_zero_test() ->
+ ok = meck:new(rpc_client, [passthrough, no_link]),
+ ok = meck:new(push_ets_cache, [passthrough, no_link]),
+ try
+ ok = meck:expect(push_ets_cache, put_badge_count, fun(_UserId, _Count, _At) -> ok end),
+ ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) ->
+ ?assertEqual([<<"1">>, <<"2">>, <<"3">>], Ids),
+ {ok, #{<<"badge_counts">> => #{<<"1">> => 1}}}
+ end),
+ ?assertEqual(#{1 => 1, 2 => 0, 3 => 0}, fetch_badge_counts([1, 2, 3], #{}, 0))
+ after
+ meck:unload(push_ets_cache),
+ meck:unload(rpc_client)
+ end.
+
+badge_fetch_failure_counts_defaulted_users_test() ->
+ ok = meck:new(rpc_client, [passthrough, no_link]),
+ ensure_test_counter_table(),
+ ok = reset_test_counter(badge_fetch_calls_failed),
+ ok = reset_test_counter(badge_fetch_users_defaulted),
+ try
+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),
+ ?assertEqual(#{}, fetch_badge_counts([1, 2, 3], #{}, 0)),
+ ?assertEqual(1, counter_value(badge_fetch_calls_failed)),
+ ?assertEqual(3, counter_value(badge_fetch_users_defaulted))
+ after
+ meck:unload(rpc_client)
+ end.
+
+counter_value(Key) ->
+ try ets:lookup(?PUSH_COUNTERS, Key) of
+ [{Key, Value}] when is_integer(Value) -> Value;
+ _ -> 0
+ catch
+ error:badarg -> 0
+ end.
+
+ensure_test_counter_table() ->
+ case ets:info(?PUSH_COUNTERS, name) of
+ undefined ->
+ _ = ets:new(?PUSH_COUNTERS, [named_table, public, set, {write_concurrency, true}]),
+ ok;
+ _ ->
+ ok
+ end.
+
+reset_test_counter(Key) ->
+ try ets:insert(?PUSH_COUNTERS, {Key, 0}) of
+ _ -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+-endif.
diff --git a/fluxer_gateway/src/push/push_sender_delivery.erl b/fluxer_gateway/src/push/push_sender_delivery.erl
index 73ab037f6..e0f7d52b3 100644
--- a/fluxer_gateway/src/push/push_sender_delivery.erl
+++ b/fluxer_gateway/src/push/push_sender_delivery.erl
@@ -561,7 +561,11 @@ maybe_retry_transient_waits_before_retry_test() ->
fun retry_request_meck/6
),
?assertEqual(false, maybe_retry_transient(Ctx, RecordSize, 0, {error, timeout})),
- ?assertEqual({retry_wait, ?BASE_RETRY_DELAY_MS}, receive_retry_wait()),
+ %% retry_delay/1 adds rand:uniform(Base div 4) of jitter, so the first
+ %% retry is anywhere in [Base, Base + Base div 4 - 1].
+ {retry_wait, DelayMs} = receive_retry_wait(),
+ ?assert(DelayMs >= ?BASE_RETRY_DELAY_MS),
+ ?assert(DelayMs =< ?BASE_RETRY_DELAY_MS + (?BASE_RETRY_DELAY_MS div 4) - 1),
?assertEqual(ok, receive_retried_push_request(Endpoint)),
?assert(meck:validate(gateway_retry_timer)),
?assert(meck:validate(push_utils)),
diff --git a/fluxer_gateway/src/push/push_subscriptions.erl b/fluxer_gateway/src/push/push_subscriptions.erl
index 5e3193651..c18e168d5 100644
--- a/fluxer_gateway/src/push/push_subscriptions.erl
+++ b/fluxer_gateway/src/push/push_subscriptions.erl
@@ -11,10 +11,18 @@
-export([fetch_and_cache_user_guild_settings/2]).
-export([delete_failed_subscriptions/1]).
-export([delivery_concurrency/0]).
+-export([delivery_stats/0]).
-define(DEFAULT_DELIVERY_CONCURRENCY, 8).
--define(DELIVERY_WORKER_TIMEOUT_MS, 30000).
--define(SUBSCRIPTION_FETCH_CHUNK_SIZE, 256).
+-define(DEFAULT_DELIVERY_MAX_WORKERS, 64).
+-define(DEFAULT_DELIVERY_TASKS_PER_WORKER, 128).
+-define(DEFAULT_SUBSCRIPTION_FETCH_BATCH, 2000).
+-define(SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES, 3).
+-define(DEFAULT_SUBSCRIPTION_FETCH_BUDGET_MS, 120000).
+-define(DELIVERY_IDLE_TIMEOUT_MS, 30000).
+-define(PUSH_COUNTERS, push_worker_counter).
+-define(CNT_LIVE_WORKERS, delivery_live_workers).
+-define(CTX_BADGE_COUNTS, 9).
-spec fetch_and_send_subscriptions(
[integer()],
@@ -92,60 +100,118 @@ send_missing_if_any(MissingUserIds, Ctx) ->
-spec fetch_and_send_missing_subscriptions([integer()], tuple()) -> ok.
fetch_and_send_missing_subscriptions(UserIds, Ctx) ->
- {FetchedUserIds, SubscriptionsData} = fetch_missing_subscriptions(UserIds),
- send_fetched_subscriptions(FetchedUserIds, SubscriptionsData, Ctx).
-
--spec fetch_missing_subscriptions([integer()]) -> {[integer()], map()}.
-fetch_missing_subscriptions(UserIds) ->
- Chunks = chunk_user_ids(UserIds, ?SUBSCRIPTION_FETCH_CHUNK_SIZE, []),
- {FetchedChunks, SubscriptionsData} = lists:foldl(
- fun fetch_subscriptions_chunk/2,
- {[], #{}},
- Chunks
- ),
- {lists:append(lists:reverse(FetchedChunks)), SubscriptionsData}.
+ Batches = chunk_user_ids(UserIds, subscription_fetch_batch_size(), []),
+ logger:debug("Push: fetching subscriptions via batched RPC", #{
+ user_count => length(UserIds), batch_count => length(Batches)
+ }),
+ {Tasks, FailedBatches, FailedUsers, _Consecutive} =
+ fetch_subscription_batches(Batches, {[], 0, 0, 0}),
+ report_subscription_fetch_failures(FailedBatches, FailedUsers, Ctx),
+ deliver_subscription_tasks(lists:reverse(Tasks), Ctx).
--spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].
-chunk_user_ids([], _ChunkSize, Acc) ->
- lists:reverse(Acc);
-chunk_user_ids(UserIds, ChunkSize, Acc) when length(UserIds) =< ChunkSize ->
- lists:reverse([UserIds | Acc]);
-chunk_user_ids(UserIds, ChunkSize, Acc) ->
- {Chunk, Rest} = lists:split(ChunkSize, UserIds),
- chunk_user_ids(Rest, ChunkSize, [Chunk | Acc]).
-
--spec fetch_subscriptions_chunk([integer()], {[[integer()]], map()}) -> {[[integer()]], map()}.
-fetch_subscriptions_chunk(UserIds, {FetchedAcc, DataAcc}) ->
- SubscriptionsReq = #{
+-type subscription_batch_acc() ::
+ {[{integer(), list()}], non_neg_integer(), non_neg_integer(), non_neg_integer()}.
+
+-spec fetch_subscription_batches([[integer()]], subscription_batch_acc()) ->
+ subscription_batch_acc().
+fetch_subscription_batches(Batches, Acc) ->
+ Deadline = erlang:monotonic_time(millisecond) + subscription_fetch_budget_ms(),
+ fetch_subscription_batches(Batches, Acc, Deadline).
+
+-spec fetch_subscription_batches([[integer()]], subscription_batch_acc(), integer()) ->
+ subscription_batch_acc().
+fetch_subscription_batches([], Acc, _Deadline) ->
+ Acc;
+fetch_subscription_batches(Remaining, Acc, Deadline) ->
+ case subscription_fetch_exhausted(Acc, Deadline) of
+ true -> abandon_subscription_batches(Remaining, Acc);
+ false -> fetch_next_subscription_batch(Remaining, Acc, Deadline)
+ end.
+
+-spec subscription_fetch_exhausted(subscription_batch_acc(), integer()) -> boolean().
+subscription_fetch_exhausted({_Tasks, _FailedBatches, _FailedUsers, Consecutive}, Deadline) ->
+ Consecutive >= ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES orelse
+ erlang:monotonic_time(millisecond) >= Deadline.
+
+-spec abandon_subscription_batches([[integer()]], subscription_batch_acc()) ->
+ subscription_batch_acc().
+abandon_subscription_batches(Remaining, {Tasks, FailedBatches, FailedUsers, Consecutive}) ->
+ {Tasks, FailedBatches + length(Remaining), FailedUsers + batched_user_count(Remaining, 0),
+ Consecutive}.
+
+-spec fetch_next_subscription_batch([[integer()]], subscription_batch_acc(), integer()) ->
+ subscription_batch_acc().
+fetch_next_subscription_batch(
+ [Batch | Rest], {Tasks, FailedBatches, FailedUsers, Consecutive}, Deadline
+) ->
+ Req = #{
<<"type">> => <<"get_push_subscriptions">>,
- <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]
+ <<"user_ids">> => [integer_to_binary(UserId) || UserId <- Batch]
},
- logger:debug("Push: fetching subscriptions via RPC", #{user_count => length(UserIds)}),
- case rpc_client:call(SubscriptionsReq) of
- {ok, SubscriptionsData} ->
- logger:debug("Push: RPC returned subscriptions", #{
- user_count => length(UserIds),
- response_keys => maps:keys(SubscriptionsData)
- }),
- {[UserIds | FetchedAcc], maps:merge(DataAcc, SubscriptionsData)};
+ case rpc_client:call(Req) of
+ {ok, BatchData} ->
+ BatchTasks = lists:foldl(
+ fun(UserId, Acc) ->
+ add_fetched_user_subscription_task(UserId, BatchData, Acc)
+ end,
+ Tasks,
+ Batch
+ ),
+ fetch_subscription_batches(
+ Rest, {BatchTasks, FailedBatches, FailedUsers, 0}, Deadline
+ );
{error, Reason} ->
logger:debug(
"Push: RPC failed to fetch subscriptions",
- #{user_count => length(UserIds), reason => Reason}
+ #{user_count => length(Batch), reason => Reason}
),
- {FetchedAcc, DataAcc}
+ fetch_subscription_batches(
+ Rest,
+ {Tasks, FailedBatches + 1, FailedUsers + length(Batch), Consecutive + 1},
+ Deadline
+ )
end.
--spec send_fetched_subscriptions([integer()], map(), tuple()) -> ok.
-send_fetched_subscriptions(UserIds, SubscriptionsData, Ctx) ->
- Tasks = lists:foldl(
- fun(UserId, Acc) ->
- add_fetched_user_subscription_task(UserId, SubscriptionsData, Acc)
- end,
- [],
- UserIds
- ),
- send_subscription_tasks(lists:reverse(Tasks), Ctx).
+-spec batched_user_count([[integer()]], non_neg_integer()) -> non_neg_integer().
+batched_user_count([], Acc) ->
+ Acc;
+batched_user_count([Batch | Rest], Acc) ->
+ batched_user_count(Rest, Acc + length(Batch)).
+
+-spec report_subscription_fetch_failures(non_neg_integer(), non_neg_integer(), tuple()) -> ok.
+report_subscription_fetch_failures(0, _FailedUsers, _Ctx) ->
+ ok;
+report_subscription_fetch_failures(FailedBatches, FailedUsers, Ctx) ->
+ bump_counter(subscription_fetch_calls_failed, FailedBatches),
+ bump_counter(subscription_fetch_users_dropped, FailedUsers),
+ {GuildId, MessageId} = delivery_ids(Ctx),
+ logger:warning("Push: subscription fetch batches failed; recipients not attempted", #{
+ guild_id => GuildId,
+ message_id => MessageId,
+ failed_batches => FailedBatches,
+ recipients_not_attempted => FailedUsers
+ }).
+
+-spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].
+chunk_user_ids([], _BatchSize, Acc) ->
+ lists:reverse(Acc);
+chunk_user_ids(UserIds, BatchSize, Acc) ->
+ {Batch, Rest} = take_user_id_batch(UserIds, BatchSize, []),
+ chunk_user_ids(Rest, BatchSize, [Batch | Acc]).
+
+-spec take_user_id_batch([integer()], non_neg_integer(), [integer()]) ->
+ {[integer()], [integer()]}.
+take_user_id_batch(Rest, 0, Acc) ->
+ {lists:reverse(Acc), Rest};
+take_user_id_batch([], _Remaining, Acc) ->
+ {lists:reverse(Acc), []};
+take_user_id_batch([UserId | Rest], Remaining, Acc) ->
+ take_user_id_batch(Rest, Remaining - 1, [UserId | Acc]).
+
+-spec count_subscription_fetch_loss(non_neg_integer()) -> ok.
+count_subscription_fetch_loss(UserCount) ->
+ bump_counter(subscription_fetch_calls_failed, 1),
+ bump_counter(subscription_fetch_users_dropped, UserCount).
-spec add_fetched_user_subscription_task(integer(), map(), [{integer(), list()}]) ->
[{integer(), list()}].
@@ -205,6 +271,7 @@ send_clear_rpc_result(UserId, ChannelId, MessageId, BadgeCount, {ok, Subscriptio
UserId, ChannelId, MessageId, BadgeCount, maps:get(UserIdBin, SubscriptionsData, [])
);
send_clear_rpc_result(UserId, _ChannelId, _MessageId, _BadgeCount, {error, Reason}) ->
+ count_subscription_fetch_loss(1),
logger:debug(
"Push: RPC failed to fetch subscriptions for notification clear",
#{user_id => UserId, reason => Reason}
@@ -238,7 +305,7 @@ send_cached_subscriptions(SubscriptionsByUser, Ctx) ->
[],
SubscriptionsByUser
),
- send_subscription_tasks(Tasks, Ctx).
+ deliver_subscription_tasks(Tasks, Ctx).
-spec add_cached_user_task(integer(), list(), [{integer(), list()}]) ->
[{integer(), list()}].
@@ -250,21 +317,335 @@ add_cached_user_task(UserId, Subscriptions, Acc) ->
[{UserId, Subscriptions} | Acc]
end.
--spec send_subscription_tasks([{integer(), list()}], tuple()) -> ok.
-send_subscription_tasks(Tasks, Ctx) ->
- send_subscription_tasks(Tasks, Ctx, delivery_concurrency()).
+-spec deliver_subscription_tasks([{integer(), list()}], tuple()) -> ok.
+deliver_subscription_tasks(Tasks, Ctx) ->
+ send_bounded_subscription_tasks(Tasks, Ctx).
--spec send_subscription_tasks([{integer(), list()}], tuple(), pos_integer()) -> ok.
-send_subscription_tasks([], _Ctx, _MaxConcurrency) ->
+-spec send_bounded_subscription_tasks([{integer(), list()}], tuple()) -> ok.
+send_bounded_subscription_tasks([], _Ctx) ->
ok;
-send_subscription_tasks(Tasks, Ctx, 1) ->
- send_subscription_task_chunk(Tasks, Ctx);
-send_subscription_tasks(Tasks, Ctx, MaxConcurrency) ->
+send_bounded_subscription_tasks(Tasks, Ctx) ->
TaskCount = length(Tasks),
- WorkerCount = min(TaskCount, MaxConcurrency),
+ Desired = bounded_worker_count(TaskCount, delivery_concurrency(), delivery_max_workers()),
+ Granted = claim_delivery_slots(Desired, gateway_http_client:push_max_concurrency()),
+ try
+ run_claimed_delivery(Tasks, TaskCount, Ctx, Granted)
+ after
+ release_delivery_slots(Granted)
+ end.
+
+-spec run_claimed_delivery([{integer(), list()}], pos_integer(), tuple(), non_neg_integer()) ->
+ ok.
+run_claimed_delivery(Tasks, _TaskCount, Ctx, 0) ->
+ bump_counter(delivery_budget_exhausted, 1),
+ run_inline_delivery(Tasks, Ctx);
+run_claimed_delivery(Tasks, TaskCount, Ctx, WorkerCount) ->
ChunkSize = max(1, (TaskCount + WorkerCount - 1) div WorkerCount),
Chunks = chunk_subscription_tasks(Tasks, ChunkSize, []),
- wait_delivery_workers(start_delivery_workers(Chunks, Ctx, #{})).
+ Workers = start_progress_workers(
+ Chunks, shared_ctx_payload(Ctx), ctx_badge_counts(Ctx), self(), #{}
+ ),
+ await_progress_workers(Workers, 0, TaskCount, delivery_ids(Ctx)).
+
+-spec run_inline_delivery([{integer(), list()}], tuple()) -> ok.
+run_inline_delivery([], _Ctx) ->
+ ok;
+run_inline_delivery([Task | Rest], Ctx) ->
+ run_subscription_task(Task, Ctx),
+ run_inline_delivery(Rest, Ctx).
+
+-spec claim_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer().
+claim_delivery_slots(Desired, Budget) ->
+ try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, Desired}) of
+ Reserved -> settle_delivery_claim(Desired, Budget, Reserved)
+ catch
+ error:badarg -> claim_first_delivery_slots(Desired, Budget)
+ end.
+
+-spec claim_first_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer().
+claim_first_delivery_slots(Desired, Budget) ->
+ try ets:insert_new(?PUSH_COUNTERS, {?CNT_LIVE_WORKERS, 0}) of
+ _Inserted -> retry_claim_delivery_slots(Desired, Budget)
+ catch
+ error:badarg -> 0
+ end.
+
+-spec retry_claim_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer().
+retry_claim_delivery_slots(Desired, Budget) ->
+ try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, Desired}) of
+ Reserved -> settle_delivery_claim(Desired, Budget, Reserved)
+ catch
+ error:badarg -> 0
+ end.
+
+-spec settle_delivery_claim(pos_integer(), pos_integer(), integer()) -> non_neg_integer().
+settle_delivery_claim(Desired, Budget, Reserved) ->
+ Granted = min(Desired, max(0, Budget - (Reserved - Desired))),
+ release_delivery_slots(Desired - Granted),
+ Granted.
+
+-spec release_delivery_slots(non_neg_integer()) -> ok.
+release_delivery_slots(0) ->
+ ok;
+release_delivery_slots(Count) ->
+ try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, -Count, 0, 0}) of
+ _Value -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+-spec bounded_worker_count(pos_integer(), pos_integer(), pos_integer()) -> pos_integer().
+bounded_worker_count(TaskCount, Base, Bound) ->
+ TasksPerWorker = delivery_tasks_per_worker(),
+ Sized = (TaskCount + TasksPerWorker - 1) div TasksPerWorker,
+ min(TaskCount, min(Bound, max(Base, Sized))).
+
+-type delivery_workers() :: #{pid() => {reference(), non_neg_integer()}}.
+
+-spec start_progress_workers(
+ [[{integer(), list()}]], binary(), map(), pid(), delivery_workers()
+) -> delivery_workers().
+start_progress_workers([], _Shared, _BadgeCounts, _Coordinator, Workers) ->
+ Workers;
+start_progress_workers([Chunk | Rest], Shared, BadgeCounts, Coordinator, Workers) ->
+ ChunkSize = length(Chunk),
+ ChunkBadges = chunk_badge_counts(Chunk, BadgeCounts),
+ bump_counter(delivery_workers_spawned, 1),
+ {Pid, Ref} = spawn_monitor(fun() ->
+ run_progress_chunk(Chunk, restore_chunk_ctx(Shared, ChunkBadges), Coordinator)
+ end),
+ start_progress_workers(
+ Rest, Shared, BadgeCounts, Coordinator, Workers#{Pid => {Ref, ChunkSize}}
+ ).
+
+-spec shared_ctx_payload(tuple()) -> binary().
+shared_ctx_payload(Ctx) ->
+ term_to_binary(setelement(?CTX_BADGE_COUNTS, Ctx, #{})).
+
+-spec restore_chunk_ctx(binary(), map()) -> tuple().
+restore_chunk_ctx(Shared, BadgeCounts) ->
+ case binary_to_term(Shared) of
+ Ctx when is_tuple(Ctx) -> setelement(?CTX_BADGE_COUNTS, Ctx, BadgeCounts)
+ end.
+
+-spec chunk_badge_counts([{integer(), list()}], map()) -> map().
+chunk_badge_counts(Chunk, BadgeCounts) ->
+ maps:with([UserId || {UserId, _Subscriptions} <- Chunk], BadgeCounts).
+
+-spec ctx_badge_counts(tuple()) -> map().
+ctx_badge_counts(Ctx) ->
+ {
+ _MsgData,
+ _GuildId,
+ _ChannelId,
+ _MessageId,
+ _GuildName,
+ _ChannelName,
+ _MarkdownContext,
+ _ContentPreview,
+ BadgeCounts
+ } = Ctx,
+ BadgeCounts.
+
+-spec run_progress_chunk([{integer(), list()}], tuple(), pid()) -> ok.
+run_progress_chunk([], _Ctx, _Coordinator) ->
+ ok;
+run_progress_chunk([Task | Rest], Ctx, Coordinator) ->
+ run_subscription_task(Task, Ctx),
+ Coordinator ! {push_delivery_progress, self()},
+ run_progress_chunk(Rest, Ctx, Coordinator).
+
+-spec run_subscription_task({integer(), list()}, tuple()) -> ok.
+run_subscription_task({UserId, _Subscriptions} = Task, Ctx) ->
+ try
+ send_subscription_task(Task, Ctx)
+ catch
+ Class:Reason ->
+ bump_counter(delivery_task_failures, 1),
+ logger:warning("Push: delivery task raised", #{
+ user_id => UserId, class => Class, reason => Reason
+ })
+ end,
+ ok.
+
+-spec await_progress_workers(
+ delivery_workers(), non_neg_integer(), non_neg_integer(), {integer(), integer()}
+) -> ok.
+await_progress_workers(Workers, Unattempted, TaskCount, Ids) when map_size(Workers) =:= 0 ->
+ report_unattempted_recipients(Unattempted, TaskCount, Ids);
+await_progress_workers(Workers, Unattempted, TaskCount, Ids) ->
+ receive
+ {push_delivery_progress, Pid} ->
+ await_progress_workers(record_task_done(Pid, Workers), Unattempted, TaskCount, Ids);
+ {'DOWN', _Ref, process, Pid, normal} ->
+ await_progress_workers(maps:remove(Pid, Workers), Unattempted, TaskCount, Ids);
+ {'DOWN', _Ref, process, Pid, _Reason} ->
+ {Lost, Rest} = discard_crashed_worker(Pid, Workers),
+ await_progress_workers(Rest, Unattempted + Lost, TaskCount, Ids)
+ after ?DELIVERY_IDLE_TIMEOUT_MS ->
+ report_stalled_delivery(Workers, Unattempted, TaskCount, Ids)
+ end.
+
+-spec record_task_done(pid(), delivery_workers()) -> delivery_workers().
+record_task_done(Pid, Workers) ->
+ case Workers of
+ #{Pid := {Ref, Left}} -> Workers#{Pid := {Ref, max(0, Left - 1)}};
+ _ -> Workers
+ end.
+
+-spec discard_crashed_worker(pid(), delivery_workers()) ->
+ {non_neg_integer(), delivery_workers()}.
+discard_crashed_worker(Pid, Workers) ->
+ case maps:take(Pid, Workers) of
+ {{_Ref, Left}, Rest} ->
+ bump_counter(delivery_worker_crashes, 1),
+ bump_counter(delivery_recipients_not_attempted, Left),
+ {Left, Rest};
+ error ->
+ {0, Workers}
+ end.
+
+-spec report_unattempted_recipients(
+ non_neg_integer(), non_neg_integer(), {integer(), integer()}
+) -> ok.
+report_unattempted_recipients(0, _TaskCount, _Ids) ->
+ ok;
+report_unattempted_recipients(Unattempted, TaskCount, {GuildId, MessageId}) ->
+ logger:warning("Push: delivery worker died with recipients unattempted", #{
+ guild_id => GuildId,
+ message_id => MessageId,
+ recipients_total => TaskCount,
+ recipients_not_attempted => Unattempted
+ }).
+
+-spec report_stalled_delivery(
+ delivery_workers(), non_neg_integer(), non_neg_integer(), {integer(), integer()}
+) -> ok.
+report_stalled_delivery(Workers, Unattempted, TaskCount, {GuildId, MessageId}) ->
+ Abandoned = map_size(Workers),
+ Stranded = abandon_delivery_workers(maps:to_list(Workers), 0),
+ bump_counter(delivery_workers_abandoned, Abandoned),
+ bump_counter(delivery_recipients_unconfirmed, Stranded),
+ logger:error("Push: subscription delivery stalled with recipients unattempted", #{
+ guild_id => GuildId,
+ message_id => MessageId,
+ recipients_total => TaskCount,
+ recipients_not_attempted => Unattempted,
+ recipients_unconfirmed => Stranded,
+ workers_still_running => Abandoned,
+ idle_timeout_ms => ?DELIVERY_IDLE_TIMEOUT_MS
+ }).
+
+-spec abandon_delivery_workers(
+ [{pid(), {reference(), non_neg_integer()}}], non_neg_integer()
+) -> non_neg_integer().
+abandon_delivery_workers([], Stranded) ->
+ Stranded;
+abandon_delivery_workers([{_Pid, {Ref, Left}} | Rest], Stranded) ->
+ _ = erlang:demonitor(Ref, [flush]),
+ abandon_delivery_workers(Rest, Stranded + Left).
+
+-spec delivery_ids(tuple()) -> {integer(), integer()}.
+delivery_ids(Ctx) ->
+ {
+ _MsgData,
+ GuildId,
+ _ChannelId,
+ MessageId,
+ _GuildName,
+ _ChannelName,
+ _MarkdownContext,
+ _ContentPreview,
+ _BadgeCounts
+ } = Ctx,
+ {GuildId, MessageId}.
+
+-spec delivery_stats() -> #{atom() => term()}.
+delivery_stats() ->
+ Counters = maps:from_list([{Key, counter_value(Key)} || Key <- delivery_counter_keys()]),
+ Counters#{delivery_last_crash => last_delivery_crash()}.
+
+-spec delivery_counter_keys() -> [atom()].
+delivery_counter_keys() ->
+ [
+ ?CNT_LIVE_WORKERS,
+ delivery_workers_spawned,
+ delivery_worker_crashes,
+ delivery_workers_abandoned,
+ delivery_recipients_not_attempted,
+ delivery_recipients_unconfirmed,
+ delivery_task_failures,
+ delivery_budget_exhausted,
+ subscription_fetch_calls_failed,
+ subscription_fetch_users_dropped,
+ badge_fetch_calls_failed,
+ badge_fetch_users_defaulted
+ ].
+
+-spec counter_value(atom()) -> non_neg_integer().
+counter_value(Key) ->
+ try ets:lookup(?PUSH_COUNTERS, Key) of
+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;
+ _ -> 0
+ catch
+ error:badarg -> 0
+ end.
+
+-spec last_delivery_crash() -> term().
+last_delivery_crash() ->
+ try ets:lookup(?PUSH_COUNTERS, delivery_last_crash) of
+ [{delivery_last_crash, Summary}] -> Summary;
+ _ -> none
+ catch
+ error:badarg -> none
+ end.
+
+-spec bump_counter(atom(), integer()) -> ok.
+bump_counter(Key, Delta) ->
+ try
+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),
+ ok
+ catch
+ error:badarg -> seed_and_bump_counter(Key, Delta)
+ end.
+
+-spec seed_and_bump_counter(atom(), integer()) -> ok.
+seed_and_bump_counter(Key, Delta) ->
+ try
+ _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}),
+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),
+ ok
+ catch
+ error:badarg -> ok
+ end.
+
+-spec delivery_max_workers() -> pos_integer().
+delivery_max_workers() ->
+ case application:get_env(fluxer_gateway, push_delivery_max_workers, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> ?DEFAULT_DELIVERY_MAX_WORKERS
+ end.
+
+-spec delivery_tasks_per_worker() -> pos_integer().
+delivery_tasks_per_worker() ->
+ case application:get_env(fluxer_gateway, push_delivery_tasks_per_worker, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> ?DEFAULT_DELIVERY_TASKS_PER_WORKER
+ end.
+
+-spec subscription_fetch_batch_size() -> pos_integer().
+subscription_fetch_batch_size() ->
+ case application:get_env(fluxer_gateway, push_subscription_fetch_batch_size, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> ?DEFAULT_SUBSCRIPTION_FETCH_BATCH
+ end.
+
+-spec subscription_fetch_budget_ms() -> pos_integer().
+subscription_fetch_budget_ms() ->
+ case application:get_env(fluxer_gateway, push_subscription_fetch_budget_ms, undefined) of
+ Value when is_integer(Value), Value > 0 -> Value;
+ _ -> ?DEFAULT_SUBSCRIPTION_FETCH_BUDGET_MS
+ end.
-spec delivery_concurrency() -> pos_integer().
delivery_concurrency() ->
@@ -290,35 +671,6 @@ take_subscription_chunk([], _Remaining, Acc) ->
take_subscription_chunk([Task | Rest], Remaining, Acc) ->
take_subscription_chunk(Rest, Remaining - 1, [Task | Acc]).
--spec start_delivery_workers([[{integer(), list()}]], tuple(), #{reference() => true}) ->
- #{reference() => true}.
-start_delivery_workers([], _Ctx, Refs) ->
- Refs;
-start_delivery_workers([Chunk | Rest], Ctx, Refs) ->
- {_Pid, Ref} = spawn_monitor(fun() ->
- send_subscription_task_chunk(Chunk, Ctx)
- end),
- start_delivery_workers(Rest, Ctx, Refs#{Ref => true}).
-
--spec wait_delivery_workers(#{reference() => true}) -> ok.
-wait_delivery_workers(Refs) when map_size(Refs) =:= 0 ->
- ok;
-wait_delivery_workers(Refs) ->
- receive
- {'DOWN', Ref, process, _Pid, _Reason} ->
- wait_delivery_workers(maps:remove(Ref, Refs))
- after ?DELIVERY_WORKER_TIMEOUT_MS ->
- logger:warning("Push: timed out waiting for subscription delivery workers", #{
- remaining_workers => map_size(Refs)
- }),
- ok
- end.
-
--spec send_subscription_task_chunk([{integer(), list()}], tuple()) -> ok.
-send_subscription_task_chunk(Tasks, Ctx) ->
- lists:foreach(fun(Task) -> send_subscription_task(Task, Ctx) end, Tasks),
- ok.
-
-spec send_subscription_task({integer(), list()}, tuple()) -> ok.
send_subscription_task({UserId, Subscriptions}, Ctx) ->
push_sender:send_to_user_subscriptions(
@@ -473,7 +825,7 @@ send_context(UserId, Ctx) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-send_subscription_tasks_sends_all_users_with_bounded_workers_test() ->
+deliver_subscription_tasks_sends_every_user_its_badge_count_test() ->
Self = self(),
ok = meck:new(push_sender, [passthrough, no_link]),
try
@@ -486,7 +838,7 @@ send_subscription_tasks_sends_all_users_with_bounded_workers_test() ->
ok
end),
Tasks = [{1, [sub1]}, {2, [sub2]}, {3, [sub3]}],
- ?assertEqual(ok, send_subscription_tasks(Tasks, test_send_ctx(), 2)),
+ ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())),
?assertEqual(
[{1, [sub1], 5}, {2, [sub2], 6}, {3, [sub3], 7}],
lists:sort(collect_sent_subscription_tasks(3, []))
@@ -503,54 +855,290 @@ chunk_subscription_tasks_uses_bounded_chunk_size_test() ->
chunk_subscription_tasks(Tasks, 2, [])
).
-chunk_user_ids_uses_bounded_chunk_size_test() ->
- ?assertEqual([], chunk_user_ids([], 2, [])),
- ?assertEqual([[1, 2]], chunk_user_ids([1, 2], 2, [])),
- ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])).
+bounded_worker_count_scales_with_recipients_and_stays_bounded_test() ->
+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),
+ ?assertEqual(3, bounded_worker_count(3, 8, 64)),
+ ?assertEqual(8, bounded_worker_count(100, 8, 64)),
+ ?assertEqual(20, bounded_worker_count(2500, 8, 64)),
+ ?assertEqual(64, bounded_worker_count(10000, 8, 64)),
+ ?assertEqual(8, bounded_worker_count(10000, 8, 8)).
+
+bounded_worker_count_treats_the_bound_as_a_real_ceiling_test() ->
+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),
+ ?assertEqual(4, bounded_worker_count(10000, 8, 4)),
+ ?assertEqual(1, bounded_worker_count(10000, 8, 1)),
+ ?assertEqual(2, bounded_worker_count(2500, 8, 2)).
-add_fetched_user_subscription_task_caches_empty_result_test() ->
- push_ets_cache:init(),
- push_ets_cache:delete_subscriptions(4242),
- ?assertEqual([], add_fetched_user_subscription_task(4242, #{}, [])),
- ?assertEqual([], push_ets_cache:get_subscriptions(4242)),
- push_ets_cache:delete_subscriptions(4242).
+shared_ctx_payload_carries_no_badge_counts_test() ->
+ Ctx = test_send_ctx(),
+ Shared = shared_ctx_payload(Ctx),
+ ?assert(is_binary(Shared)),
+ ?assertEqual(#{}, element(?CTX_BADGE_COUNTS, binary_to_term(Shared))),
+ ?assert(byte_size(Shared) < byte_size(term_to_binary(Ctx))).
+
+restore_chunk_ctx_rebuilds_the_ctx_with_only_the_chunk_badges_test() ->
+ Ctx = test_send_ctx(),
+ Shared = shared_ctx_payload(Ctx),
+ Chunk = [{2, [sub]}],
+ Restored = restore_chunk_ctx(Shared, chunk_badge_counts(Chunk, ctx_badge_counts(Ctx))),
+ ?assertEqual(9, tuple_size(Restored)),
+ ?assertEqual(#{2 => 6}, element(?CTX_BADGE_COUNTS, Restored)),
+ ?assertEqual(element(1, Ctx), element(1, Restored)),
+ ?assertEqual(element(7, Ctx), element(7, Restored)),
+ ?assertEqual(element(8, Ctx), element(8, Restored)),
+ ?assertEqual(#{}, element(?CTX_BADGE_COUNTS, restore_chunk_ctx(Shared, #{}))).
+
+bounded_fanout_isolates_a_raising_task_and_counts_it_test() ->
+ Self = self(),
+ ok = meck:new(push_sender, [passthrough, no_link]),
+ ensure_test_counter_table(),
+ Before = counter_value(delivery_task_failures),
+ application:set_env(fluxer_gateway, push_delivery_tasks_per_worker, 1),
+ try
+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) ->
+ case UserId of
+ 3 -> error(deliberate_task_crash);
+ _ -> Self ! {bounded_sent, UserId}
+ end,
+ ok
+ end),
+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 5)],
+ ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())),
+ ?assertEqual([1, 2, 4, 5], lists:sort(collect_bounded_sent(4, []))),
+ ?assertEqual(Before + 1, counter_value(delivery_task_failures))
+ after
+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),
+ meck:unload(push_sender)
+ end.
-fetch_missing_subscriptions_chunks_and_merges_test() ->
+bounded_fanout_returns_reserved_slots_to_the_push_budget_test() ->
+ ok = meck:new(push_sender, [passthrough, no_link]),
+ ensure_test_counter_table(),
+ ok = reset_test_counter(?CNT_LIVE_WORKERS),
+ try
+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(_UserId, _Subs, _Ctx) ->
+ ok
+ end),
+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 40)],
+ ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())),
+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS))
+ after
+ meck:unload(push_sender)
+ end.
+
+bounded_fanout_releases_reserved_slots_when_workers_are_killed_test() ->
+ ok = meck:new(push_sender, [passthrough, no_link]),
+ ensure_test_counter_table(),
+ ok = reset_test_counter(?CNT_LIVE_WORKERS),
+ ok = reset_test_counter(delivery_recipients_not_attempted),
+ application:set_env(fluxer_gateway, push_delivery_tasks_per_worker, 1),
+ try
+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(_UserId, _Subs, _Ctx) ->
+ exit(self(), kill)
+ end),
+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 4)],
+ ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())),
+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)),
+ ?assertEqual(4, counter_value(delivery_recipients_not_attempted))
+ after
+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),
+ meck:unload(push_sender)
+ end.
+
+claim_delivery_slots_reserves_before_the_next_job_reads_the_budget_test() ->
+ ensure_test_counter_table(),
+ ok = reset_test_counter(?CNT_LIVE_WORKERS),
+ ?assertEqual(8, claim_delivery_slots(8, 10)),
+ ?assertEqual(8, counter_value(?CNT_LIVE_WORKERS)),
+ ?assertEqual(2, claim_delivery_slots(8, 10)),
+ ?assertEqual(10, counter_value(?CNT_LIVE_WORKERS)),
+ ?assertEqual(0, claim_delivery_slots(4, 10)),
+ ?assertEqual(10, counter_value(?CNT_LIVE_WORKERS)),
+ ok = release_delivery_slots(10),
+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)),
+ ok = release_delivery_slots(5),
+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)).
+
+bounded_fanout_delivers_inline_when_the_push_budget_is_exhausted_test() ->
+ Self = self(),
+ ok = meck:new(push_sender, [passthrough, no_link]),
+ ensure_test_counter_table(),
+ ok = reset_test_counter(delivery_budget_exhausted),
+ Budget = gateway_http_client:push_max_concurrency(),
+ ok = reset_test_counter(?CNT_LIVE_WORKERS),
+ _ = claim_delivery_slots(Budget, Budget),
+ try
+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) ->
+ Self ! {bounded_sent, UserId},
+ ok
+ end),
+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 6)],
+ ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())),
+ ?assertEqual(lists:seq(1, 6), lists:sort(collect_bounded_sent(6, []))),
+ ?assertEqual(1, counter_value(delivery_budget_exhausted)),
+ ?assertEqual(Budget, counter_value(?CNT_LIVE_WORKERS))
+ after
+ ok = release_delivery_slots(Budget),
+ meck:unload(push_sender)
+ end.
+
+delivery_stats_exposes_cumulative_loss_without_logs_test() ->
+ ensure_test_counter_table(),
+ Before = maps:get(delivery_recipients_not_attempted, delivery_stats()),
+ ok = bump_counter(delivery_recipients_not_attempted, 3),
+ Stats = delivery_stats(),
+ ?assertEqual(Before + 3, maps:get(delivery_recipients_not_attempted, Stats)),
+ ?assertEqual(
+ lists:sort([delivery_last_crash | delivery_counter_keys()]),
+ lists:sort(maps:keys(Stats))
+ ).
+
+bump_counter_never_creates_the_counter_table_test() ->
+ case drop_test_counter_table() of
+ ok ->
+ ?assertEqual(ok, bump_counter(delivery_recipients_not_attempted, 1)),
+ ?assertEqual(0, counter_value(delivery_recipients_not_attempted)),
+ ?assertEqual(undefined, ets:info(?PUSH_COUNTERS, name));
+ skip ->
+ ok
+ end.
+
+ensure_test_counter_table() ->
+ case ets:info(?PUSH_COUNTERS, name) of
+ undefined ->
+ _ = ets:new(?PUSH_COUNTERS, [
+ named_table, public, set, {write_concurrency, true}
+ ]),
+ ok;
+ _ ->
+ ok
+ end.
+
+drop_test_counter_table() ->
+ case ets:info(?PUSH_COUNTERS, name) of
+ undefined ->
+ ok;
+ _ ->
+ try ets:delete(?PUSH_COUNTERS) of
+ _ -> ok
+ catch
+ error:badarg -> skip
+ end
+ end.
+
+reset_test_counter(Key) ->
+ try ets:insert(?PUSH_COUNTERS, {Key, 0}) of
+ _ -> ok
+ catch
+ error:badarg -> ok
+ end.
+
+bounded_fanout_delivers_every_recipient_test() ->
+ Self = self(),
+ ok = meck:new(push_sender, [passthrough, no_link]),
+ try
+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) ->
+ Self ! {bounded_sent, UserId},
+ ok
+ end),
+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 200)],
+ ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())),
+ ?assertEqual(lists:seq(1, 200), lists:sort(collect_bounded_sent(200, [])))
+ after
+ meck:unload(push_sender)
+ end.
+
+chunk_user_ids_uses_bounded_batches_test() ->
+ ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])),
+ ?assertEqual([], chunk_user_ids([], 2, [])).
+
+fetch_subscription_batches_stops_after_consecutive_failures_test() ->
ok = meck:new(rpc_client, [passthrough, no_link]),
try
- ok = meck:expect(rpc_client, call, fun subscriptions_chunk_stub/1),
- UserIds = lists:seq(1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2),
- Expected = #{
- <<"1">> => [{chunk, ?SUBSCRIPTION_FETCH_CHUNK_SIZE}],
- integer_to_binary(?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1) => [{chunk, 2}]
- },
- ?assertEqual({UserIds, Expected}, fetch_missing_subscriptions(UserIds)),
- ?assertEqual(2, meck:num_calls(rpc_client, call, '_'))
+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),
+ Batches = [[N] || N <- lists:seq(1, 10)],
+ ?assertEqual(
+ {[], 10, 10, ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES},
+ fetch_subscription_batches(Batches, {[], 0, 0, 0})
+ ),
+ ?assertEqual(
+ ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES,
+ length(meck:history(rpc_client))
+ )
after
meck:unload(rpc_client)
end.
-fetch_missing_subscriptions_keeps_other_chunks_when_one_fails_test() ->
+fetch_missing_in_batches_sends_only_successful_batches_test() ->
+ Self = self(),
+ ok = meck:new(push_sender, [passthrough, no_link]),
ok = meck:new(rpc_client, [passthrough, no_link]),
+ ok = meck:new(push_ets_cache, [passthrough, no_link]),
+ application:set_env(fluxer_gateway, push_subscription_fetch_batch_size, 2),
try
- ok = meck:expect(rpc_client, call, fun failing_first_chunk_stub/1),
- UserIds = lists:seq(1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2),
- TailUserIds = [?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2],
- Expected = #{
- integer_to_binary(?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1) => [{chunk, 2}]
- },
- ?assertEqual({TailUserIds, Expected}, fetch_missing_subscriptions(UserIds))
+ ok = meck:expect(push_ets_cache, put_subscriptions, fun(_UserId, _Subs) -> ok end),
+ ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) ->
+ case Ids of
+ [<<"1">>, <<"2">>] -> {ok, #{<<"1">> => [sub1], <<"2">> => [sub2]}};
+ _ -> {error, unavailable}
+ end
+ end),
+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, Subs, _Ctx) ->
+ Self ! {bounded_sent, {UserId, Subs}},
+ ok
+ end),
+ ?assertEqual(
+ ok, fetch_and_send_missing_subscriptions([1, 2, 3, 4], test_send_ctx())
+ ),
+ ?assertEqual([{1, [sub1]}, {2, [sub2]}], lists:sort(collect_bounded_sent(2, [])))
+ after
+ application:unset_env(fluxer_gateway, push_subscription_fetch_batch_size),
+ meck:unload(push_ets_cache),
+ meck:unload(rpc_client),
+ meck:unload(push_sender)
+ end.
+
+subscription_fetch_failure_counts_dropped_recipients_test() ->
+ ok = meck:new(rpc_client, [passthrough, no_link]),
+ ensure_test_counter_table(),
+ ok = reset_test_counter(subscription_fetch_calls_failed),
+ ok = reset_test_counter(subscription_fetch_users_dropped),
+ try
+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),
+ ?assertEqual(
+ ok, fetch_and_send_missing_subscriptions([1, 2, 3], test_send_ctx())
+ ),
+ ?assertEqual(1, counter_value(subscription_fetch_calls_failed)),
+ ?assertEqual(3, counter_value(subscription_fetch_users_dropped))
after
meck:unload(rpc_client)
end.
-subscriptions_chunk_stub(#{<<"user_ids">> := [FirstUserIdBin | _] = UserIdBins}) ->
- {ok, #{FirstUserIdBin => [{chunk, length(UserIdBins)}]}}.
+clear_notification_fetch_failure_counts_the_dropped_recipient_test() ->
+ ok = meck:new(rpc_client, [passthrough, no_link]),
+ ensure_test_counter_table(),
+ ok = reset_test_counter(subscription_fetch_calls_failed),
+ ok = reset_test_counter(subscription_fetch_users_dropped),
+ try
+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),
+ ?assertEqual(
+ ok, fetch_and_send_clear_notification_from_rpc(7, 8, 9, 0)
+ ),
+ ?assertEqual(1, counter_value(subscription_fetch_calls_failed)),
+ ?assertEqual(1, counter_value(subscription_fetch_users_dropped))
+ after
+ meck:unload(rpc_client)
+ end.
-failing_first_chunk_stub(#{<<"user_ids">> := [<<"1">> | _]}) ->
- {error, timeout};
-failing_first_chunk_stub(Request) ->
- subscriptions_chunk_stub(Request).
+collect_bounded_sent(0, Acc) ->
+ Acc;
+collect_bounded_sent(Count, Acc) ->
+ receive
+ {bounded_sent, Sent} ->
+ collect_bounded_sent(Count - 1, [Sent | Acc])
+ after 5000 ->
+ ?assert(false)
+ end.
collect_sent_subscription_tasks(0, Acc) ->
Acc;
diff --git a/fluxer_gateway/src/session/session_init.erl b/fluxer_gateway/src/session/session_init.erl
index 0270fbb43..cd0156745 100644
--- a/fluxer_gateway/src/session/session_init.erl
+++ b/fluxer_gateway/src/session/session_init.erl
@@ -63,9 +63,21 @@ normalize_seq(_) -> 0.
-spec normalize_buffer(term()) -> [map()].
normalize_buffer(Buffer) when is_list(Buffer) ->
lists:filtermap(fun normalize_buffer_event/1, Buffer);
+normalize_buffer(Buffer) when is_map(Buffer) ->
+ lists:filtermap(fun normalize_buffer_event/1, deque_to_list(Buffer));
normalize_buffer(_) ->
[].
+-spec deque_to_list(map()) -> [term()].
+deque_to_list(Buffer) ->
+ try limited_deque:to_list(eqwalizer:dynamic_cast(Buffer)) of
+ List when is_list(List) -> List;
+ _ -> []
+ catch
+ error:_Reason -> [];
+ exit:_Reason -> []
+ end.
+
-spec normalize_buffer_event(term()) -> {true, map()} | false.
normalize_buffer_event(Event) when is_map(Event) ->
case maps:get(seq, Event, -1) of
@@ -75,19 +87,43 @@ normalize_buffer_event(Event) when is_map(Event) ->
normalize_buffer_event(_) ->
false.
+-spec init_buffer_bytes(term(), [map()]) -> non_neg_integer().
+init_buffer_bytes(_Raw, []) ->
+ 0;
+init_buffer_bytes(Raw, Buffer) when is_map(Raw) ->
+ restored_deque_bytes(Raw, Buffer);
+init_buffer_bytes(_Raw, Buffer) ->
+ replay_buffer_bytes(Buffer).
+
+-spec restored_deque_bytes(map(), [map()]) -> non_neg_integer().
+restored_deque_bytes(Raw, Buffer) ->
+ try limited_deque:bytes(eqwalizer:dynamic_cast(Raw)) of
+ Bytes when is_integer(Bytes), Bytes >= 0 -> Bytes;
+ _ -> replay_buffer_bytes(Buffer)
+ catch
+ error:_Reason -> replay_buffer_bytes(Buffer);
+ exit:_Reason -> replay_buffer_bytes(Buffer)
+ end.
+
-spec replay_buffer_bytes([map()]) -> non_neg_integer().
replay_buffer_bytes(Buffer) ->
WordSize = erlang:system_info(wordsize),
- trunc(
- lists:foldl(
- fun(Event, Acc) ->
- Acc + erts_debug:flat_size(Event) * WordSize
- end,
- 0,
- Buffer
- )
+ lists:foldl(
+ fun(Event, Acc) -> Acc + erts_debug:flat_size(Event) * WordSize end,
+ 0,
+ Buffer
).
+-spec replay_payload_bytes([map()]) -> non_neg_integer().
+replay_payload_bytes(Buffer) ->
+ lists:foldl(fun(Event, Acc) -> Acc + entry_payload_bytes(Event) end, 0, Buffer).
+
+-spec entry_payload_bytes(term()) -> non_neg_integer().
+entry_payload_bytes(#{data := {pre_encoded, Payload}}) when is_binary(Payload) ->
+ byte_size(Payload);
+entry_payload_bytes(_Event) ->
+ 0.
+
-spec build_ignored_events_map(term()) -> #{binary() => true}.
build_ignored_events_map(Events) when is_list(Events) ->
maps:from_list([{Event, true} || Event <- Events, is_binary(Event)]);
@@ -193,7 +229,8 @@ extract_core_fields(
token_hash => TokenHash,
auth_session_id_hash => AuthSessionIdHash,
buffer => Buffer,
- buffer_bytes => replay_buffer_bytes(Buffer),
+ buffer_bytes => init_buffer_bytes(maps:get(buffer, D, []), Buffer),
+ replay_payload_bytes => replay_payload_bytes(Buffer),
seq => Seq,
ack_seq => AckSeq,
properties => Properties,
@@ -285,7 +322,7 @@ schedule_timers(#{bot := Bot, guilds := GuildsMap}) ->
init_prewarm_guilds(Bot, GuildIds),
lists:foreach(fun(Gid) -> self() ! {guild_connect, Gid, 0} end, GuildIds),
erlang:send_after(5000, self(), premature_readiness),
- erlang:send_after(200, self(), enable_presence_updates),
+ erlang:send_after(10000, self(), enable_presence_updates),
erlang:send_after(60000, self(), check_ack_lag),
ok.
@@ -468,4 +505,68 @@ normalize_buffer_filters_invalid_entries_test() ->
?assertEqual([#{seq => 1}], normalize_buffer(Buffer)),
?assertEqual([], normalize_buffer(undefined)).
+normalize_buffer_accepts_deque_test() ->
+ Deque = test_deque([#{seq => 1}, #{seq => 2}]),
+ ?assertEqual([#{seq => 1}, #{seq => 2}], normalize_buffer(Deque)).
+
+normalize_buffer_filters_invalid_deque_entries_test() ->
+ Deque = test_deque([#{seq => 1}, #{seq => -1}, #{no_seq => true}, <<"bad">>]),
+ ?assertEqual([#{seq => 1}], normalize_buffer(Deque)).
+
+normalize_buffer_ignores_non_deque_map_test() ->
+ ?assertEqual([], normalize_buffer(#{seq => 1})),
+ ?assertEqual([], normalize_buffer(#{})),
+ ?assertEqual([], normalize_buffer(undefined)).
+
+build_state_preserves_transferred_deque_buffer_test() ->
+ Data = transferred_session_data(test_deque([#{seq => 13}, #{seq => 14}])),
+ State = build_state(Data),
+ ?assertEqual([#{seq => 13}, #{seq => 14}], maps:get(buffer, State)),
+ ?assertEqual(
+ replay_buffer_bytes([#{seq => 13}, #{seq => 14}]),
+ maps:get(buffer_bytes, State)
+ ),
+ ?assertEqual(14, maps:get(seq, State)),
+ ?assertEqual(12, maps:get(ack_seq, State)).
+
+build_state_reuses_deque_byte_total_test() ->
+ Events = [#{seq => 21}, #{seq => 22}],
+ Data = transferred_session_data(test_deque(Events)),
+ State = build_state(Data),
+ ?assertEqual(limited_deque:bytes(test_deque(Events)), maps:get(buffer_bytes, State)),
+ ?assertEqual(replay_buffer_bytes(Events), maps:get(buffer_bytes, State)).
+
+replay_buffer_bytes_agrees_with_deque_accounting_test() ->
+ Entries = [
+ #{seq => 1, event => message_create, data => #{<<"content">> => <<"hi">>}},
+ #{seq => 2, event => message_create, data => {pre_encoded, binary:copy(<<"x">>, 70000)}}
+ ],
+ lists:foreach(
+ fun(Entry) ->
+ ?assertEqual(
+ limited_deque:bytes(limited_deque:from_list([Entry], 4096, 0)),
+ replay_buffer_bytes([Entry])
+ )
+ end,
+ Entries
+ ).
+
+build_state_walks_list_buffer_bytes_test() ->
+ Events = [#{seq => 13}, #{seq => 14}],
+ Data = transferred_session_data(Events),
+ State = build_state(Data),
+ ?assertEqual(replay_buffer_bytes(Events), maps:get(buffer_bytes, State)).
+
+build_state_zeroes_bytes_for_non_deque_map_buffer_test() ->
+ Data = transferred_session_data(#{seq => 13}),
+ State = build_state(Data),
+ ?assertEqual([], maps:get(buffer, State)),
+ ?assertEqual(0, maps:get(buffer_bytes, State)).
+
+transferred_session_data(Buffer) ->
+ (base_session_data(#{}))#{buffer => Buffer, seq => 14, ack_seq => 12}.
+
+test_deque(Events) ->
+ limited_deque:from_list(Events, 4096, 16777216).
+
-endif.
diff --git a/fluxer_gateway/src/session/session_lifecycle.erl b/fluxer_gateway/src/session/session_lifecycle.erl
index 8d77ae453..54f2577dc 100644
--- a/fluxer_gateway/src/session/session_lifecycle.erl
+++ b/fluxer_gateway/src/session/session_lifecycle.erl
@@ -34,6 +34,9 @@
-type status() :: session:status().
-type voice_state_entry() :: {binary(), map()}.
+-define(MAX_INLINE_RESUME_REPLAY, 256).
+-define(MAX_TRANSFER_PRE_ENCODED_BYTES, 4194304).
+
-spec handle_terminate_call([binary()], session_state()) ->
{stop, normal, terminated, session_state()} | {reply, ignored, session_state()}.
handle_terminate_call(Hashes, #{auth_session_id_hash := AuthHash} = State) ->
@@ -287,10 +290,25 @@ handle_heartbeat_ack(Seq, #{buffer := Buffer} = State) ->
true -> session_init:replay_buffer_bytes(NewBuffer);
false -> limited_deque:bytes(NewBuffer)
end,
- NewState0 = State#{ack_seq => AckedSeq, buffer => NewBuffer, buffer_bytes => NewBytes},
+ NewState0 = ack_state(State, AckedSeq, NewBuffer, NewBytes),
NewState = session_connection_guild:repair_stalled_guild_connects(NewState0),
{reply, true, NewState}.
+-spec ack_state(session_state(), seq(), term(), non_neg_integer()) -> session_state().
+ack_state(State, AckedSeq, NewBuffer, NewBytes) ->
+ Base = State#{ack_seq => AckedSeq, buffer => NewBuffer, buffer_bytes => NewBytes},
+ Base#{replay_payload_bytes => buffer_payload_bytes(NewBuffer)}.
+
+-spec buffer_payload_bytes(term()) -> non_neg_integer().
+buffer_payload_bytes(Buffer) when is_list(Buffer) ->
+ lists:foldl(fun(Entry, Acc) -> Acc + entry_payload_bytes(Entry) end, 0, Buffer);
+buffer_payload_bytes(Buffer) ->
+ try limited_deque:to_list(Buffer) of
+ Entries -> buffer_payload_bytes(Entries)
+ catch
+ _:_ -> 0
+ end.
+
-spec drop_acked_buffer(
seq(), eqwalizer:dynamic(limited_deque:deque() | [map()])
) -> eqwalizer:dynamic(limited_deque:deque() | [map()]).
@@ -325,11 +343,41 @@ handle_resume(Seq, SocketPid, #{seq := CurrentSeq} = State) ->
MissedEvents = events_after_seq(Seq, BufferList),
NewState0 = cancel_offline_timer(cancel_resume_timer(State)),
NewState1 = replace_socket(SocketPid, NewState0),
+ ReplyEvents = replay_missed_events_inline(MissedEvents, SocketPid),
NewState = NewState1#{status => ResumeStatus, resume_status => ResumeStatus},
NewState2 = ensure_presence_attached_on_resume(
NewState, SessionId, ResumeStatus, Afk, Mobile
),
- {reply, {ok, MissedEvents, CurrentSeq}, NewState2}.
+ {reply, {ok, ReplyEvents, CurrentSeq}, NewState2}.
+
+-spec replay_missed_events_inline([map()], pid()) -> [map()].
+replay_missed_events_inline(MissedEvents, SocketPid) ->
+ case inline_replay_allowed(MissedEvents) of
+ true ->
+ lists:foreach(fun(Event) -> send_missed_event(SocketPid, Event) end, MissedEvents),
+ [];
+ false ->
+ MissedEvents
+ end.
+
+-spec inline_replay_allowed([map()]) -> boolean().
+inline_replay_allowed(MissedEvents) ->
+ within_inline_replay_limit(MissedEvents, ?MAX_INLINE_RESUME_REPLAY).
+
+-spec within_inline_replay_limit([map()], non_neg_integer()) -> boolean().
+within_inline_replay_limit([], _Remaining) ->
+ true;
+within_inline_replay_limit(_MissedEvents, 0) ->
+ false;
+within_inline_replay_limit([_Event | Rest], Remaining) ->
+ within_inline_replay_limit(Rest, Remaining - 1).
+
+-spec send_missed_event(pid(), map()) -> ok.
+send_missed_event(SocketPid, #{event := Event, data := Data, seq := EventSeq}) ->
+ SocketPid ! {dispatch, Event, guild_data_wire:payload(Data), EventSeq},
+ ok;
+send_missed_event(_SocketPid, _Event) ->
+ ok.
-spec ensure_presence_attached_on_resume(
session_state(), session_id(), status(), boolean(), boolean()
@@ -509,7 +557,6 @@ serialize_state(State) ->
resume_status => maps:get(resume_status, State, maps:get(status, State)),
afk => maps:get(afk, State),
mobile => maps:get(mobile, State),
- buffer => maps:get(buffer, State),
ready => maps:get(ready, State),
bot => maps:get(bot, State, false),
shard => maps:get(shard, State, undefined),
@@ -553,20 +600,340 @@ serialize_transfer_identity(State) ->
debounce_reactions => maps:get(debounce_reactions, State, false)
}.
--spec transfer_buffer(eqwalizer:dynamic(limited_deque:deque() | [map()])) -> [term()].
-transfer_buffer(Buffer) when is_list(Buffer) -> Buffer;
-transfer_buffer(Buffer) -> limited_deque:to_list(Buffer).
-
-spec serialize_transfer_runtime(session_state()) -> map().
serialize_transfer_runtime(State) ->
+ {Buffer, AckSeq} = transfer_buffer(
+ maps:get(buffer, State, []), maps:get(ack_seq, State, 0)
+ ),
#{
channels => maps:get(channels, State, #{}),
relationships => maps:get(relationships, State, #{}),
seq => maps:get(seq, State, 0),
- ack_seq => maps:get(ack_seq, State, 0),
- buffer => transfer_buffer(maps:get(buffer, State, [])),
+ ack_seq => AckSeq,
+ buffer => Buffer,
collected_guild_states => maps:get(collected_guild_states, State, []),
collected_sessions => maps:get(collected_sessions, State, []),
collected_presences => maps:get(collected_presences, State, []),
guild_subscription_state => maps:get(guild_subscription_state, State, #{})
}.
+
+%% Always normalises to a list before trimming: a limited_deque left in the transfer
+%% map does not survive the handoff to the receiving node's session_init.
+-spec transfer_buffer(
+ eqwalizer:dynamic(limited_deque:deque() | [map()]), seq()
+) -> {[map()], seq()}.
+transfer_buffer(Buffer, AckSeq) when is_list(Buffer) ->
+ trim_transfer_list(Buffer, transfer_payload_bytes(Buffer), AckSeq);
+transfer_buffer(Buffer, AckSeq) ->
+ ListBuffer = limited_deque:to_list(Buffer),
+ trim_transfer_list(ListBuffer, transfer_payload_bytes(ListBuffer), AckSeq).
+
+-spec trim_transfer_list([map()], non_neg_integer(), seq()) -> {[map()], seq()}.
+trim_transfer_list(Buffer, Bytes, AckSeq) when Bytes =< ?MAX_TRANSFER_PRE_ENCODED_BYTES ->
+ {Buffer, AckSeq};
+trim_transfer_list([], _Bytes, AckSeq) ->
+ {[], AckSeq};
+trim_transfer_list([Dropped | Rest], Bytes, AckSeq) ->
+ trim_transfer_list(
+ Rest,
+ max(0, Bytes - entry_payload_bytes(Dropped)),
+ dropped_ack_seq(Dropped, AckSeq)
+ ).
+
+-spec transfer_payload_bytes([term()]) -> non_neg_integer().
+transfer_payload_bytes(Entries) ->
+ lists:foldl(fun(Entry, Acc) -> Acc + entry_payload_bytes(Entry) end, 0, Entries).
+
+-spec entry_payload_bytes(term()) -> non_neg_integer().
+entry_payload_bytes(#{data := {pre_encoded, Payload}}) when is_binary(Payload) ->
+ byte_size(Payload);
+entry_payload_bytes(_Entry) ->
+ 0.
+
+-spec dropped_ack_seq(term(), seq()) -> seq().
+dropped_ack_seq(#{seq := Seq}, AckSeq) when is_integer(Seq), Seq > AckSeq ->
+ Seq;
+dropped_ack_seq(_Dropped, AckSeq) ->
+ AckSeq.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+resume_flag_state(Overrides) ->
+ maps:merge(
+ #{
+ id => <<"session-resume-flag-test">>,
+ status => online,
+ afk => false,
+ mobile => false,
+ presence_pid => undefined,
+ presence_mref => undefined,
+ socket_pid => undefined,
+ socket_mref => undefined,
+ resume_timer => undefined,
+ offline_timer => undefined,
+ ack_seq => 0,
+ buffer => [],
+ seq => 0
+ },
+ Overrides
+ ).
+
+buffered_event(Seq) ->
+ #{seq => Seq, event => message_create, data => #{<<"id">> => Seq}}.
+
+with_env(Key, Value, Fun) ->
+ Previous = application:get_env(fluxer_gateway, Key),
+ application:set_env(fluxer_gateway, Key, Value),
+ try
+ Fun()
+ after
+ restore_env(Key, Previous)
+ end.
+
+restore_env(Key, {ok, Previous}) ->
+ application:set_env(fluxer_gateway, Key, Previous);
+restore_env(Key, undefined) ->
+ application:unset_env(fluxer_gateway, Key).
+
+next_dispatch() ->
+ receive
+ {dispatch, _Event, _Data, _Seq} = Msg -> Msg
+ after 200 -> no_dispatch
+ end.
+
+drain_dispatches() ->
+ receive
+ {dispatch, _Event, _Data, _Seq} -> drain_dispatches()
+ after 0 -> ok
+ end.
+
+count_dispatches() ->
+ count_dispatches(0).
+
+count_dispatches(Count) ->
+ receive
+ {dispatch, _Event, _Data, _Seq} -> count_dispatches(Count + 1)
+ after 0 -> Count
+ end.
+
+inline_replay_sends_backlog_and_replies_empty_test() ->
+ drain_dispatches(),
+ State0 = resume_flag_state(#{seq => 2, buffer => [buffered_event(1), buffered_event(2)]}),
+ {reply, {ok, [], 2}, _State1} = handle_resume(0, self(), State0),
+ First = next_dispatch(),
+ Second = next_dispatch(),
+ ?assertEqual(
+ {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 1}), 1}, First
+ ),
+ ?assertEqual(
+ {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 2}), 2}, Second
+ ),
+ ?assertEqual(no_dispatch, next_dispatch()).
+
+inline_replay_sends_wire_payload_test() ->
+ drain_dispatches(),
+ State0 = resume_flag_state(#{seq => 1, buffer => [buffered_event(1)]}),
+ {reply, {ok, [], 1}, _State1} = handle_resume(0, self(), State0),
+ {dispatch, message_create, Data, 1} = next_dispatch(),
+ ?assertEqual(#{<<"id">> => <<"1">>}, Data).
+
+inline_replay_skips_malformed_buffer_entry_test() ->
+ drain_dispatches(),
+ State0 = resume_flag_state(#{seq => 2, buffer => [#{seq => 1}, buffered_event(2)]}),
+ {reply, {ok, [], 2}, _State1} = handle_resume(0, self(), State0),
+ ?assertEqual(
+ {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 2}), 2},
+ next_dispatch()
+ ),
+ ?assertEqual(no_dispatch, next_dispatch()).
+
+inline_replay_sends_nothing_when_backlog_empty_test() ->
+ drain_dispatches(),
+ State0 = resume_flag_state(#{seq => 1, ack_seq => 1, buffer => [buffered_event(1)]}),
+ {reply, {ok, [], 1}, _State1} = handle_resume(1, self(), State0),
+ ?assertEqual(no_dispatch, next_dispatch()).
+
+inline_replay_bounded_batch_is_sent_inline_test() ->
+ drain_dispatches(),
+ Events = [buffered_event(Seq) || Seq <- lists:seq(1, ?MAX_INLINE_RESUME_REPLAY)],
+ State0 = resume_flag_state(#{seq => ?MAX_INLINE_RESUME_REPLAY, buffer => Events}),
+ {reply, {ok, [], ?MAX_INLINE_RESUME_REPLAY}, _State1} = handle_resume(0, self(), State0),
+ ?assertEqual(?MAX_INLINE_RESUME_REPLAY, count_dispatches()).
+
+inline_replay_oversized_batch_falls_back_to_reply_test() ->
+ drain_dispatches(),
+ Over = ?MAX_INLINE_RESUME_REPLAY + 1,
+ Events = [buffered_event(Seq) || Seq <- lists:seq(1, Over)],
+ State0 = resume_flag_state(#{seq => Over, buffer => Events}),
+ {reply, {ok, Missed, Over}, _State1} = handle_resume(0, self(), State0),
+ ?assertEqual(Over, length(Missed)),
+ ?assertEqual(no_dispatch, next_dispatch()).
+
+inline_replay_oversized_batch_still_attaches_socket_test() ->
+ drain_dispatches(),
+ Over = ?MAX_INLINE_RESUME_REPLAY + 1,
+ Events = [buffered_event(Seq) || Seq <- lists:seq(1, Over)],
+ State0 = resume_flag_state(#{seq => Over, buffer => Events}),
+ {reply, {ok, _Missed, Over}, State1} = handle_resume(0, self(), State0),
+ ?assertEqual(self(), maps:get(socket_pid, State1)).
+
+pre_encoded_buffered_event(Seq) ->
+ #{seq => Seq, event => message_create, data => {pre_encoded, <<"{}">>}}.
+
+heartbeat_ack_trims_pre_encoded_entries_test() ->
+ Buffer = limited_deque:from_list(
+ [pre_encoded_buffered_event(1), pre_encoded_buffered_event(2)], 4096, 16777216
+ ),
+ State0 = resume_flag_state(#{seq => 2, buffer => Buffer}),
+ {reply, true, State1} = handle_heartbeat_ack(1, State0),
+ Remaining = limited_deque:to_list(maps:get(buffer, State1)),
+ ?assertEqual([2], [maps:get(seq, Event) || Event <- Remaining]).
+
+heartbeat_ack_keeps_trimming_pre_encoded_head_test() ->
+ Buffer = limited_deque:from_list(
+ [pre_encoded_buffered_event(Seq) || Seq <- lists:seq(1, 4)], 4096, 16777216
+ ),
+ State0 = resume_flag_state(#{seq => 4, buffer => Buffer}),
+ {reply, true, State1} = handle_heartbeat_ack(4, State0),
+ ?assertEqual(0, limited_deque:size(maps:get(buffer, State1))).
+
+inline_replay_sends_pre_encoded_entry_unchanged_test() ->
+ drain_dispatches(),
+ Entry = pre_encoded_buffered_event(1),
+ State0 = resume_flag_state(#{seq => 1, buffer => [Entry]}),
+ {reply, {ok, [], 1}, _State1} = handle_resume(0, self(), State0),
+ ?assertEqual({dispatch, message_create, maps:get(data, Entry), 1}, next_dispatch()).
+
+oversized_replay_returns_pre_encoded_entries_unchanged_test() ->
+ drain_dispatches(),
+ Over = ?MAX_INLINE_RESUME_REPLAY + 1,
+ Entries = [pre_encoded_buffered_event(Seq) || Seq <- lists:seq(1, Over)],
+ State0 = resume_flag_state(#{seq => Over, buffer => Entries}),
+ {reply, {ok, Missed, Over}, _State1} = handle_resume(0, self(), State0),
+ ?assertEqual(Entries, Missed),
+ ?assertEqual(no_dispatch, next_dispatch()).
+
+resume_ignores_withdrawn_buffer_floor_env_test() ->
+ drain_dispatches(),
+ State0 = resume_flag_state(#{seq => 5, buffer => [buffered_event(4), buffered_event(5)]}),
+ with_env(session_resume_buffer_floor, true, fun() ->
+ {reply, {ok, [], 5}, _State1} = handle_resume(2, self(), State0),
+ ?assertMatch({dispatch, _, _, 4}, next_dispatch()),
+ ?assertMatch({dispatch, _, _, 5}, next_dispatch()),
+ ?assertEqual(no_dispatch, next_dispatch())
+ end).
+
+serialize_state_omits_replay_buffer_test() ->
+ State = resume_flag_state(#{
+ user_id => 12345,
+ user_data => #{},
+ version => 9,
+ properties => #{},
+ ready => undefined,
+ guilds => #{},
+ collected_guild_states => [],
+ collected_sessions => [],
+ buffer => [buffered_event(1)]
+ }),
+ Serialized = serialize_state(State),
+ ?assertEqual(false, maps:is_key(buffer, Serialized)),
+ ?assertEqual(false, maps:is_key(buffer_bytes, Serialized)),
+ ?assertEqual(0, maps:get(seq, Serialized)).
+
+transfer_entry(Seq, PayloadBytes) ->
+ #{
+ seq => Seq,
+ event => message_create,
+ data => {pre_encoded, binary:copy(<<"x">>, PayloadBytes)}
+ }.
+
+transfer_deque(Entries) ->
+ limited_deque:from_list(Entries, 4096, 16777216).
+
+transfer_runtime_state(Buffer, AckSeq) ->
+ #{
+ channels => #{},
+ relationships => #{},
+ seq => 4,
+ ack_seq => AckSeq,
+ buffer => Buffer,
+ collected_guild_states => [],
+ collected_sessions => [],
+ collected_presences => [],
+ guild_subscription_state => #{}
+ }.
+
+oversized_transfer_entries() ->
+ [transfer_entry(Seq, 1500000) || Seq <- lists:seq(1, 4)].
+
+transfer_buffer_is_bounded_test() ->
+ Deque = transfer_deque(oversized_transfer_entries()),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),
+ Entries = maps:get(buffer, Runtime),
+ ?assert(transfer_payload_bytes(Entries) =< ?MAX_TRANSFER_PRE_ENCODED_BYTES),
+ ?assertEqual([3, 4], [maps:get(seq, Entry) || Entry <- Entries]),
+ ?assertEqual(2, maps:get(ack_seq, Runtime)).
+
+transfer_buffer_admits_one_full_size_event_test() ->
+ Single = transfer_entry(1, 2097152),
+ Deque = transfer_deque([Single]),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),
+ ?assertEqual([Single], maps:get(buffer, Runtime)),
+ ?assertEqual(0, maps:get(ack_seq, Runtime)).
+
+transfer_buffer_bound_keeps_entry_representation_test() ->
+ Deque = transfer_deque(oversized_transfer_entries()),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),
+ Entries = maps:get(buffer, Runtime),
+ ?assertEqual(Entries, session_init:normalize_buffer(Entries)),
+ ?assertEqual([transfer_entry(3, 1500000), transfer_entry(4, 1500000)], Entries).
+
+transfer_buffer_bound_leaves_small_buffers_alone_test() ->
+ Deque = transfer_deque([buffered_event(1), buffered_event(2)]),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),
+ ?assertEqual(limited_deque:to_list(Deque), maps:get(buffer, Runtime)),
+ ?assertEqual(0, maps:get(ack_seq, Runtime)).
+
+fat_map_entry(Seq) ->
+ #{
+ seq => Seq,
+ event => message_create,
+ data => #{<<"content">> => lists:duplicate(200000, $x)}
+ }.
+
+transfer_buffer_bound_never_truncates_map_entries_test() ->
+ Entries = [fat_map_entry(Seq) || Seq <- lists:seq(1, 2)],
+ Deque = transfer_deque(Entries),
+ ?assert(limited_deque:bytes(Deque) > ?MAX_TRANSFER_PRE_ENCODED_BYTES),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),
+ ?assertEqual(limited_deque:to_list(Deque), maps:get(buffer, Runtime)),
+ ?assertEqual(0, maps:get(ack_seq, Runtime)).
+
+transfer_buffer_bound_never_lowers_ack_seq_test() ->
+ Deque = transfer_deque([buffered_event(9)]),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 8)),
+ ?assertEqual(8, maps:get(ack_seq, Runtime)).
+
+transfer_buffer_bounds_list_buffers_test() ->
+ Entries = oversized_transfer_entries(),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Entries, 0)),
+ ?assertEqual([3, 4], [maps:get(seq, Entry) || Entry <- maps:get(buffer, Runtime)]),
+ ?assertEqual(2, maps:get(ack_seq, Runtime)).
+
+transfer_buffer_bound_makes_stale_resume_invalid_test() ->
+ Deque = transfer_deque(oversized_transfer_entries()),
+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),
+ Restored = resume_flag_state(#{
+ seq => 4,
+ ack_seq => maps:get(ack_seq, Runtime),
+ buffer => maps:get(buffer, Runtime)
+ }),
+ ?assertMatch({reply, invalid_seq, _}, handle_resume(1, self(), Restored)),
+ drain_dispatches(),
+ {reply, {ok, [], 4}, _} = handle_resume(2, self(), Restored),
+ ?assertMatch({dispatch, _, _, 3}, next_dispatch()),
+ ?assertMatch({dispatch, _, _, 4}, next_dispatch()),
+ ?assertEqual(no_dispatch, next_dispatch()).
+
+-endif.
diff --git a/fluxer_gateway/src/session/session_ready_dispatch.erl b/fluxer_gateway/src/session/session_ready_dispatch.erl
index 910c564b6..905c30d9a 100644
--- a/fluxer_gateway/src/session/session_ready_dispatch.erl
+++ b/fluxer_gateway/src/session/session_ready_dispatch.erl
@@ -70,7 +70,8 @@ dispatch_ready_to_socket(State) ->
IsBot, CollectedGuilds, Guilds, StateAfterReady
),
schedule_call_creates(StateAfterGuilds, SessionId),
- FinalState = StateAfterGuilds#{
+ StateAfterPresences = release_pending_presences(StateAfterGuilds, CollectedPresences),
+ FinalState = StateAfterPresences#{
ready => undefined,
collected_guild_states => [],
collected_sessions => [],
@@ -79,6 +80,65 @@ dispatch_ready_to_socket(State) ->
erlang:garbage_collect(),
{noreply, FinalState}.
+-spec release_pending_presences(session_state(), [map()]) -> session_state().
+release_pending_presences(State, CollectedPresences) ->
+ Covered = covered_presence_ids(CollectedPresences),
+ Pruned = drop_covered_pending_presences(State, Covered),
+ Flushed = session_dispatch:flush_all_pending_presences(Pruned),
+ Flushed#{suppress_presence_updates => false}.
+
+-spec covered_presence_ids([map()]) -> #{integer() => true}.
+covered_presence_ids(CollectedPresences) ->
+ lists:foldl(
+ fun(Presence, Acc) ->
+ case presence_user_id(Presence) of
+ undefined -> Acc;
+ UserId -> Acc#{UserId => true}
+ end
+ end,
+ #{},
+ CollectedPresences
+ ).
+
+-spec drop_covered_pending_presences(session_state(), #{integer() => true}) -> session_state().
+drop_covered_pending_presences(State, Covered) when map_size(Covered) =:= 0 ->
+ State;
+drop_covered_pending_presences(State, Covered) ->
+ case maps:get(pending_presences, State, undefined) of
+ undefined ->
+ State;
+ Pending ->
+ State#{pending_presences => filter_pending_presences(Pending, Covered)}
+ end.
+
+-spec filter_pending_presences(term(), #{integer() => true}) -> term().
+filter_pending_presences(Pending, Covered) ->
+ Keep = fun(Entry) -> not covered_entry(Entry, Covered) end,
+ try queue:filter(Keep, Pending) of
+ Filtered -> Filtered
+ catch
+ error:_ when is_list(Pending) -> lists:filter(Keep, Pending);
+ error:_ -> Pending
+ end.
+
+-spec covered_entry(term(), #{integer() => true}) -> boolean().
+covered_entry(#{user_id := UserId}, Covered) when is_integer(UserId) ->
+ maps:is_key(UserId, Covered);
+covered_entry(_Entry, _Covered) ->
+ false.
+
+-spec presence_user_id(term()) -> integer() | undefined.
+presence_user_id(Presence) when is_map(Presence) ->
+ User = maps:get(<<"user">>, Presence, #{}),
+ presence_id(User);
+presence_user_id(_Presence) ->
+ undefined.
+
+-spec presence_id(term()) -> integer() | undefined.
+presence_id(User) when is_map(User) ->
+ snowflake_id:parse_maybe(maps:get(<<"id">>, User, undefined));
+presence_id(_User) ->
+ undefined.
-spec is_staff_session(session_state()) -> boolean().
is_staff_session(#{is_staff := true}) ->
true;
diff --git a/fluxer_gateway/src/utils/custom_status_expiry.erl b/fluxer_gateway/src/utils/custom_status_expiry.erl
new file mode 100644
index 000000000..b0f47974a
--- /dev/null
+++ b/fluxer_gateway/src/utils/custom_status_expiry.erl
@@ -0,0 +1,406 @@
+%% SPDX-License-Identifier: AGPL-3.0-or-later
+
+-module(custom_status_expiry).
+-typing([eqwalizer]).
+
+-export([
+ enabled/0,
+ clear_if_expired/1,
+ clear_if_expired/2,
+ remaining_ms/2,
+ next_wakeup_ms/1,
+ reconcile_message/0,
+ repair/1
+]).
+
+-export_type([custom_status/0]).
+
+-type custom_status() :: map() | null.
+-type user_id() :: integer().
+-type wakeup() :: {ok, pos_integer()} | none.
+-type offset() :: {ok, integer()} | none.
+
+-define(ENABLED_KEY, custom_status_expiry_enabled).
+-define(MAX_REPAIR_WINDOW_SECONDS, 86400).
+-define(MAX_TIMER_MS, 86400000).
+-define(WAKEUP_JITTER_MS, 5000).
+
+-spec enabled() -> boolean().
+enabled() ->
+ application:get_env(fluxer_gateway, ?ENABLED_KEY, false) =:= true.
+
+-spec clear_if_expired(term()) -> custom_status().
+clear_if_expired(CustomStatus) ->
+ clear_when_enabled(enabled(), CustomStatus).
+
+-spec clear_when_enabled(boolean(), term()) -> custom_status().
+clear_when_enabled(true, CustomStatus) ->
+ clear_if_expired(CustomStatus, erlang:system_time(millisecond));
+clear_when_enabled(false, CustomStatus) when is_map(CustomStatus) ->
+ CustomStatus;
+clear_when_enabled(false, _CustomStatus) ->
+ null.
+
+-spec clear_if_expired(term(), term()) -> custom_status().
+clear_if_expired(CustomStatus, NowMs) when is_map(CustomStatus), is_integer(NowMs) ->
+ keep_or_clear(expired(CustomStatus, NowMs), CustomStatus);
+clear_if_expired(CustomStatus, _NowMs) when is_map(CustomStatus) ->
+ CustomStatus;
+clear_if_expired(_CustomStatus, _NowMs) ->
+ null.
+
+-spec keep_or_clear(boolean(), map()) -> custom_status().
+keep_or_clear(true, _CustomStatus) -> null;
+keep_or_clear(false, CustomStatus) -> CustomStatus.
+
+-spec expired(map(), integer()) -> boolean().
+expired(CustomStatus, NowMs) ->
+ case remaining_ms(CustomStatus, NowMs) of
+ {ok, RemainingMs} -> RemainingMs =< 0;
+ none -> false
+ end.
+
+-spec remaining_ms(term(), term()) -> offset().
+remaining_ms(CustomStatus, NowMs) when is_map(CustomStatus), is_integer(NowMs) ->
+ offset_from(expires_at_ms(maps:get(<<"expires_at">>, CustomStatus, undefined)), NowMs);
+remaining_ms(_CustomStatus, _NowMs) ->
+ none.
+
+-spec offset_from(offset(), integer()) -> offset().
+offset_from({ok, ExpiryMs}, NowMs) -> {ok, ExpiryMs - NowMs};
+offset_from(none, _NowMs) -> none.
+
+-spec expires_at_ms(term()) -> offset().
+expires_at_ms(Value) when is_binary(Value) ->
+ parse_rfc3339(binary_to_list(Value));
+expires_at_ms(Value) when is_list(Value) ->
+ parse_rfc3339(Value);
+expires_at_ms(_Value) ->
+ none.
+
+-spec parse_rfc3339(term()) -> offset().
+parse_rfc3339(Chars) ->
+ try calendar:rfc3339_to_system_time(Chars, [{unit, millisecond}]) of
+ Ms when is_integer(Ms) -> {ok, Ms};
+ _Other -> none
+ catch
+ _Class:_Reason -> none
+ end.
+
+-spec next_wakeup_ms(term()) -> wakeup().
+next_wakeup_ms(CustomStatus) ->
+ next_wakeup_ms(enabled(), CustomStatus).
+
+-spec next_wakeup_ms(boolean(), term()) -> wakeup().
+next_wakeup_ms(false, _CustomStatus) ->
+ none;
+next_wakeup_ms(true, CustomStatus) ->
+ wakeup_delay(remaining_ms(CustomStatus, erlang:system_time(millisecond))).
+
+-spec wakeup_delay(offset()) -> wakeup().
+wakeup_delay(none) ->
+ none;
+wakeup_delay({ok, RemainingMs}) when RemainingMs =< 0 ->
+ none;
+wakeup_delay({ok, RemainingMs}) when RemainingMs > ?MAX_TIMER_MS ->
+ {ok, jittered(?MAX_TIMER_MS)};
+wakeup_delay({ok, RemainingMs}) ->
+ {ok, jittered(RemainingMs)}.
+
+-spec jittered(pos_integer()) -> pos_integer().
+jittered(DelayMs) ->
+ DelayMs + rand:uniform(?WAKEUP_JITTER_MS).
+
+-spec reconcile_message() -> {'$gen_cast', reconcile_flattened_presence}.
+reconcile_message() ->
+ {'$gen_cast', reconcile_flattened_presence}.
+
+-spec repair(term()) -> map().
+repair(WindowSeconds) when
+ is_integer(WindowSeconds), WindowSeconds >= 0, WindowSeconds =< ?MAX_REPAIR_WINDOW_SECONDS
+->
+ Considered = local_presence_user_ids(),
+ Pids = expiring_presence_pids(Considered),
+ Total = length(Pids),
+ ok = schedule_reconciles(Pids, Total, WindowSeconds * 1000),
+ #{
+ scheduled => Total,
+ considered => length(Considered),
+ window_seconds => WindowSeconds,
+ enabled => enabled()
+ };
+repair(_WindowSeconds) ->
+ #{error => invalid_window_seconds}.
+
+%% Only presences whose cached payload actually carries an expires_at are worth
+%% nudging. Nudging every local presence would cost one payload rebuild and one
+%% replicated cache write each, thousands per node, to correct a few dozen.
+%% Filtering on "has an expiry" rather than "is expired" is deliberate: it also
+%% arms the ongoing wakeup for statuses that have not expired yet.
+-spec expiring_presence_pids([integer()]) -> [pid()].
+expiring_presence_pids(UserIds) ->
+ [
+ Pid
+ || UserId <- UserIds,
+ has_expires_at(UserId),
+ Pid <- [presence_pid(UserId)],
+ is_pid(Pid)
+ ].
+
+-spec has_expires_at(integer()) -> boolean().
+has_expires_at(UserId) ->
+ case catch presence_cache:get(UserId) of
+ {ok, Presence} when is_map(Presence) ->
+ presence_carries_expiry(Presence);
+ _ ->
+ false
+ end.
+
+%% presence_cache:get/1 returns {ok, map()} | not_found, never a bare map, and
+%% expires_at_ms/1 takes the timestamp value rather than the custom_status map.
+-spec presence_carries_expiry(map()) -> boolean().
+presence_carries_expiry(Presence) ->
+ case maps:get(<<"custom_status">>, Presence, null) of
+ CustomStatus when is_map(CustomStatus) ->
+ expires_at_ms(maps:get(<<"expires_at">>, CustomStatus, null)) =/= none;
+ _ ->
+ false
+ end.
+
+-spec local_presence_user_ids() -> [user_id()].
+local_presence_user_ids() ->
+ try presence_manager_shards:determine_count() of
+ {ShardCount, _Source} -> shard_user_ids(ShardCount)
+ catch
+ _Class:_Reason -> []
+ end.
+
+-spec shard_user_ids(pos_integer()) -> [user_id()].
+shard_user_ids(ShardCount) ->
+ lists:append([
+ presence_manager:get_shard_user_ids(Index)
+ || Index <- lists:seq(0, ShardCount - 1)
+ ]).
+
+-spec presence_pid(user_id()) -> pid() | undefined.
+presence_pid(UserId) ->
+ process_registry:registry_whereis(process_registry:build_process_key(presence, UserId)).
+
+-spec schedule_reconciles([pid()], non_neg_integer(), non_neg_integer()) -> ok.
+schedule_reconciles(Pids, Total, WindowMs) ->
+ Message = reconcile_message(),
+ _ = lists:foldl(
+ fun(Pid, Index) ->
+ _ = schedule_one(Pid, spread_ms(Index, Total, WindowMs), Message),
+ Index + 1
+ end,
+ 0,
+ Pids
+ ),
+ ok.
+
+%% A single unreachable destination must not abort the sweep and leave the
+%% operator unable to tell how many nudges were actually scheduled.
+-spec schedule_one(pid(), non_neg_integer(), term()) -> ok.
+schedule_one(Pid, Delay, Message) ->
+ try erlang:send_after(Delay, Pid, Message) of
+ _Ref -> ok
+ catch
+ _Class:_Reason -> ok
+ end.
+
+-spec spread_ms(non_neg_integer(), non_neg_integer(), non_neg_integer()) -> non_neg_integer().
+spread_ms(_Index, 0, _WindowMs) -> 0;
+spread_ms(_Index, _Total, 0) -> 0;
+spread_ms(Index, Total, WindowMs) -> (Index * WindowMs) div Total.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+presence_carries_expiry_detects_an_expiry_test() ->
+ ?assert(
+ presence_carries_expiry(#{
+ <<"custom_status">> => #{<<"expires_at">> => <<"2026-05-13T13:02:27.497Z">>}
+ })
+ ),
+ ?assertNot(presence_carries_expiry(#{<<"custom_status">> => #{<<"text">> => <<"hi">>}})),
+ ?assertNot(presence_carries_expiry(#{<<"custom_status">> => null})),
+ ?assertNot(presence_carries_expiry(#{})).
+
+-define(LIVE_EXPIRES_AT, <<"2026-05-13T13:02:27.497Z">>).
+
+epoch_ms(Iso) ->
+ calendar:rfc3339_to_system_time(binary_to_list(Iso), [{unit, millisecond}]).
+
+rfc3339(EpochMs) ->
+ list_to_binary(
+ calendar:system_time_to_rfc3339(EpochMs, [{unit, millisecond}, {offset, "Z"}])
+ ).
+
+expiry_ms() ->
+ epoch_ms(?LIVE_EXPIRES_AT).
+
+status(ExpiresAt) ->
+ #{
+ <<"emoji_animated">> => false,
+ <<"emoji_name">> => null,
+ <<"expires_at">> => ExpiresAt,
+ <<"text">> => <<"brb">>
+ }.
+
+status_without_expiry() ->
+ #{<<"emoji_animated">> => false, <<"emoji_name">> => null, <<"text">> => <<"brb">>}.
+
+parser_anchors_on_the_unix_epoch_test() ->
+ ?assertEqual(0, epoch_ms(<<"1970-01-01T00:00:00.000Z">>)),
+ ?assertEqual(1500, epoch_ms(<<"1970-01-01T00:00:01.500Z">>)).
+
+parser_keeps_milliseconds_test() ->
+ ?assertEqual(497, expiry_ms() rem 1000).
+
+not_expired_test() ->
+ Status = status(?LIVE_EXPIRES_AT),
+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 60000)).
+
+expires_one_second_in_the_future_test() ->
+ Status = status(?LIVE_EXPIRES_AT),
+ ?assertEqual({ok, 1000}, remaining_ms(Status, expiry_ms() - 1000)),
+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 1000)).
+
+boundary_is_expired_test() ->
+ Status = status(?LIVE_EXPIRES_AT),
+ ?assertEqual({ok, 0}, remaining_ms(Status, expiry_ms())),
+ ?assertEqual(null, clear_if_expired(Status, expiry_ms())).
+
+one_millisecond_before_boundary_is_kept_test() ->
+ Status = status(?LIVE_EXPIRES_AT),
+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 1)).
+
+expired_test() ->
+ Status = status(?LIVE_EXPIRES_AT),
+ ?assertEqual(null, clear_if_expired(Status, expiry_ms() + 1)),
+ ?assertEqual(null, clear_if_expired(Status, expiry_ms() + 194 * 86400000)).
+
+absent_expires_at_is_kept_test() ->
+ Status = status_without_expiry(),
+ ?assertEqual(none, remaining_ms(Status, expiry_ms())),
+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms())).
+
+null_expires_at_is_kept_test() ->
+ Status = status(null),
+ ?assertEqual(none, remaining_ms(Status, expiry_ms())),
+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms())).
+
+malformed_expires_at_is_kept_test() ->
+ lists:foreach(
+ fun(Value) ->
+ Status = status(Value),
+ ?assertEqual(none, remaining_ms(Status, expiry_ms())),
+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms()))
+ end,
+ [<<"">>, <<"not-a-date">>, <<"2026-13-45T99:99:99Z">>, 1747141347, [], #{}, true]
+ ).
+
+custom_status_null_test() ->
+ ?assertEqual(none, remaining_ms(null, expiry_ms())),
+ ?assertEqual(null, clear_if_expired(null, expiry_ms())).
+
+custom_status_absent_test() ->
+ ?assertEqual(none, remaining_ms(undefined, expiry_ms())),
+ ?assertEqual(null, clear_if_expired(undefined, expiry_ms())).
+
+non_map_custom_status_test() ->
+ ?assertEqual(null, clear_if_expired(<<"garbage">>, expiry_ms())),
+ ?assertEqual(null, clear_if_expired(123, expiry_ms())).
+
+non_integer_reference_time_keeps_status_test() ->
+ Status = status(?LIVE_EXPIRES_AT),
+ ?assertEqual(Status, clear_if_expired(Status, not_a_time)).
+
+wakeup_none_without_expiry_test() ->
+ ?assertEqual(none, wakeup_delay(remaining_ms(status_without_expiry(), expiry_ms()))),
+ ?assertEqual(none, wakeup_delay(remaining_ms(null, expiry_ms()))).
+
+wakeup_none_when_already_expired_test() ->
+ ?assertEqual(none, wakeup_delay({ok, 0})),
+ ?assertEqual(none, wakeup_delay({ok, -1000})).
+
+wakeup_is_never_early_test() ->
+ {ok, DelayMs} = wakeup_delay({ok, 1000}),
+ ?assert(DelayMs > 1000),
+ ?assert(DelayMs =< 1000 + ?WAKEUP_JITTER_MS).
+
+wakeup_clamps_to_the_timer_limit_test() ->
+ {ok, DelayMs} = wakeup_delay({ok, 400 * ?MAX_TIMER_MS}),
+ ?assert(DelayMs > ?MAX_TIMER_MS),
+ ?assert(DelayMs =< ?MAX_TIMER_MS + ?WAKEUP_JITTER_MS).
+
+disabled_by_default_test() ->
+ application:unset_env(fluxer_gateway, ?ENABLED_KEY),
+ Status = status(?LIVE_EXPIRES_AT),
+ ?assertEqual(false, enabled()),
+ ?assertEqual(Status, clear_if_expired(Status)),
+ ?assertEqual(none, next_wakeup_ms(future_status())).
+
+enabled_clears_expired_and_arms_future_test() ->
+ Live = future_status(),
+ with_expiry_enabled(fun() ->
+ ?assertEqual(true, enabled()),
+ ?assertEqual(null, clear_if_expired(status(?LIVE_EXPIRES_AT))),
+ ?assertEqual(Live, clear_if_expired(Live)),
+ ?assertMatch({ok, _}, next_wakeup_ms(Live))
+ end).
+
+future_status() ->
+ status(rfc3339(erlang:system_time(millisecond) + 3600000)).
+
+reconcile_message_is_a_gen_server_cast_test() ->
+ ?assertEqual({'$gen_cast', reconcile_flattened_presence}, reconcile_message()).
+
+spread_ms_test() ->
+ ?assertEqual(0, spread_ms(0, 0, 60000)),
+ ?assertEqual(0, spread_ms(5, 10, 0)),
+ ?assertEqual(0, spread_ms(0, 10, 60000)),
+ ?assertEqual(30000, spread_ms(5, 10, 60000)),
+ ?assertEqual(54000, spread_ms(9, 10, 60000)).
+
+repair_rejects_a_bad_window_test() ->
+ ?assertEqual(#{error => invalid_window_seconds}, repair(-1)),
+ ?assertEqual(#{error => invalid_window_seconds}, repair(<<"900">>)).
+
+repair_reports_what_it_scheduled_test() ->
+ Result = repair(0),
+ ?assert(is_integer(maps:get(scheduled, Result))),
+ ?assert(maps:get(scheduled, Result) >= 0),
+ ?assertEqual(0, maps:get(window_seconds, Result)),
+ ?assertEqual(enabled(), maps:get(enabled, Result)).
+
+schedule_reconciles_delivers_the_cast_test() ->
+ flush_mailbox(),
+ ok = schedule_reconciles([self()], 1, 0),
+ receive
+ Message -> ?assertEqual(reconcile_message(), Message)
+ after 1000 -> ?assert(false)
+ end.
+
+flush_mailbox() ->
+ receive
+ _Any -> flush_mailbox()
+ after 0 -> ok
+ end.
+
+with_expiry_enabled(Fun) ->
+ Previous = application:get_env(fluxer_gateway, ?ENABLED_KEY),
+ application:set_env(fluxer_gateway, ?ENABLED_KEY, true),
+ try
+ Fun()
+ after
+ restore_expiry_env(Previous)
+ end.
+
+restore_expiry_env(undefined) ->
+ application:unset_env(fluxer_gateway, ?ENABLED_KEY);
+restore_expiry_env({ok, Value}) ->
+ application:set_env(fluxer_gateway, ?ENABLED_KEY, Value).
+
+-endif.
diff --git a/fluxer_gateway/test/guild_member_list_tests.erl b/fluxer_gateway/test/guild_member_list_tests.erl
index 2d609390c..dfbb11d59 100644
--- a/fluxer_gateway/test/guild_member_list_tests.erl
+++ b/fluxer_gateway/test/guild_member_list_tests.erl
@@ -118,6 +118,7 @@ run_broadcast_channel_list_dispatches_sync_immediately() ->
<<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>},
<<"roles">> => []
},
+ ok = guild_member_list_engine:bulk_load(Ref, [{1, <<"one">>, [], true}], []),
OldState = channel_list_state(Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), [
Member
]),
@@ -144,6 +145,9 @@ run_broadcast_channel_list_fans_out_repeated_syncs() ->
#{<<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>}},
#{<<"user">> => #{<<"id">> => <<"2">>, <<"username">> => <<"two">>}}
],
+ ok = guild_member_list_engine:bulk_load(
+ Ref, [{1, <<"one">>, [], true}, {2, <<"two">>, [], true}], []
+ ),
OldState = channel_list_state(
Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), Members
),
@@ -408,6 +412,7 @@ presence_delta_state(Ref) ->
<<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>},
<<"roles">> => []
},
+ ok = guild_member_list_engine:bulk_load(Ref, [{1, <<"one">>, [], true}], []),
channel_list_state(Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), [Member]).
presence_map(Status, Mobile, CustomStatus) ->
diff --git a/fluxer_gateway/test/guild_voice_move_tests.erl b/fluxer_gateway/test/guild_voice_move_tests.erl
index a25ef606d..81629c7f2 100644
--- a/fluxer_gateway/test/guild_voice_move_tests.erl
+++ b/fluxer_gateway/test/guild_voice_move_tests.erl
@@ -271,6 +271,9 @@ guild_server_stub(StateData, TestPid) ->
guild_server_stub_loop(StateData, TestPid) ->
receive
+ {'$gen_call', From, {get_voice_guild_state}} ->
+ gen:reply(From, undefined),
+ guild_server_stub_loop(StateData, TestPid);
{'$gen_call', From, {get_sessions}} ->
gen:reply(From, StateData),
guild_server_stub_loop(StateData, TestPid);
diff --git a/fluxer_gateway/test/session_lifecycle_tests.erl b/fluxer_gateway/test/session_lifecycle_tests.erl
index ca825e4f5..2da65a101 100644
--- a/fluxer_gateway/test/session_lifecycle_tests.erl
+++ b/fluxer_gateway/test/session_lifecycle_tests.erl
@@ -171,8 +171,8 @@ handle_resume_clamps_truncated_gap_in_replay_buffer_test() ->
#{seq => 5, event => message_create, data => #{}}
]
}),
- {reply, {ok, Missed, 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),
- ?assertEqual([4, 5], [maps:get(seq, Event) || Event <- Missed]),
+ {reply, {ok, [], 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),
+ ?assertEqual([4, 5], collect_dispatched_seqs(2)),
?assertEqual(self(), maps:get(socket_pid, State1)).
handle_resume_clamps_skipped_event_hole_test() ->
@@ -183,8 +183,8 @@ handle_resume_clamps_skipped_event_hole_test() ->
#{seq => 5, event => message_create, data => #{}}
]
}),
- {reply, {ok, Missed, 5}, _State1} = session_lifecycle:handle_resume(2, self(), State0),
- ?assertEqual([3, 5], [maps:get(seq, Event) || Event <- Missed]).
+ {reply, {ok, [], 5}, _State1} = session_lifecycle:handle_resume(2, self(), State0),
+ ?assertEqual([3, 5], collect_dispatched_seqs(2)).
handle_resume_rejects_seq_ahead_of_current_test() ->
State0 = resume_test_state(#{
@@ -211,8 +211,8 @@ handle_resume_accepts_seq_at_ack_seq_test() ->
#{seq => 10, event => message_create, data => #{}}
]
}),
- {reply, {ok, Missed, 10}, _State1} = session_lifecycle:handle_resume(8, self(), State0),
- ?assertEqual([9, 10], [maps:get(seq, Event) || Event <- Missed]).
+ {reply, {ok, [], 10}, _State1} = session_lifecycle:handle_resume(8, self(), State0),
+ ?assertEqual([9, 10], collect_dispatched_seqs(2)).
handle_resume_accepts_contiguous_replay_buffer_test() ->
State0 = resume_test_state(#{
@@ -223,8 +223,8 @@ handle_resume_accepts_contiguous_replay_buffer_test() ->
#{seq => 5, event => message_create, data => #{}}
]
}),
- {reply, {ok, Missed, 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),
- ?assertEqual([3, 4, 5], [maps:get(seq, Event) || Event <- Missed]),
+ {reply, {ok, [], 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),
+ ?assertEqual([3, 4, 5], collect_dispatched_seqs(3)),
?assertEqual(self(), maps:get(socket_pid, State1)).
handle_resume_cancels_pending_resume_timer_test() ->
@@ -445,6 +445,19 @@ resume_test_state(Overrides) ->
Overrides
).
+%% Small replay buffers are delivered inline (dispatched straight to the socket) rather
+%% than returned in the resume reply, so these tests read the missed events back off
+%% self()'s mailbox instead of the (now empty) reply list.
+collect_dispatched_seqs(Count) ->
+ lists:sort([collect_dispatched_seq() || _ <- lists:seq(1, Count)]).
+
+collect_dispatched_seq() ->
+ receive
+ {dispatch, _Event, _Payload, Seq} -> Seq
+ after 200 ->
+ ?assert(false)
+ end.
+
fenced_terminate_releases_the_user_session_count_test() ->
ok = session_abuse_protection:ensure_tables(),
UserId = 900201,