A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(gateway): reconcile hotpatched member-list and push fixes (#2425)
Hampusdd1d554cc618
diff --git a/fluxer_gateway/src/gateway/gateway_http_client_response.erl b/fluxer_gateway/src/gateway/gateway_http_client_response.erlindex 6b8ec7591..dc69b48e4 100644--- a/fluxer_gateway/src/gateway/gateway_http_client_response.erl+++ b/fluxer_gateway/src/gateway/gateway_http_client_response.erl@@ -38,7 +38,7 @@ allow_circuit_request(CircuitKey, RecoveryTimeoutMs) ->-spec update_circuit_state_direct({atom(), binary()}, response(), pos_integer()) -> ok.update_circuit_state_direct(CircuitKey, Result, FailureThreshold) ->Now = erlang:system_time(millisecond),- IsFailure = is_countable_failure(Result),+ IsFailure = is_countable_circuit_failure(CircuitKey, Result),record_result(CircuitKey, IsFailure, Now, FailureThreshold).-spec acquire_inflight_slot(atom(), pos_integer()) -> ok | {error, overloaded}.@@ -185,14 +185,29 @@ retry_update_counter(Table, Key, Op) ->error:badarg -> {error, badarg}end.--spec is_countable_failure(response()) -> boolean().-is_countable_failure({error, nxdomain}) -> false;-is_countable_failure({error, {failed_connect, _}}) -> false;-is_countable_failure({error, timeout}) -> false;-is_countable_failure({error, {timeout, _}}) -> false;-is_countable_failure({error, _}) -> true;-is_countable_failure({ok, StatusCode, _, _}) when StatusCode >= 500 -> true;-is_countable_failure(_) -> false.+-spec is_countable_circuit_failure({atom(), binary()}, response()) -> boolean().+is_countable_circuit_failure({rpc, _Host}, Result) ->+ is_countable_failure_with_transport(Result);+is_countable_circuit_failure({_Workload, _Host}, Result) ->+ is_countable_failure_without_transport(Result).++-spec is_countable_failure_without_transport(response()) -> boolean().+is_countable_failure_without_transport({error, nxdomain}) -> false;+is_countable_failure_without_transport({error, {failed_connect, _}}) -> false;+is_countable_failure_without_transport({error, timeout}) -> false;+is_countable_failure_without_transport({error, {timeout, _}}) -> false;+is_countable_failure_without_transport({error, _}) -> true;+is_countable_failure_without_transport({ok, StatusCode, _, _}) when StatusCode >= 500 -> true;+is_countable_failure_without_transport(_) -> false.++-spec is_countable_failure_with_transport(response()) -> boolean().+is_countable_failure_with_transport({error, nxdomain}) -> true;+is_countable_failure_with_transport({error, {failed_connect, _}}) -> true;+is_countable_failure_with_transport({error, timeout}) -> true;+is_countable_failure_with_transport({error, {timeout, _}}) -> true;+is_countable_failure_with_transport({error, _}) -> true;+is_countable_failure_with_transport({ok, StatusCode, _, _}) when StatusCode >= 500 -> true;+is_countable_failure_with_transport(_) -> false.-spec record_result({atom(), binary()}, boolean(), integer(), pos_integer()) -> ok.record_result(CircuitKey, IsFailure, Now, FailureThreshold) ->@@ -314,3 +329,34 @@ safe_delete(Table, Key) ->catcherror:badarg -> okend.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++circuit_ignores_transport_failures_for_non_rpc_workloads_test() ->+ ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {error, nxdomain})),+ ?assertEqual(+ false, is_countable_circuit_failure({push, <<"h">>}, {error, {failed_connect, []}})+ ),+ ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {error, timeout})),+ ?assertEqual(+ false, is_countable_circuit_failure({push, <<"h">>}, {error, {timeout, connect}})+ ),+ ?assertEqual(true, is_countable_circuit_failure({push, <<"h">>}, {error, closed})),+ ?assertEqual(true, is_countable_circuit_failure({push, <<"h">>}, {ok, 500, [], <<>>})),+ ?assertEqual(false, is_countable_circuit_failure({push, <<"h">>}, {ok, 200, [], <<>>})).++circuit_counts_transport_failures_for_rpc_test() ->+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, nxdomain})),+ ?assertEqual(+ true, is_countable_circuit_failure({rpc, <<"h">>}, {error, {failed_connect, []}})+ ),+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, timeout})),+ ?assertEqual(+ true, is_countable_circuit_failure({rpc, <<"h">>}, {error, {timeout, connect}})+ ),+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {error, closed})),+ ?assertEqual(true, is_countable_circuit_failure({rpc, <<"h">>}, {ok, 500, [], <<>>})),+ ?assertEqual(false, is_countable_circuit_failure({rpc, <<"h">>}, {ok, 200, [], <<>>})).++-endif.diff --git a/fluxer_gateway/src/guild/guild.erl b/fluxer_gateway/src/guild/guild.erlindex 00d190746..b44133945 100644--- a/fluxer_gateway/src/guild/guild.erl+++ b/fluxer_gateway/src/guild/guild.erl@@ -9,6 +9,7 @@-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).-define(HIBERNATE_TIMEOUT, 60000).+-define(VOICE_MEMBERS_TABLE_WARNED, {?MODULE, voice_members_table_unavailable}).-type guild_state() :: map().-type call_reply() ::@@ -46,6 +47,10 @@ handle_call(export_handoff_state, _From, State) ->{reply, {ok, guild_handoff:export_handoff_state(State)}, State};handle_call({get_cached_voice_state_by_connection, ConnectionId}, _From, State) ->handle_cached_voice_state_call(ConnectionId, State);+handle_call({get_guild_id}, _From, State) ->+ {reply, maps:get(id, State, undefined), State};+handle_call({get_voice_guild_state}, _From, State) ->+ {reply, voice_guild_state(State), State};handle_call({dispatch, Request}, _From, State) ->handle_dispatch_call(Request, State);handle_call({reload, NewData}, _From, State) ->@@ -603,6 +608,112 @@ require_map(Value) when is_map(Value) ->require_map(Value) ->erlang:error({badmap, Value}).+-spec voice_guild_state(guild_state()) -> map().+voice_guild_state(State) ->+ case maps:get(data, State, #{}) of+ #{members_ets := Tab} = Data ->+ case voice_members_table_healthy(Tab) of+ true ->+ ok = clear_voice_members_table_warning(),+ project_voice_guild_state(Data, State);+ false ->+ voice_members_table_unavailable(State)+ end;+ _ ->+ voice_members_table_unavailable(State)+ end.++-spec voice_members_table_healthy(term()) -> boolean().+voice_members_table_healthy(Tab) when is_reference(Tab) ->+ try ets:info(eqwalizer:dynamic_cast(Tab), owner) of+ Owner when is_pid(Owner) -> Owner =:= self();+ _ -> false+ catch+ error:badarg -> false+ end;+voice_members_table_healthy(_Tab) ->+ false.++-spec project_voice_guild_state(map(), guild_state()) -> map().+project_voice_guild_state(Data, State) ->+ Projected = maps:with(voice_guild_state_keys(), State),+ project_voice_sessions(Projected#{data => maps:with(voice_guild_data_keys(), Data)}).++%% The sessions map keeps every key it had, so fold order over it is unchanged and the+%% dispatch order of every voice broadcast is unchanged. Only session ENTRIES are narrowed.+-spec project_voice_sessions(map()) -> map().+project_voice_sessions(#{sessions := Sessions} = Projected) when is_map(Sessions) ->+ Projected#{sessions => maps:map(fun project_voice_session/2, Sessions)};+project_voice_sessions(Projected) ->+ Projected.++-spec project_voice_session(term(), term()) -> term().+project_voice_session(_SessionId, Session) when is_map(Session) ->+ maps:without(voice_session_drop_keys(), Session);+project_voice_session(_SessionId, Session) ->+ Session.++%% Every reader reachable from the voice guild state takes only pending_connect, user_id,+%% viewable_channels and pid out of a session entry: guild_sessions:filter_sessions_for_channel,+%% guild_voice_broadcast and guild_virtual_channel_access. These five are read nowhere on that path.+-spec voice_session_drop_keys() -> [atom()].+voice_session_drop_keys() ->+ [active_guilds, user_roles, mref, bot, is_staff].++-spec voice_guild_state_keys() -> [atom() | binary()].+voice_guild_state_keys() ->+ [+ id,+ <<"id">>,+ sessions,+ guild_pid,+ voice_states,+ pending_voice_connections,+ recently_disconnected_voice_states,+ e2ee_room_keys,+ virtual_channel_access,+ virtual_channel_access_pending,+ virtual_channel_access_preserve,+ virtual_channel_access_move_pending,+ test_perm_fun,+ test_force_disconnect_fun,+ test_livekit_fun,+ test_permission_sync_fun+ ].++-spec voice_guild_data_keys() -> [atom() | binary()].+voice_guild_data_keys() ->+ [+ <<"id">>,+ <<"guild">>,+ <<"roles">>,+ <<"role_index">>,+ role_perms_cache,+ <<"channels">>,+ <<"channel_index">>,+ overwrite_perms_cache,+ members_ets+ ].++-spec voice_members_table_unavailable(guild_state()) -> map().+voice_members_table_unavailable(State) ->+ ok = log_voice_members_table_unavailable(maps:get(id, State, undefined)),+ State.++-spec log_voice_members_table_unavailable(term()) -> ok.+log_voice_members_table_unavailable(GuildId) ->+ case erlang:put(?VOICE_MEMBERS_TABLE_WARNED, true) of+ true ->+ ok;+ _ ->+ logger:warning("guild_voice_members_table_unavailable: guild_id=~p", [GuildId])+ end.++-spec clear_voice_members_table_warning() -> ok.+clear_voice_members_table_warning() ->+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED),+ ok.+-spec maybe_report_crash(term(), term()) -> ok.maybe_report_crash(normal, _) ->ok;@@ -630,4 +741,349 @@ handle_non_voice_exit_other_linked_stops_guild_test() ->handle_non_voice_exit(OtherPid, boom, State)).+voice_guild_state_pins_projected_key_set_test() ->+ with_voice_projection_state(fun(Full) ->+ Projected = voice_guild_state(Full),+ ?assertEqual(+ lists:sort([+ id,+ <<"id">>,+ data,+ sessions,+ guild_pid,+ voice_states,+ pending_voice_connections,+ recently_disconnected_voice_states,+ e2ee_room_keys,+ virtual_channel_access,+ virtual_channel_access_pending,+ virtual_channel_access_preserve,+ virtual_channel_access_move_pending+ ]),+ lists:sort(maps:keys(Projected))+ ),+ ?assertEqual(+ lists:sort([+ <<"id">>,+ <<"guild">>,+ <<"roles">>,+ <<"role_index">>,+ role_perms_cache,+ <<"channels">>,+ <<"channel_index">>,+ overwrite_perms_cache,+ members_ets+ ]),+ lists:sort(maps:keys(maps:get(data, Projected)))+ )+ end).++voice_guild_state_drops_non_voice_payload_test() ->+ with_voice_projection_state(fun(Full) ->+ Projected = voice_guild_state(Full),+ ?assertNot(maps:is_key(member_list_subscriptions, Projected)),+ ?assertNot(maps:is_key(presence_subscriptions, Projected)),+ ?assertNot(maps:is_key(connected_user_ids, Projected)),+ Data = maps:get(data, Projected),+ ?assertNot(maps:is_key(<<"members">>, Data)),+ ?assertNot(maps:is_key(members_normalized, Data)),+ ?assertNot(maps:is_key(<<"member_role_index">>, Data)),+ ?assertNot(maps:is_key(<<"emojis">>, Data))+ end).++voice_guild_state_preserves_member_permissions_test() ->+ with_voice_projection_state(fun(Full) ->+ Projected = voice_guild_state(Full),+ Expected = guild_permissions:get_member_permissions(10, 500, Full),+ ?assert(Expected > 0),+ ?assert(permission_bits:has(Expected, constants:speak_permission())),+ ?assertEqual(Expected, guild_permissions:get_member_permissions(10, 500, Projected)),+ ?assertEqual(+ voice_utils:compute_voice_permissions(10, 500, Full),+ voice_utils:compute_voice_permissions(10, 500, Projected)+ ),+ ?assertEqual(+ guild_permissions:find_member_by_user_id(10, Full),+ guild_permissions:find_member_by_user_id(10, Projected)+ ),+ ?assertEqual(+ guild_permissions:can_view_channel(10, 500, undefined, Full),+ guild_permissions:can_view_channel(10, 500, undefined, Projected)+ )+ end).++voice_guild_state_absent_member_still_resolves_to_zero_test() ->+ with_voice_projection_state(fun(Full) ->+ Projected = voice_guild_state(Full),+ ?assertEqual(undefined, guild_permissions:find_member_by_user_id(11, Projected)),+ ?assertEqual(0, guild_permissions:get_member_permissions(11, 500, Projected)),+ ?assertEqual(+ guild_permissions:get_member_permissions(11, 500, Full),+ guild_permissions:get_member_permissions(11, 500, Projected)+ )+ end).++voice_guild_state_put_member_keeps_permissions_resolvable_test() ->+ with_voice_projection_state(fun(Full) ->+ Projected = voice_guild_state(Full),+ Data = maps:get(data, Projected),+ Member = guild_permissions:find_member_by_user_id(10, Projected),+ ?assert(is_map(Member)),+ Rebuilt = guild_data_index:put_member(Member#{<<"mute">> => true}, Data),+ RebuiltState = Projected#{data => Rebuilt},+ ?assertEqual(1, map_size(maps:get(<<"members">>, Rebuilt))),+ ?assertEqual(1, map_size(maps:get(members_normalized, Rebuilt))),+ ?assertEqual(maps:get(members_ets, Data), maps:get(members_ets, Rebuilt)),+ ?assertEqual(+ guild_permissions:get_member_permissions(10, 500, Full),+ guild_permissions:get_member_permissions(10, 500, RebuiltState)+ ),+ ?assertMatch(+ #{<<"mute">> := true},+ guild_permissions:find_member_by_user_id(10, RebuiltState)+ ),+ ?assertEqual(+ guild_permissions:can_view_channel(10, 500, undefined, Full),+ guild_permissions:can_view_channel(10, 500, undefined, RebuiltState)+ )+ end).++voice_guild_state_without_members_table_returns_full_state_test() ->+ with_voice_projection_state(fun(Full) ->+ Data = maps:get(data, Full),+ Broken = Full#{data => maps:remove(members_ets, Data)},+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED),+ ?assertEqual(Broken, voice_guild_state(Broken)),+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED)+ end).++voice_members_table_warning_rearms_after_recovery_test() ->+ with_voice_projection_state(fun(Full) ->+ Data = maps:get(data, Full),+ Broken = Full#{data => maps:remove(members_ets, Data)},+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED),+ _ = voice_guild_state(Broken),+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),+ _ = voice_guild_state(Broken),+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),+ _ = voice_guild_state(Full),+ ?assertEqual(undefined, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),+ _ = voice_guild_state(Broken),+ ?assertEqual(true, erlang:get(?VOICE_MEMBERS_TABLE_WARNED)),+ _ = erlang:erase(?VOICE_MEMBERS_TABLE_WARNED)+ end).++voice_guild_state_session_entries_keep_only_reader_keys_test() ->+ with_voice_projection_state(fun(Base) ->+ Full = Base#{sessions => voice_projection_sessions()},+ Projected = voice_guild_state(Full),+ Sessions = maps:get(sessions, Projected),+ Session = maps:get(<<"s-view">>, Sessions),+ ?assertEqual(+ lists:sort([session_id, user_id, pid, pending_connect, viewable_channels]),+ lists:sort(maps:keys(Session))+ ),+ ?assertNot(maps:is_key(active_guilds, Session)),+ ?assertNot(maps:is_key(user_roles, Session)),+ ?assertNot(maps:is_key(mref, Session)),+ ?assertNot(maps:is_key(bot, Session)),+ ?assertNot(maps:is_key(is_staff, Session))+ end).++voice_guild_state_session_projection_matches_reference_test() ->+ with_voice_projection_state(fun(Base) ->+ Full = Base#{sessions => voice_projection_sessions()},+ Reference = reference_voice_guild_state(Full),+ Projected = voice_guild_state(Full),+ ?assertEqual(maps:remove(sessions, Reference), maps:remove(sessions, Projected)),+ RefSessions = maps:get(sessions, Reference),+ NewSessions = maps:get(sessions, Projected),+ ?assertEqual(maps:keys(RefSessions), maps:keys(NewSessions)),+ ?assertEqual(+ maps:map(fun project_voice_session/2, RefSessions),+ NewSessions+ )+ end).++voice_guild_state_session_readers_see_identical_results_test() ->+ with_voice_projection_state(fun(Base) ->+ Full = Base#{sessions => voice_projection_sessions()},+ Reference = reference_voice_guild_state(Full),+ Projected = voice_guild_state(Full),+ Expected = voice_session_reader_result(500, Reference),+ ?assertEqual(Expected, voice_session_reader_result(500, Projected)),+ {SessionIds, Pids} = Expected,+ ?assertEqual([<<"s-cached">>, <<"s-perm">>, <<"s-view">>], lists:sort(SessionIds)),+ ?assertEqual([self(), self(), self()], Pids),+ Hidden = voice_session_reader_result(501, Reference),+ ?assertEqual(Hidden, voice_session_reader_result(501, Projected)),+ ?assertEqual({[<<"s-cached">>], [self()]}, Hidden)+ end).++voice_guild_state_session_projection_boundaries_test() ->+ with_voice_projection_state(fun(Base) ->+ Bare = #{user_id => 10, pid => self()},+ Full = Base#{sessions => #{<<"s-bare">> => Bare, <<"s-odd">> => not_a_map}},+ Sessions = maps:get(sessions, voice_guild_state(Full)),+ ?assertEqual(Bare, maps:get(<<"s-bare">>, Sessions)),+ ?assertEqual(not_a_map, maps:get(<<"s-odd">>, Sessions)),+ NoSessions = maps:remove(sessions, Base),+ ?assertNot(maps:is_key(sessions, voice_guild_state(NoSessions))),+ BadSessions = Base#{sessions => not_a_map},+ ?assertEqual(not_a_map, maps:get(sessions, voice_guild_state(BadSessions)))+ end).++voice_guild_state_preserves_session_fold_order_at_scale_test() ->+ with_voice_projection_state(fun voice_session_fold_order_scenario/1).++%% 200 entries forces the hashmap representation, where iteration order is driven by key+%% hashes. maps:map/2 rewrites values only, so the key set and therefore the order is identical.+voice_session_fold_order_scenario(Base) ->+ Sessions = maps:from_list(lists:map(fun voice_scale_session/1, lists:seq(1, 200))),+ Full = Base#{sessions => Sessions},+ Projected = voice_guild_state(Full),+ ?assertEqual(maps:keys(Sessions), maps:keys(maps:get(sessions, Projected))),+ ?assertEqual(+ voice_session_reader_result(500, reference_voice_guild_state(Full)),+ voice_session_reader_result(500, Projected)+ ).++voice_scale_session(N) ->+ SessionId = integer_to_binary(N),+ {SessionId, voice_projection_session(SessionId, 10, #{500 => true}, false)}.++voice_session_drop_keys_never_drops_a_reader_key_test() ->+ Dropped = voice_session_drop_keys(),+ Readers = [pending_connect, user_id, viewable_channels, pid],+ ?assertEqual([], [K || K <- Readers, lists:member(K, Dropped)]).++%% Reproduces the projection exactly as it was before session entries were narrowed, so the+%% tests above compare the new path against a live oracle rather than a hand-written literal.+reference_voice_guild_state(State) ->+ Data = maps:get(data, State, #{}),+ Projected = maps:with(voice_guild_state_keys(), State),+ Projected#{data => maps:with(voice_guild_data_keys(), Data)}.++%% The reader tuple is everything the voice path can observe about the sessions map: which+%% sessions match, in which fold order, and which pids the broadcast dispatches to.+voice_session_reader_result(ChannelId, State) ->+ Sessions = maps:get(sessions, State, #{}),+ Pairs = guild_sessions:filter_sessions_for_channel(Sessions, ChannelId, undefined, State),+ {[Sid || {Sid, _S} <- Pairs], [maps:get(pid, S) || {_Sid, S} <- Pairs]}.++voice_projection_sessions() ->+ #{+ <<"s-view">> => voice_projection_session(<<"s-view">>, 10, #{500 => true}, false),+ <<"s-perm">> => voice_projection_session(<<"s-perm">>, 10, #{}, false),+ <<"s-pending">> => voice_projection_session(<<"s-pending">>, 10, #{500 => true}, true),+ <<"s-stranger">> => voice_projection_session(<<"s-stranger">>, 11, #{}, false),+ <<"s-cached">> => voice_projection_session(<<"s-cached">>, 10, #{501 => true}, false)+ }.++voice_projection_session(SessionId, UserId, ViewableChannels, Pending) ->+ #{+ session_id => SessionId,+ user_id => UserId,+ pid => self(),+ mref => make_ref(),+ active_guilds => sets:from_list([42, 43, 44]),+ user_roles => [77],+ bot => false,+ is_staff => false,+ pending_connect => Pending,+ viewable_channels => ViewableChannels+ }.++voice_members_table_warning_key_is_module_scoped_test() ->+ ?assertEqual({?MODULE, voice_members_table_unavailable}, ?VOICE_MEMBERS_TABLE_WARNED),+ ?assertNot(is_atom(?VOICE_MEMBERS_TABLE_WARNED)).++with_voice_projection_state(Fun) ->+ Tab = ets:new(guild_members_data, [set, public, {read_concurrency, true}]),+ try+ Fun(voice_projection_state(Tab))+ after+ ets:delete(Tab)+ end.++voice_projection_state(Tab) ->+ Data = voice_projection_data(),+ maps:foreach(+ fun(UserId, Member) -> ets:insert(Tab, {UserId, Member}) end,+ maps:get(members_normalized, Data)+ ),+ #{+ id => 42,+ <<"id">> => <<"42">>,+ data => Data#{members_ets => Tab, <<"emojis">> => [], channels_stale => true},+ sessions => #{},+ guild_pid => self(),+ voice_states => #{},+ pending_voice_connections => #{},+ recently_disconnected_voice_states => #{},+ e2ee_room_keys => #{},+ virtual_channel_access => #{},+ virtual_channel_access_pending => #{},+ virtual_channel_access_preserve => #{},+ virtual_channel_access_move_pending => #{},+ presence_subscriptions => #{},+ member_list_subscriptions => #{},+ connected_user_ids => sets:new(),+ counts => #{member_count => 1}+ }.++voice_projection_data() ->+ Everyone = #{+ <<"id">> => <<"42">>,+ <<"name">> => <<"@everyone">>,+ <<"permissions">> => integer_to_binary(constants:view_channel_permission())+ },+ Speaker = #{+ <<"id">> => <<"77">>,+ <<"name">> => <<"speaker">>,+ <<"permissions">> => integer_to_binary(+ constants:connect_permission() bor constants:speak_permission()+ )+ },+ Channel = #{+ <<"id">> => <<"500">>,+ <<"type">> => 2,+ <<"name">> => <<"General">>,+ <<"permission_overwrites">> => [+ #{+ <<"id">> => <<"77">>,+ <<"type">> => 0,+ <<"allow">> => integer_to_binary(constants:stream_permission()),+ <<"deny">> => <<"0">>+ }+ ]+ },+ HiddenChannel = #{+ <<"id">> => <<"501">>,+ <<"type">> => 2,+ <<"name">> => <<"Hidden">>,+ <<"permission_overwrites">> => [+ #{+ <<"id">> => <<"42">>,+ <<"type">> => 0,+ <<"allow">> => <<"0">>,+ <<"deny">> => integer_to_binary(constants:view_channel_permission())+ }+ ]+ },+ Member = #{+ <<"user">> => #{<<"id">> => <<"10">>, <<"username">> => <<"speaker">>},+ <<"roles">> => [<<"77">>],+ <<"mute">> => false,+ <<"deaf">> => false+ },+ guild_data_index:normalize_data(#{+ <<"id">> => <<"42">>,+ <<"guild">> => #{<<"id">> => <<"42">>, <<"owner_id">> => <<"9999">>},+ <<"roles">> => [Everyone, Speaker],+ <<"channels">> => [Channel, HiddenChannel],+ <<"members">> => [Member]+ }).+-endif.diff --git a/fluxer_gateway/src/guild/guild_connect_async.erl b/fluxer_gateway/src/guild/guild_connect_async.erlindex a0d81a736..b75bbe960 100644--- a/fluxer_gateway/src/guild/guild_connect_async.erl+++ b/fluxer_gateway/src/guild/guild_connect_async.erl@@ -644,3 +644,61 @@ send_result(GuildId, Attempt, Result0, SessionPid) ->end,SessionPid ! {guild_connect_result, GuildId, Attempt, Reply},ok.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++pending_connect_state(Sessions) ->+ #{+ id => 42,+ sessions => Sessions,+ session_connect_pending => #{},+ session_connect_queue => queue:new(),+ session_connect_inflight => ?SESSION_CONNECT_MAX_WORKERS,+ session_connect_max_queue => ?SESSION_CONNECT_DEFAULT_MAX_QUEUE+ }.++pending_connect_request(SessionId, UserId) ->+ #{+ session_id => SessionId,+ user_id => UserId,+ session_pid => self(),+ bot => false,+ is_staff => false,+ active_guilds => sets:new()+ }.++enqueued_session_entry(SessionId, UserId, State0) ->+ State1 = enqueue_session_connect_async(+ 42, 1, pending_connect_request(SessionId, UserId), #{}, State0+ ),+ maps:get(SessionId, maps:get(sessions, State1)).++enqueue_marks_existing_session_pending_connect_test() ->+ SessionId = <<"s-existing">>,+ UserId = 21,+ MRef = make_ref(),+ Existing = #{+ session_id => SessionId,+ user_id => UserId,+ pid => self(),+ mref => MRef,+ pending_connect => false,+ active_guilds => sets:new()+ },+ Entry = enqueued_session_entry(+ SessionId, UserId, pending_connect_state(#{SessionId => Existing})+ ),+ ?assertEqual(true, maps:get(pending_connect, Entry)),+ ?assertEqual(MRef, maps:get(mref, Entry)),+ ?assertEqual(UserId, maps:get(user_id, Entry)).++enqueue_creates_pending_session_entry_test() ->+ SessionId = <<"s-fresh">>,+ UserId = 22,+ Entry = enqueued_session_entry(SessionId, UserId, pending_connect_state(#{})),+ demonitor(maps:get(mref, Entry), [flush]),+ ?assertEqual(true, maps:get(pending_connect, Entry)),+ ?assertEqual(UserId, maps:get(user_id, Entry)).++-endif.diff --git a/fluxer_gateway/src/guild/guild_data.erl b/fluxer_gateway/src/guild/guild_data.erlindex 6b2b69ccc..0854cc23e 100644--- a/fluxer_gateway/src/guild/guild_data.erl+++ b/fluxer_gateway/src/guild/guild_data.erl@@ -25,6 +25,7 @@-define(CONNECT_SNAPSHOT_HEAVY_MEMBER_KEYS, [<<"members">>, members_normalized, <<"member_role_index">>, members_sorted_ids]).+-define(CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, [active_guilds, user_roles, viewable_channels]).-export_type([guild_state/0, guild_reply/1, user_id/0]).@@ -167,15 +168,56 @@ build_connect_snapshot(Item, State) ->],State),- maybe_trim_connect_snapshot(Item, Base, State).+ project_snapshot_sessions(maybe_trim_connect_snapshot(Item, Base, State)).++-spec project_snapshot_sessions(map()) -> map().+project_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) ->+ Projected = maps:map(+ fun(_SessionId, SessionData) -> project_snapshot_session(SessionData) end,+ Sessions+ ),+ Snapshot#{sessions => Projected};+project_snapshot_sessions(Snapshot) ->+ Snapshot.++-spec project_snapshot_session(term()) -> term().+project_snapshot_session(SessionData) when is_map(SessionData) ->+ maps:without(?CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS, SessionData);+project_snapshot_session(SessionData) ->+ SessionData.-spec maybe_trim_connect_snapshot(map(), map(), guild_state()) -> map().maybe_trim_connect_snapshot(Item, Base, State) ->- case has_members_ets(State) of+ case should_trim_connect_snapshot(State) oftrue -> trim_connect_snapshot(Item, Base);false -> Baseend.+-spec should_trim_connect_snapshot(guild_state()) -> boolean().+should_trim_connect_snapshot(State) ->+ case connect_snapshot_trim_member_threshold() of+ undefined ->+ false;+ Threshold ->+ member_count_at_least(Threshold, State) andalso has_members_ets(State)+ end.++-spec member_count_at_least(pos_integer(), guild_state()) -> boolean().+member_count_at_least(Threshold, State) ->+ case maps:get(member_count, State, undefined) of+ Count when is_integer(Count) -> Count >= Threshold;+ _ -> false+ end.++-spec connect_snapshot_trim_member_threshold() -> pos_integer() | undefined.+connect_snapshot_trim_member_threshold() ->+ case+ application:get_env(fluxer_gateway, connect_snapshot_trim_member_threshold, undefined)+ of+ N when is_integer(N), N > 0 -> N;+ _ -> undefined+ end.+-spec has_members_ets(guild_state()) -> boolean().has_members_ets(#{data := #{members_ets := Tab}}) -> is_reference(Tab);has_members_ets(_) -> false.@@ -448,3 +490,107 @@ guild_id_wire_value(undefined) ->null;guild_id_wire_value(GuildId) ->integer_to_binary(GuildId).++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++projection_snapshot() ->+ #{+ sessions => #{+ <<"s1">> => #{+ session_id => <<"s1">>,+ user_id => 7,+ pid => self(),+ bot => false,+ is_staff => true,+ pending_connect => false,+ active_guilds => sets:new(),+ user_roles => [1, 2],+ viewable_channels => #{10 => true}+ }+ }+ }.++projection_entry(Snapshot) ->+ maps:get(<<"s1">>, maps:get(sessions, Snapshot)).++project_snapshot_sessions_drops_only_heavy_keys_test() ->+ Entry = projection_entry(project_snapshot_sessions(projection_snapshot())),+ ?assertEqual(7, maps:get(user_id, Entry)),+ ?assertEqual(true, maps:get(is_staff, Entry)),+ ?assertEqual(false, maps:get(pending_connect, Entry)),+ ?assertEqual(<<"s1">>, maps:get(session_id, Entry)),+ ?assertNot(maps:is_key(active_guilds, Entry)),+ ?assertNot(maps:is_key(user_roles, Entry)),+ ?assertNot(maps:is_key(viewable_channels, Entry)).++projection_keeps_session_staff_lookup_test() ->+ Projected = project_snapshot_sessions(projection_snapshot()),+ ?assertEqual(true, guild_availability_check:is_user_staff(7, Projected)).++projection_leaves_snapshot_data_untouched_test() ->+ Data = #{<<"members">> => #{7 => #{}}, members_ets => make_ref()},+ Snapshot = maps:put(data, Data, projection_snapshot()),+ ?assertEqual(Data, maps:get(data, project_snapshot_sessions(Snapshot))).++projection_keeps_non_map_session_entries_test() ->+ Snapshot = #{sessions => #{<<"broken">> => not_a_map}},+ ?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)).++projection_keeps_snapshot_without_sessions_test() ->+ Snapshot = #{id => 42, member_count => 3},+ ?assertEqual(Snapshot, project_snapshot_sessions(Snapshot)).++trim_state(MemberCount) ->+ #{id => 42, member_count => MemberCount, data => #{members_ets => make_ref()}}.++with_trim_threshold(Threshold, Fun) ->+ application:set_env(fluxer_gateway, connect_snapshot_trim_member_threshold, Threshold),+ try+ Fun()+ after+ application:unset_env(fluxer_gateway, connect_snapshot_trim_member_threshold)+ end.++trim_is_off_without_threshold_test() ->+ application:unset_env(fluxer_gateway, connect_snapshot_trim_member_threshold),+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435))).++trim_needs_member_count_at_threshold_test() ->+ with_trim_threshold(20000, fun() ->+ ?assertEqual(true, should_trim_connect_snapshot(trim_state(49435))),+ ?assertEqual(true, should_trim_connect_snapshot(trim_state(20000))),+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(19999))),+ NoCount = maps:remove(member_count, trim_state(1)),+ ?assertEqual(false, should_trim_connect_snapshot(NoCount))+ end).++trim_needs_members_ets_test() ->+ with_trim_threshold(1, fun() ->+ ?assertEqual(false, should_trim_connect_snapshot(#{member_count => 10, data => #{}})),+ ?assertEqual(+ false,+ should_trim_connect_snapshot(#{member_count => 10, data => #{members_ets => 7}})+ ),+ ?assertEqual(false, should_trim_connect_snapshot(#{member_count => 10}))+ end).++trim_ignores_invalid_threshold_test() ->+ with_trim_threshold(0, fun() ->+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435)))+ end),+ with_trim_threshold(not_an_integer, fun() ->+ ?assertEqual(false, should_trim_connect_snapshot(trim_state(49435)))+ end).++trim_and_projection_target_disjoint_keys_test() ->+ ?assertEqual(+ [],+ [+ Key+ || Key <- ?CONNECT_SNAPSHOT_HEAVY_SESSION_KEYS,+ lists:member(Key, ?CONNECT_SNAPSHOT_HEAVY_MEMBER_KEYS)+ ]+ ).++-endif.diff --git a/fluxer_gateway/src/guild/guild_data_index_members.erl b/fluxer_gateway/src/guild/guild_data_index_members.erlindex f333c1ad1..e43d02739 100644--- a/fluxer_gateway/src/guild/guild_data_index_members.erl+++ b/fluxer_gateway/src/guild/guild_data_index_members.erl@@ -9,6 +9,7 @@member_list/1,member_count/1,member_ids/1,+ sorted_member_ids/2,member_role_index/1,get_member/2,get_member_ets/2,@@ -84,6 +85,8 @@ member_list(Data) ->SortedIds = sorted_member_ids(Data, MemberMap),[Member || UserId <- SortedIds, {ok, Member} <- [maps:find(UserId, MemberMap)]].+%% Every member mutation drops members_sorted_ids, so a cache that is present always+%% equals lists:sort(maps:keys(member_map(Data))) and callers may page it as-is.-spec sorted_member_ids(term(), #{user_id() => member()}) -> [user_id()].sorted_member_ids(Data, MemberMap) when is_map(Data) ->case maps:find(members_sorted_ids, Data) of@@ -369,3 +372,50 @@ members_ets_table(#{members_ets := Tab}) ->Tab;members_ets_table(_) ->undefined.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++sorted_member_ids_equals_sorted_keys_test() ->+ lists:foreach(fun assert_sorted_ids_match_keys/1, sorted_ids_fixtures()).++assert_sorted_ids_match_keys(Data) ->+ MemberMap = member_map(Data),+ ?assertEqual(lists:sort(maps:keys(MemberMap)), sorted_member_ids(Data, MemberMap)).++sorted_member_ids_cache_dropped_by_mutations_test() ->+ Base = put_member_map(fixture_member_map([12, 3, 7]), #{}),+ ?assertEqual([3, 7, 12], maps:get(members_sorted_ids, Base)),+ ?assertNot(maps:is_key(members_sorted_ids, put_member(fixture_member(5), Base))),+ ?assertNot(maps:is_key(members_sorted_ids, remove_member(3, Base))).++sorted_ids_fixtures() ->+ Base = put_member_map(fixture_member_map([12, 3, 7]), #{}),+ Large = put_member_map(fixture_member_map(fixture_user_ids(40)), #{}),+ [+ #{},+ #{<<"members">> => #{}},+ #{<<"members">> => fixture_member_list([9, 2, 5])},+ #{<<"members">> => #{<<"9">> => fixture_member(9), <<"2">> => fixture_member(2)}},+ #{<<"members">> => fixture_member_list(fixture_user_ids(40))},+ Base,+ Large,+ put_member(fixture_member(5), Base),+ put_member(fixture_member(500), Large),+ remove_member(3, Base),+ put_member_list(fixture_member_list([4, 1]), Base)+ ].++fixture_user_ids(Count) ->+ lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)).++fixture_member_map(UserIds) ->+ maps:from_list([{UserId, fixture_member(UserId)} || UserId <- UserIds]).++fixture_member_list(UserIds) ->+ [fixture_member(UserId) || UserId <- UserIds].++fixture_member(UserId) ->+ #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}.++-endif.diff --git a/fluxer_gateway/src/guild/guild_data_members.erl b/fluxer_gateway/src/guild/guild_data_members.erlindex e91c51331..35ff5caba 100644--- a/fluxer_gateway/src/guild/guild_data_members.erl+++ b/fluxer_gateway/src/guild/guild_data_members.erl@@ -56,10 +56,15 @@ has_member(#{user_id := UserId}, State) ->-spec list_guild_members(map(), guild_state()) -> {reply, map(), guild_state()}.list_guild_members(#{limit := Limit, offset := Offset}, State) ->Data = guild_data_index:ensure_data_map(State),- AllMembers = guild_data_index:member_list(Data),- TotalCount = length(AllMembers),- PaginatedMembers = paginate_members(AllMembers, Limit, Offset),- {reply, #{members => PaginatedMembers, total => TotalCount}, State}.+ MemberMap = guild_data_index:member_map(Data),+ SortedIds = guild_data_index_members:sorted_member_ids(Data, MemberMap),+ PageIds = paginate_members(SortedIds, Limit, Offset),+ PaginatedMembers = lookup_members(PageIds, MemberMap),+ {reply, #{members => PaginatedMembers, total => map_size(MemberMap)}, State}.++-spec lookup_members([term()], map()) -> [guild_member()].+lookup_members(UserIds, MemberMap) ->+ [Member || UserId <- UserIds, {ok, Member} <- [maps:find(UserId, MemberMap)]].-spec search_guild_members(map(), guild_state()) -> {reply, map(), guild_state()}.search_guild_members(#{query := Query, limit := Limit}, State) when@@ -113,23 +118,69 @@ collect_query_match(Member, NormalizedQuery, Remaining, Acc, NextIterator) ->find_member_by_user_id(UserId, State) ->guild_permissions:find_member_by_user_id(UserId, State).--spec paginate_members([guild_member()], non_neg_integer(), non_neg_integer()) ->- [guild_member()].-paginate_members(Members, Limit, Offset) ->- take_page(drop_members(Members, Offset), Limit, []).+-spec paginate_members([term()], non_neg_integer(), non_neg_integer()) -> [term()].+paginate_members(Items, Limit, Offset) ->+ take_page(drop_items(Items, Offset), Limit, []).--spec drop_members([guild_member()], non_neg_integer()) -> [guild_member()].-drop_members(Members, 0) ->- Members;-drop_members([], _Offset) ->+-spec drop_items([term()], non_neg_integer()) -> [term()].+drop_items(Items, 0) ->+ Items;+drop_items([], _Offset) ->[];-drop_members([_ | Rest], Offset) ->- drop_members(Rest, Offset - 1).+drop_items([_ | Rest], Offset) ->+ drop_items(Rest, Offset - 1).--spec take_page([guild_member()], non_neg_integer(), [guild_member()]) -> [guild_member()].+-spec take_page([term()], non_neg_integer(), [term()]) -> [term()].take_page(_Remaining, 0, Acc) ->lists:reverse(Acc);take_page([], _Limit, Acc) ->lists:reverse(Acc);-take_page([Member | Rest], Limit, Acc) ->- take_page(Rest, Limit - 1, [Member | Acc]).+take_page([Item | Rest], Limit, Acc) ->+ take_page(Rest, Limit - 1, [Item | Acc]).++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++list_guild_members_matches_full_list_paging_test() ->+ lists:foreach(fun assert_list_page_matches/1, list_paging_cases()).++assert_list_page_matches({State, Limit, Offset}) ->+ {reply, Reply, _} = list_guild_members(#{limit => Limit, offset => Offset}, State),+ ?assertEqual(legacy_list_guild_members(Limit, Offset, State), Reply).++legacy_list_guild_members(Limit, Offset, State) ->+ Data = guild_data_index:ensure_data_map(State),+ AllMembers = guild_data_index:member_list(Data),+ #{members => paginate_members(AllMembers, Limit, Offset), total => length(AllMembers)}.++list_paging_cases() ->+ [+ {State, Limit, Offset}+ || State <- list_paging_states(),+ Limit <- [0, 1, 2, 5, 99],+ Offset <- [0, 1, 3, 5, 6, 20, 39, 40, 41, 99]+ ].++list_paging_states() ->+ Members = paging_members(paging_user_ids(40)),+ Cached = guild_data_index:normalize_data(#{<<"members">> => Members}),+ [+ #{data => Cached},+ #{data => guild_data_index:put_member(paging_member(500), Cached)},+ #{data => guild_data_index:remove_member(7, Cached)},+ #{data => #{<<"members">> => Members}},+ #{data => guild_data_index:normalize_data(#{<<"members">> => []})},+ #{data => #{<<"members">> => #{}}},+ #{}+ ].++paging_members(UserIds) ->+ [paging_member(UserId) || UserId <- UserIds].++paging_user_ids(Count) ->+ lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)).++paging_member(UserId) ->+ #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}.++-endif.diff --git a/fluxer_gateway/src/guild/guild_data_wire.erl b/fluxer_gateway/src/guild/guild_data_wire.erlindex dd7aee96b..cb972d164 100644--- a/fluxer_gateway/src/guild/guild_data_wire.erl+++ b/fluxer_gateway/src/guild/guild_data_wire.erl@@ -5,219 +5,523 @@-export([payload/1]).--type field_kind() :: snowflake | permission | snowflake_list | id_string | generic.--type path() :: [binary()].+-type value_kind() :: scalar | scalar_list | maybe_scalar_list | id | restrict | generic.+-type key_kind() :: drop | value_kind().-spec payload(term()) -> term().payload(Value) ->- payload(Value, []).+ fast_payload(Value, false).--spec payload(term(), path()) -> term().-payload(Value, Path) when is_map(Value) ->+-spec fast_payload(term(), boolean()) -> term().+fast_payload(Value, Restricted) when is_map(Value) ->maps:fold(fun(Key, FieldValue, Acc) ->- payload_map_field(Path, Key, FieldValue, Acc)+ fast_map_field(Restricted, Key, FieldValue, Acc)end,#{},Value);-payload(Value, Path) when is_list(Value) ->- [payload(Item, Path) || Item <- Value];-payload(Value, _Path) ->+fast_payload(Value, Restricted) when is_list(Value) ->+ [fast_payload(Item, Restricted) || Item <- Value];+fast_payload(Value, _Restricted) ->Value.--spec payload_map_field(path(), term(), term(), map()) -> map().-payload_map_field(Path, Key, FieldValue, Acc) ->- KeyBinary = key_binary(Key),- case keep_payload_field(KeyBinary) of- true -> Acc#{payload_key(Key) => payload_field(Path, Key, KeyBinary, FieldValue)};- false -> Acc+-spec fast_map_field(boolean(), term(), term(), map()) -> map().+fast_map_field(Restricted, Key, Value, Acc) when is_binary(Key) ->+ fast_put(binary_key_kind(Key), Restricted, Key, Value, Acc);+fast_map_field(Restricted, Key, Value, Acc) when is_atom(Key) ->+ Name = atom_to_binary(Key, utf8),+ fast_put(named_or_suffix_kind(Name), Restricted, Name, Value, Acc);+fast_map_field(Restricted, Key, Value, Acc) when is_integer(Key), Key > 0 ->+ fast_put(maybe_scalar_list, Restricted, integer_to_binary(Key), Value, Acc);+fast_map_field(Restricted, Key, Value, Acc) when is_integer(Key) ->+ fast_put(generic, Restricted, integer_to_binary(Key), Value, Acc);+fast_map_field(Restricted, Key, Value, Acc) ->+ fast_put(generic, Restricted, Key, Value, Acc).++-spec fast_put(key_kind(), boolean(), term(), term(), map()) -> map().+fast_put(drop, _Restricted, _Key, _Value, Acc) ->+ Acc;+fast_put(Kind, Restricted, Key, Value, Acc) ->+ Acc#{Key => fast_field(Kind, Restricted, Value)}.++-spec fast_field(key_kind(), boolean(), term()) -> term().+fast_field(generic, Restricted, Value) ->+ fast_payload(Value, Restricted);+fast_field(scalar, Restricted, Value) ->+ fast_scalar(Value, Restricted);+fast_field(scalar_list, Restricted, Value) ->+ fast_scalar_list(Value, Restricted);+fast_field(maybe_scalar_list, Restricted, Value) ->+ fast_maybe_scalar_list(Value, Restricted);+fast_field(id, false, Value) ->+ fast_scalar(Value, false);+fast_field(id, true, Value) ->+ fast_payload(Value, true);+fast_field(restrict, _Restricted, Value) ->+ fast_payload(Value, true).++-spec fast_scalar(term(), boolean()) -> term().+fast_scalar(Value, _Restricted) when is_integer(Value) ->+ integer_to_binary(Value);+fast_scalar(Value, Restricted) ->+ fast_payload(Value, Restricted).++-spec fast_scalar_list(term(), boolean()) -> term().+fast_scalar_list(Values, Restricted) when is_list(Values) ->+ [fast_scalar(Item, Restricted) || Item <- Values];+fast_scalar_list(Value, Restricted) ->+ fast_payload(Value, Restricted).++-spec fast_maybe_scalar_list(term(), boolean()) -> term().+fast_maybe_scalar_list(Value, Restricted) ->+ case is_fast_scalar_snowflake_list(Value) of+ true -> fast_scalar_list(Value, Restricted);+ false -> fast_payload(Value, Restricted)end.--spec keep_payload_field(binary() | undefined) -> boolean().-keep_payload_field(<<"recipient_ids">>) -> false;-keep_payload_field(<<"role_index">>) -> false;-keep_payload_field(<<"channel_index">>) -> false;-keep_payload_field(<<"member_role_index">>) -> false;-keep_payload_field(<<"role_perms_cache">>) -> false;-keep_payload_field(<<"overwrite_perms_cache">>) -> false;-keep_payload_field(_KeyBinary) -> true.---spec payload_key(term()) -> term().-payload_key(Key) when is_integer(Key) ->+-spec is_fast_scalar_snowflake_list(term()) -> boolean().+is_fast_scalar_snowflake_list([]) ->+ true;+is_fast_scalar_snowflake_list(Values) when is_list(Values) ->+ lists:all(fun is_fast_snowflake_scalar/1, Values);+is_fast_scalar_snowflake_list(_) ->+ false.++-spec is_fast_snowflake_scalar(term()) -> boolean().+is_fast_snowflake_scalar(Value) when is_integer(Value), Value > 0 ->+ true;+is_fast_snowflake_scalar(<<First, _/binary>> = Value) when First >= $1, First =< $9 ->+ snowflake_id:is_valid(Value);+is_fast_snowflake_scalar(_) ->+ false.++-spec binary_key_kind(binary()) -> key_kind().+binary_key_kind(<<First, _/binary>> = Key) when First >= $1, First =< $9 ->+ numeric_binary_key_kind(Key);+binary_key_kind(Key) ->+ named_or_suffix_kind(Key).++-spec named_or_suffix_kind(binary()) -> key_kind().+named_or_suffix_kind(Key) ->+ case named_key_kind(Key) of+ unknown -> suffix_key_kind(Key);+ Kind -> Kind+ end.++-spec numeric_binary_key_kind(binary()) -> value_kind().+numeric_binary_key_kind(Key) ->+ case snowflake_id:is_valid(Key) of+ true -> maybe_scalar_list;+ false -> suffix_key_kind(Key)+ end.++-spec suffix_key_kind(binary()) -> value_kind().+suffix_key_kind(Key) ->+ case has_suffix(Key, <<"_ids">>) of+ true -> scalar_list;+ false -> trailing_id_key_kind(Key)+ end.++-spec trailing_id_key_kind(binary()) -> value_kind().+trailing_id_key_kind(Key) ->+ case has_suffix(Key, <<"_id">>) of+ true -> scalar;+ false -> generic+ end.++-spec named_key_kind(binary()) -> key_kind() | unknown.+named_key_kind(<<"id">>) -> id;+named_key_kind(<<"permissions">>) -> scalar;+named_key_kind(<<"allow">>) -> scalar;+named_key_kind(<<"deny">>) -> scalar;+named_key_kind(<<"session_id">>) -> scalar;+named_key_kind(<<"connection_id">>) -> scalar;+named_key_kind(<<"subscription_id">>) -> scalar;+named_key_kind(<<"app_id">>) -> scalar;+named_key_kind(<<"device_id">>) -> scalar;+named_key_kind(<<"region_id">>) -> scalar;+named_key_kind(<<"server_id">>) -> scalar;+named_key_kind(<<"target_id">>) -> scalar;+named_key_kind(<<"mention_roles">>) -> scalar_list;+named_key_kind(<<"participants">>) -> scalar_list;+named_key_kind(<<"ringing">>) -> scalar_list;+named_key_kind(<<"nsfw_emojis">>) -> scalar_list;+named_key_kind(<<"pinned_dms">>) -> scalar_list;+named_key_kind(<<"restricted_guilds">>) -> scalar_list;+named_key_kind(<<"bot_restricted_guilds">>) -> scalar_list;+named_key_kind(<<"roles">>) -> maybe_scalar_list;+named_key_kind(<<"mentions">>) -> maybe_scalar_list;+named_key_kind(<<"recipients">>) -> maybe_scalar_list;+named_key_kind(<<"guild_folders">>) -> restrict;+named_key_kind(<<"rtc_regions">>) -> restrict;+named_key_kind(<<"recipient_ids">>) -> drop;+named_key_kind(<<"role_index">>) -> drop;+named_key_kind(<<"channel_index">>) -> drop;+named_key_kind(<<"member_role_index">>) -> drop;+named_key_kind(<<"role_perms_cache">>) -> drop;+named_key_kind(<<"overwrite_perms_cache">>) -> drop;+named_key_kind(_) -> unknown.++-spec has_suffix(binary(), binary()) -> boolean().+has_suffix(Value, Suffix) ->+ Size = byte_size(Value),+ SuffixSize = byte_size(Suffix),+ case Size >= SuffixSize of+ true ->+ has_suffix(Value, Suffix, Size - SuffixSize, SuffixSize);+ false ->+ false+ end.++-spec has_suffix(binary(), binary(), non_neg_integer(), non_neg_integer()) -> boolean().+has_suffix(Value, Suffix, PrefixSize, SuffixSize) ->+ case Value of+ <<_:PrefixSize/binary, Suffix:SuffixSize/binary>> -> true;+ _ -> false+ end.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++-type field_kind() :: snowflake | permission | snowflake_list | id_string | generic.+-type path() :: [binary()].++fast_payload_matches_original_test() ->+ lists:foreach(fun assert_fast_matches_original/1, corpus()).++fast_payload_matches_original_for_mixed_keys_test() ->+ lists:foreach(fun assert_fast_matches_original/1, mixed_key_maps()).++fast_payload_matches_original_under_restricted_paths_test() ->+ lists:foreach(fun assert_fast_matches_original/1, restricted_corpus()).++fast_payload_keeps_restricted_ids_opaque_test() ->+ Data = #{<<"rtc_regions">> => [#{<<"id">> => <<"us-east">>, <<"guild_id">> => 7}]},+ ?assertEqual(+ #{<<"rtc_regions">> => [#{<<"id">> => <<"us-east">>, <<"guild_id">> => <<"7">>}]},+ fast_payload(Data, false)+ ).++fast_payload_drops_internal_keys_test() ->+ Data = #{<<"id">> => 1, <<"role_index">> => #{}, role_perms_cache => #{}},+ ?assertEqual(#{<<"id">> => <<"1">>}, fast_payload(Data, false)).++pre_encoded_payload_passes_through_unchanged_test() ->+ Data = {pre_encoded, <<"{}">>},+ ?assertEqual(Data, payload(Data)).++assert_fast_matches_original(Value) ->+ ?assertEqual({Value, reference_payload(Value, [])}, {Value, fast_payload(Value, false)}).++corpus() ->+ [#{Key => Value} || Key <- sample_keys(), Value <- sample_values()].++restricted_corpus() ->+ lists:flatmap(fun restricted_variants/1, corpus()).++restricted_variants(Map) ->+ [+ #{<<"guild_folders">> => [Map]},+ #{<<"rtc_regions">> => Map},+ #{guild_folders => #{<<"nested">> => Map}},+ #{<<"data">> => [#{<<"rtc_regions">> => [Map]}]},+ #{<<"data">> => #{<<"more">> => Map}}+ ].++mixed_key_maps() ->+ [+ #{<<"id">> => 1, id => 2},+ #{<<"100">> => [1], 100 => [2]},+ #{<<"permissions">> => [1], permissions => 2},+ #{<<"recipient_ids">> => [1], recipient_ids => [2], <<"id">> => 3},+ #{1 => [2], -1 => [3], 0 => [4]},+ #{<<"guild_folders">> => [#{<<"id">> => 1, <<"guild_ids">> => [2], id => 3}]},+ #{<<"roles">> => [1, 2], <<"mentions">> => [#{<<"id">> => 3}], <<"deny">> => 4}+ ].++sample_keys() ->+ [+ <<"id">>,+ <<"guild_id">>,+ <<"session_id">>,+ <<"connection_id">>,+ <<"subscription_id">>,+ <<"app_id">>,+ <<"device_id">>,+ <<"region_id">>,+ <<"server_id">>,+ <<"target_id">>,+ <<"permissions">>,+ <<"allow">>,+ <<"deny">>,+ <<"roles">>,+ <<"mentions">>,+ <<"recipients">>,+ <<"ringing">>,+ <<"mention_roles">>,+ <<"participants">>,+ <<"nsfw_emojis">>,+ <<"pinned_dms">>,+ <<"restricted_guilds">>,+ <<"bot_restricted_guilds">>,+ <<"recipient_ids">>,+ <<"role_index">>,+ <<"channel_index">>,+ <<"member_role_index">>,+ <<"role_perms_cache">>,+ <<"overwrite_perms_cache">>,+ <<"guild_folders">>,+ <<"rtc_regions">>,+ <<"guild_ids">>,+ <<"username">>,+ <<"_id">>,+ <<"_ids">>,+ <<"id_">>,+ <<"9_ids">>,+ <<"100">>,+ <<"0100">>,+ <<"1x">>,+ <<>>,+ id,+ guild_id,+ session_id,+ roles,+ permissions,+ role_perms_cache,+ guild_folders,+ '100',+ 100,+ 0,+ -1,+ 1.5,+ {tuple, key}+ ].++sample_values() ->+ [+ 42,+ -7,+ 0,+ <<"text">>,+ <<"42">>,+ null,+ true,+ undefined,+ [],+ [1, 2],+ [<<"3">>, 4],+ [<<"abc">>],+ [[1, 2]],+ [#{<<"id">> => 5}],+ #{<<"id">> => 6, <<"session_id">> => <<"s">>, <<"user_id">> => 7},+ #{<<"id">> => 8, <<"roles">> => [9], <<"role_index">> => #{}},+ #{<<"nested">> => #{<<"id">> => 10, <<"guild_ids">> => [11]}}+ ].++-spec reference_payload(term(), path()) -> term().+reference_payload(Value, Path) when is_map(Value) ->+ maps:fold(+ fun(Key, FieldValue, Acc) ->+ reference_payload_map_field(Path, Key, FieldValue, Acc)+ end,+ #{},+ Value+ );+reference_payload(Value, Path) when is_list(Value) ->+ [reference_payload(Item, Path) || Item <- Value];+reference_payload(Value, _Path) ->+ Value.++-spec reference_payload_map_field(path(), term(), term(), map()) -> map().+reference_payload_map_field(Path, Key, FieldValue, Acc) ->+ case reference_keep_payload_field(Key) of+ true ->+ Acc#{reference_payload_key(Key) => reference_payload_field(Path, Key, FieldValue)};+ false ->+ Acc+ end.++-spec reference_keep_payload_field(term()) -> boolean().+reference_keep_payload_field(Key) ->+ not lists:member(reference_key_binary(Key), [+ <<"recipient_ids">>,+ <<"role_index">>,+ <<"channel_index">>,+ <<"member_role_index">>,+ <<"role_perms_cache">>,+ <<"overwrite_perms_cache">>+ ]).++-spec reference_payload_key(term()) -> term().+reference_payload_key(Key) when is_integer(Key) ->integer_to_binary(Key);-payload_key(Key) when is_atom(Key) ->+reference_payload_key(Key) when is_atom(Key) ->atom_to_binary(Key, utf8);-payload_key(Key) ->+reference_payload_key(Key) ->Key.--spec payload_field(path(), term(), binary() | undefined, term()) -> term().-payload_field(Path, Key, KeyBinary, Value) ->- FieldPath = path_push(KeyBinary, Path),- case field_kind(Path, Key, KeyBinary, Value) of- snowflake -> payload_snowflake(Value, FieldPath);- permission -> payload_permission(Value, FieldPath);- snowflake_list -> payload_snowflake_list(Value, FieldPath);- id_string -> payload_id_string(Value, FieldPath);- generic -> payload(Value, FieldPath)+-spec reference_payload_field(path(), term(), term()) -> term().+reference_payload_field(Path, Key, Value) ->+ FieldPath = reference_path_push(Key, Path),+ case reference_field_kind(Path, Key, Value) of+ snowflake -> reference_payload_snowflake(Value, FieldPath);+ permission -> reference_payload_permission(Value, FieldPath);+ snowflake_list -> reference_payload_snowflake_list(Value, FieldPath);+ id_string -> reference_payload_id_string(Value, FieldPath);+ generic -> reference_payload(Value, FieldPath)end.--spec payload_snowflake(term(), path()) -> term().-payload_snowflake(Value, _Path) when is_integer(Value) ->+-spec reference_payload_snowflake(term(), path()) -> term().+reference_payload_snowflake(Value, _Path) when is_integer(Value) ->integer_to_binary(Value);-payload_snowflake(Value, Path) ->- payload(Value, Path).+reference_payload_snowflake(Value, Path) ->+ reference_payload(Value, Path).--spec payload_permission(term(), path()) -> term().-payload_permission(Value, _Path) when is_integer(Value) ->+-spec reference_payload_permission(term(), path()) -> term().+reference_payload_permission(Value, _Path) when is_integer(Value) ->integer_to_binary(Value);-payload_permission(Value, Path) ->- payload(Value, Path).+reference_payload_permission(Value, Path) ->+ reference_payload(Value, Path).--spec payload_snowflake_list(term(), path()) -> term().-payload_snowflake_list(Values, Path) when is_list(Values) ->- [payload_snowflake(Item, Path) || Item <- Values];-payload_snowflake_list(Value, Path) ->- payload(Value, Path).+-spec reference_payload_snowflake_list(term(), path()) -> term().+reference_payload_snowflake_list(Values, Path) when is_list(Values) ->+ [reference_payload_snowflake(Item, Path) || Item <- Values];+reference_payload_snowflake_list(Value, Path) ->+ reference_payload(Value, Path).--spec payload_id_string(term(), path()) -> term().-payload_id_string(Value, _Path) when is_integer(Value) ->+-spec reference_payload_id_string(term(), path()) -> term().+reference_payload_id_string(Value, _Path) when is_integer(Value) ->integer_to_binary(Value);-payload_id_string(Value, Path) ->- payload(Value, Path).+reference_payload_id_string(Value, Path) ->+ reference_payload(Value, Path).--spec field_kind(path(), term(), binary() | undefined, term()) -> field_kind().-field_kind(Path, Key, KeyBinary, Value) ->- case is_snowflake_record_list_value(Key, Value) of+-spec reference_field_kind(path(), term(), term()) -> field_kind().+reference_field_kind(Path, Key, Value) ->+ case reference_is_snowflake_record_list_value(Key, Value) oftrue -> snowflake_list;- false -> field_kind_binary(Path, KeyBinary, Value)+ false -> reference_field_kind_binary(Path, reference_key_binary(Key), Value)end.--spec field_kind_binary(path(), binary() | undefined, term()) -> field_kind().-field_kind_binary(_Path, <<"permissions">>, _Value) ->+-spec reference_field_kind_binary(path(), binary() | undefined, term()) -> field_kind().+reference_field_kind_binary(_Path, <<"permissions">>, _Value) ->permission;-field_kind_binary(_Path, <<"allow">>, _Value) ->+reference_field_kind_binary(_Path, <<"allow">>, _Value) ->permission;-field_kind_binary(_Path, <<"deny">>, _Value) ->+reference_field_kind_binary(_Path, <<"deny">>, _Value) ->permission;-field_kind_binary(Path, Key, Value) when is_binary(Key) ->- field_kind_for_binary(Path, Key, Value);-field_kind_binary(_Path, _Key, _Value) ->+reference_field_kind_binary(Path, Key, Value) when is_binary(Key) ->+ reference_field_kind_for_binary(Path, Key, Value);+reference_field_kind_binary(_Path, _Key, _Value) ->generic.--spec field_kind_for_binary(path(), binary(), term()) -> field_kind().-field_kind_for_binary(Path, Key, Value) ->- case is_snowflake_list_key(Key, Value) of+-spec reference_field_kind_for_binary(path(), binary(), term()) -> field_kind().+reference_field_kind_for_binary(Path, Key, Value) ->+ case reference_is_snowflake_list_key(Key, Value) oftrue -> snowflake_list;- false -> field_kind_scalar(Path, Key)+ false -> reference_field_kind_scalar(Path, Key)end.--spec field_kind_scalar(path(), binary()) -> snowflake | id_string | generic.-field_kind_scalar(Path, Key) ->- case is_snowflake_key(Path, Key) of+-spec reference_field_kind_scalar(path(), binary()) -> snowflake | id_string | generic.+reference_field_kind_scalar(Path, Key) ->+ case reference_is_snowflake_key(Path, Key) oftrue -> snowflake;- false -> opaque_id_kind(Key)+ false -> reference_opaque_id_kind(Key)end.--spec opaque_id_kind(binary()) -> id_string | generic.-opaque_id_kind(Key) ->- case is_opaque_id_key(Key) of+-spec reference_opaque_id_kind(binary()) -> id_string | generic.+reference_opaque_id_kind(Key) ->+ case reference_is_opaque_id_key(Key) oftrue -> id_string;false -> genericend.--spec is_snowflake_key(path(), binary()) -> boolean().-is_snowflake_key(Path, <<"id">>) ->- not has_any_path([<<"guild_folders">>, <<"rtc_regions">>], Path);-is_snowflake_key(_Path, Key) ->- has_suffix(Key, <<"_id">>) andalso not is_opaque_id_key(Key).---spec is_opaque_id_key(binary()) -> boolean().-is_opaque_id_key(<<"session_id">>) -> true;-is_opaque_id_key(<<"connection_id">>) -> true;-is_opaque_id_key(<<"subscription_id">>) -> true;-is_opaque_id_key(<<"app_id">>) -> true;-is_opaque_id_key(<<"device_id">>) -> true;-is_opaque_id_key(<<"region_id">>) -> true;-is_opaque_id_key(<<"server_id">>) -> true;-is_opaque_id_key(<<"target_id">>) -> true;-is_opaque_id_key(_) -> false.---spec is_snowflake_list_key(binary(), term()) -> boolean().-is_snowflake_list_key(<<"mention_roles">>, _Value) -> true;-is_snowflake_list_key(<<"participants">>, _Value) -> true;-is_snowflake_list_key(<<"ringing">>, _Value) -> true;-is_snowflake_list_key(<<"nsfw_emojis">>, _Value) -> true;-is_snowflake_list_key(<<"pinned_dms">>, _Value) -> true;-is_snowflake_list_key(<<"restricted_guilds">>, _Value) -> true;-is_snowflake_list_key(<<"bot_restricted_guilds">>, _Value) -> true;-is_snowflake_list_key(<<"roles">>, Value) -> is_scalar_snowflake_list(Value);-is_snowflake_list_key(<<"mentions">>, Value) -> is_scalar_snowflake_list(Value);-is_snowflake_list_key(<<"recipients">>, Value) -> is_scalar_snowflake_list(Value);-is_snowflake_list_key(Key, _Value) -> has_suffix(Key, <<"_ids">>).---spec is_snowflake_record_list_value(term(), term()) -> boolean().-is_snowflake_record_list_value(Key, Value) ->- key_is_snowflake(Key) andalso is_scalar_snowflake_list(Value).---spec key_is_snowflake(term()) -> boolean().-key_is_snowflake(Key) when is_integer(Key), Key > 0 ->+-spec reference_is_snowflake_key(path(), binary()) -> boolean().+reference_is_snowflake_key(Path, <<"id">>) ->+ not reference_has_any_path([<<"guild_folders">>, <<"rtc_regions">>], Path);+reference_is_snowflake_key(_Path, Key) ->+ has_suffix(Key, <<"_id">>) andalso not reference_is_opaque_id_key(Key).++-spec reference_is_opaque_id_key(binary()) -> boolean().+reference_is_opaque_id_key(<<"session_id">>) -> true;+reference_is_opaque_id_key(<<"connection_id">>) -> true;+reference_is_opaque_id_key(<<"subscription_id">>) -> true;+reference_is_opaque_id_key(<<"app_id">>) -> true;+reference_is_opaque_id_key(<<"device_id">>) -> true;+reference_is_opaque_id_key(<<"region_id">>) -> true;+reference_is_opaque_id_key(<<"server_id">>) -> true;+reference_is_opaque_id_key(<<"target_id">>) -> true;+reference_is_opaque_id_key(_) -> false.++-spec reference_is_snowflake_list_key(binary(), term()) -> boolean().+reference_is_snowflake_list_key(<<"mention_roles">>, _Value) ->+ true;+reference_is_snowflake_list_key(<<"participants">>, _Value) ->+ true;+reference_is_snowflake_list_key(<<"ringing">>, _Value) ->+ true;+reference_is_snowflake_list_key(<<"nsfw_emojis">>, _Value) ->+ true;+reference_is_snowflake_list_key(<<"pinned_dms">>, _Value) ->+ true;+reference_is_snowflake_list_key(<<"restricted_guilds">>, _Value) ->+ true;+reference_is_snowflake_list_key(<<"bot_restricted_guilds">>, _Value) ->+ true;+reference_is_snowflake_list_key(<<"roles">>, Value) ->+ reference_is_scalar_snowflake_list(Value);+reference_is_snowflake_list_key(<<"mentions">>, Value) ->+ reference_is_scalar_snowflake_list(Value);+reference_is_snowflake_list_key(<<"recipients">>, Value) ->+ reference_is_scalar_snowflake_list(Value);+reference_is_snowflake_list_key(Key, _Value) ->+ has_suffix(Key, <<"_ids">>).++-spec reference_is_snowflake_record_list_value(term(), term()) -> boolean().+reference_is_snowflake_record_list_value(Key, Value) ->+ reference_key_is_snowflake(Key) andalso reference_is_scalar_snowflake_list(Value).++-spec reference_key_is_snowflake(term()) -> boolean().+reference_key_is_snowflake(Key) when is_integer(Key), Key > 0 ->true;-key_is_snowflake(<<First, _/binary>>) when First < $1; First > $9 ->- false;-key_is_snowflake(Key) when is_binary(Key) ->+reference_key_is_snowflake(Key) when is_binary(Key) ->snowflake_id:is_valid(Key);-key_is_snowflake(_) ->+reference_key_is_snowflake(_) ->false.--spec is_scalar_snowflake_list(term()) -> boolean().-is_scalar_snowflake_list([]) ->+-spec reference_is_scalar_snowflake_list(term()) -> boolean().+reference_is_scalar_snowflake_list([]) ->true;-is_scalar_snowflake_list(Values) when is_list(Values) ->- lists:all(fun is_snowflake_scalar/1, Values);-is_scalar_snowflake_list(_) ->+reference_is_scalar_snowflake_list(Values) when is_list(Values) ->+ lists:all(fun reference_is_snowflake_scalar/1, Values);+reference_is_scalar_snowflake_list(_) ->false.--spec is_snowflake_scalar(term()) -> boolean().-is_snowflake_scalar(Value) when is_integer(Value), Value > 0 ->+-spec reference_is_snowflake_scalar(term()) -> boolean().+reference_is_snowflake_scalar(Value) when is_integer(Value), Value > 0 ->true;-is_snowflake_scalar(Value) when is_binary(Value) ->+reference_is_snowflake_scalar(Value) when is_binary(Value) ->snowflake_id:is_valid(Value);-is_snowflake_scalar(_) ->+reference_is_snowflake_scalar(_) ->false.--spec has_suffix(binary(), binary()) -> boolean().-has_suffix(Value, Suffix) ->- Size = byte_size(Value),- SuffixSize = byte_size(Suffix),- case Size >= SuffixSize of- true ->- has_suffix(Value, Suffix, Size - SuffixSize, SuffixSize);- false ->- false- end.---spec has_suffix(binary(), binary(), non_neg_integer(), non_neg_integer()) -> boolean().-has_suffix(Value, Suffix, PrefixSize, SuffixSize) ->- case Value of- <<_:PrefixSize/binary, Suffix:SuffixSize/binary>> -> true;- _ -> false- end.---spec key_binary(term()) -> binary() | undefined.-key_binary(Key) when is_binary(Key) ->+-spec reference_key_binary(term()) -> binary() | undefined.+reference_key_binary(Key) when is_binary(Key) ->Key;-key_binary(Key) when is_atom(Key) ->+reference_key_binary(Key) when is_atom(Key) ->atom_to_binary(Key, utf8);-key_binary(_) ->+reference_key_binary(_) ->undefined.--spec path_push(binary() | undefined, path()) -> path().-path_push(undefined, Path) ->- Path;-path_push(Binary, Path) ->- [Binary | Path].+-spec reference_path_push(term(), path()) -> path().+reference_path_push(Key, Path) ->+ case reference_key_binary(Key) of+ undefined -> Path;+ Binary -> [Binary | Path]+ end.--spec has_any_path([binary()], path()) -> boolean().-has_any_path(Keys, Path) ->+-spec reference_has_any_path([binary()], path()) -> boolean().+reference_has_any_path(Keys, Path) ->lists:any(fun(Key) -> lists:member(Key, Path) end, Keys).++-endif.diff --git a/fluxer_gateway/src/guild/guild_dispatch_member_list.erl b/fluxer_gateway/src/guild/guild_dispatch_member_list.erlindex 88441d289..08a2b14fe 100644--- a/fluxer_gateway/src/guild/guild_dispatch_member_list.erl+++ b/fluxer_gateway/src/guild/guild_dispatch_member_list.erl@@ -43,12 +43,12 @@ dispatch_member_list_event(channel_create, _EventData, _OldState, UpdatedState)broadcast_all_updates(UpdatedState);dispatch_member_list_event(channel_delete, _EventData, _OldState, UpdatedState) ->broadcast_all_updates(UpdatedState);-dispatch_member_list_event(channel_update, EventData, _OldState, UpdatedState) ->- broadcast_channel_update(EventData, UpdatedState);-dispatch_member_list_event(channel_update_bulk, EventData, _OldState, UpdatedState) ->+dispatch_member_list_event(channel_update, EventData, OldState, UpdatedState) ->+ broadcast_channel_update(EventData, OldState, UpdatedState);+dispatch_member_list_event(channel_update_bulk, EventData, OldState, UpdatedState) ->Channels = maps:get(<<"channels">>, EventData, []),lists:foldl(- fun broadcast_channel_update/2,+ fun(Channel, AccState) -> broadcast_channel_update(Channel, OldState, AccState) end,UpdatedState,Channels);@@ -73,19 +73,59 @@ broadcast_all_updates(UpdatedState) ->{ok, NewState} = guild_member_list:broadcast_all_member_list_updates(UpdatedState),NewState.--spec broadcast_channel_update(event_data(), guild_state()) -> guild_state().-broadcast_channel_update(EventData, UpdatedState) ->+-spec broadcast_channel_update(event_data(), guild_state(), guild_state()) -> guild_state().+broadcast_channel_update(EventData, OldState, UpdatedState) ->ChannelIdBin = maps:get(<<"id">>, EventData, undefined),case guild_dispatch_decorate:parse_snowflake(<<"id">>, ChannelIdBin) ofundefined ->UpdatedState;ChannelId ->+ broadcast_channel_update_for_id(ChannelId, OldState, UpdatedState)+ end.++%% guild_state:post_update_channel/2 has already rebuilt this channel's engine+%% earlier in the same dispatch, through+%% guild_member_list_write:rebuild_channels_for_permission_change/2. Rebuilding+%% again here is a second full O(members) build over the same state, so when the+%% engine already exists and nothing in the window can have changed what a+%% rebuild would see, queue the sync and skip the rebuild.+-spec broadcast_channel_update_for_id(integer(), guild_state(), guild_state()) -> guild_state().+broadcast_channel_update_for_id(ChannelId, OldState, UpdatedState) ->+ case rebuild_is_redundant(ChannelId, OldState, UpdatedState) of+ true ->+ sync_channel_list_without_rebuild(ChannelId, UpdatedState);+ false ->{ok, NewState} = guild_member_list:broadcast_member_list_updates_for_channel(ChannelId, UpdatedState),NewStateend.+%% The engine must already exist, because the earlier pass only rebuilds+%% ALREADY-LOADED engines while this pass would build a missing one. Virtual+%% channel access is compared across the whole dispatch, which is a conservative+%% superset of the window between the two rebuilds: guild_visibility can grant a+%% user virtual access to a channel they just lost permission for, and that does+%% change what a rebuild sees.+-spec rebuild_is_redundant(integer(), guild_state(), guild_state()) -> boolean().+rebuild_is_redundant(ChannelId, OldState, UpdatedState) ->+ guild_member_list_channel_engine:ref(integer_to_binary(ChannelId), UpdatedState) =/=+ undefined andalso+ maps:get(virtual_channel_access, OldState, #{}) =:=+ maps:get(virtual_channel_access, UpdatedState, #{}).++%% Mirrors guild_member_list_write:broadcast_list_by_id/4 with the rebuild+%% removed: a list nobody is subscribed to is left alone, otherwise the sync is+%% queued exactly as that function would have queued it.+-spec sync_channel_list_without_rebuild(integer(), guild_state()) -> guild_state().+sync_channel_list_without_rebuild(ChannelId, State) ->+ ListId = integer_to_binary(ChannelId),+ SubsTab = maps:get(member_list_subscriptions, State),+ case map_size(guild_member_list_subs:get_list_subs(ListId, SubsTab)) of+ 0 -> State;+ _ -> guild_member_list_sync_batch:queue_list_sync(ListId, State)+ end.+-spec extract_user_id_from_event(event_data()) -> user_id() | undefined.extract_user_id_from_event(EventData) ->MUser = maps:get(<<"user">>, EventData, #{}),diff --git a/fluxer_gateway/src/guild/guild_dispatch_push.erl b/fluxer_gateway/src/guild/guild_dispatch_push.erlindex 2e72a38b6..700737321 100644--- a/fluxer_gateway/src/guild/guild_dispatch_push.erl+++ b/fluxer_gateway/src/guild/guild_dispatch_push.erl@@ -8,11 +8,30 @@collect_and_send_push_notifications/3]).+-define(MAX_FORMAT_MEMBERS, 50).+-define(CONCURRENCY_LIMIT_KEY, guild_push_concurrency_limit).+-define(DEFAULT_PUSH_CONCURRENCY, 2).+-define(MAX_PUSH_CONCURRENCY, 8).+-define(PUSH_WORKER_MAX_AGE_MS, 60000).+-define(PUSH_COUNTERS, guild_push_counters).+-define(PUSH_COUNTER_KEYS, [+ worker_started,+ worker_completed,+ worker_failed,+ dropped_at_limit,+ spawn_failed,+ slot_reclaimed,+ slot_deduped,+ members_table_missing,+ scan_table_unavailable+]).+-type event() :: atom().-type event_data() :: map().-type guild_state() :: map().-type guild_id() :: integer().-type user_id() :: integer().+-type push_worker() :: {integer(), pid(), integer()}.-export_type([event/0, event_data/0, guild_state/0, guild_id/0]).-spec maybe_send_push_notifications(event(), event_data(), guild_id(), guild_state()) -> ok.@@ -26,32 +45,496 @@ maybe_send_push_notifications(_Event, _FinalData, _GuildId, _UpdatedState) ->-spec maybe_spawn_push(event_data(), guild_id(), guild_state()) -> ok.maybe_spawn_push(FinalData, GuildId, UpdatedState) ->- case push_inflight_alive() of- true -> ok;- false -> spawn_push(FinalData, GuildId, UpdatedState)+ Limit = push_concurrency_limit(),+ Workers = live_push_workers(),+ put(push_inflight_workers, Workers),+ case length(Workers) < Limit of+ true -> spawn_push(FinalData, GuildId, UpdatedState);+ false -> count_push_event(dropped_at_limit)end.--spec push_inflight_alive() -> boolean().-push_inflight_alive() ->- case get(push_inflight) of- Pid when is_pid(Pid) -> is_process_alive(Pid);- _ -> false+-spec local_process_alive(pid()) -> boolean().+local_process_alive(Pid) ->+ node(Pid) =:= node() andalso erlang:is_process_alive(Pid).++-spec push_concurrency_limit() -> pos_integer().+push_concurrency_limit() ->+ case application:get_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, undefined) of+ Value when is_integer(Value), Value > 0 -> min(Value, ?MAX_PUSH_CONCURRENCY);+ _ -> ?DEFAULT_PUSH_CONCURRENCY+ end.++-spec live_push_workers() -> [push_worker()].+live_push_workers() ->+ Now = erlang:monotonic_time(millisecond),+ prune_push_workers(tracked_push_workers(Now), Now, #{}, []).++-spec tracked_push_workers(integer()) -> [term()].+tracked_push_workers(Now) ->+ merge_inflight_push_worker(get(push_inflight), tracked_worker_list(), Now).++-spec tracked_worker_list() -> [term()].+tracked_worker_list() ->+ case get(push_inflight_workers) of+ Workers when is_list(Workers) -> Workers;+ _ -> []+ end.++-spec merge_inflight_push_worker(term(), [term()], integer()) -> [term()].+merge_inflight_push_worker(Pid, Workers, Now) when is_pid(Pid) ->+ case lists:any(fun(Worker) -> push_worker_pid(Worker) =:= Pid end, Workers) of+ true -> Workers;+ false -> Workers ++ [{0, Pid, Now}]+ end;+merge_inflight_push_worker(_Inflight, Workers, _Now) ->+ Workers.++-spec push_worker_pid(term()) -> pid() | undefined.+push_worker_pid({_Gen, Pid, _StartedMs}) when is_pid(Pid) -> Pid;+push_worker_pid(_Worker) -> undefined.++-spec prune_push_workers([term()], integer(), map(), [push_worker()]) -> [push_worker()].+prune_push_workers([], _Now, _Seen, Acc) ->+ lists:reverse(Acc);+prune_push_workers([Worker | Rest], Now, Seen, Acc) ->+ case keep_push_worker(Worker, Now, Seen) of+ {keep, {_Gen, Pid, _StartedMs} = Kept} ->+ prune_push_workers(Rest, Now, Seen#{Pid => true}, [Kept | Acc]);+ drop ->+ prune_push_workers(Rest, Now, Seen, Acc)+ end.++-spec keep_push_worker(term(), integer(), map()) -> {keep, push_worker()} | drop.+keep_push_worker({Gen, Pid, StartedMs} = Worker, Now, Seen) when+ is_integer(Gen), is_pid(Pid), is_integer(StartedMs)+->+ case maps:is_key(Pid, Seen) of+ true -> drop_duplicate_push_worker();+ false -> keep_live_push_worker(Worker, Now - StartedMs, local_process_alive(Pid))+ end;+keep_push_worker(_Worker, _Now, _Seen) ->+ drop.++-spec drop_duplicate_push_worker() -> drop.+drop_duplicate_push_worker() ->+ count_push_event(slot_deduped),+ drop.++-spec keep_live_push_worker(push_worker(), integer(), boolean()) ->+ {keep, push_worker()} | drop.+keep_live_push_worker(_Worker, _AgeMs, false) ->+ drop;+keep_live_push_worker(Worker, AgeMs, true) when AgeMs < ?PUSH_WORKER_MAX_AGE_MS ->+ {keep, Worker};+keep_live_push_worker(_Worker, _AgeMs, true) ->+ count_push_event(slot_reclaimed),+ drop.++-spec note_push_worker(pid()) -> ok.+note_push_worker(Pid) ->+ Gen = erlang:unique_integer([monotonic, positive]),+ StartedMs = erlang:monotonic_time(millisecond),+ put(push_inflight_workers, [{Gen, Pid, StartedMs} | tracked_worker_list()]),+ ok.++-spec count_push_event(atom()) -> ok.+count_push_event(Key) ->+ bump_push_counter(Key, true).++-spec bump_push_counter(atom(), boolean()) -> ok.+bump_push_counter(Key, Retry) ->+ try ets:update_counter(?PUSH_COUNTERS, Key, {2, 1}) of+ _ -> ok+ catch+ error:badarg -> retry_push_counter(Key, Retry)+ end.++-spec retry_push_counter(atom(), boolean()) -> ok.+retry_push_counter(Key, true) ->+ ok = ensure_push_counters(),+ bump_push_counter(Key, false);+retry_push_counter(_Key, false) ->+ ok.++-spec ensure_push_counters() -> ok.+ensure_push_counters() ->+ case ets:whereis(?PUSH_COUNTERS) of+ undefined -> create_push_counters();+ _ -> ensure_push_counter_keys()+ end.++-spec create_push_counters() -> ok.+create_push_counters() ->+ ok = guild_ets_utils:ensure_table(?PUSH_COUNTERS, push_counter_table_options()),+ ensure_push_counter_keys().++-spec push_counter_table_options() -> [term()].+push_counter_table_options() ->+ [named_table, public, set, {read_concurrency, true}, {write_concurrency, true}].++-spec ensure_push_counter_keys() -> ok.+ensure_push_counter_keys() ->+ lists:foreach(fun ensure_push_counter_key/1, ?PUSH_COUNTER_KEYS).++-spec ensure_push_counter_key(atom()) -> ok.+ensure_push_counter_key(Key) ->+ try+ _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}),+ ok+ catch+ error:badarg -> okend.-spec spawn_push(event_data(), guild_id(), guild_state()) -> ok.spawn_push(FinalData, GuildId, UpdatedState) ->- PushState = #{+ Data = maps:get(data, UpdatedState, #{}),+ case maps:get(members_ets, Data, undefined) of+ MembersTab when is_reference(MembersTab) ->+ CompactState = compact_push_state(MembersTab, Data, GuildId, UpdatedState),+ spawn_compact_push(FinalData, GuildId, CompactState);+ _ ->+ missing_members_table(FinalData, GuildId, UpdatedState)+ end.++-spec compact_push_state(ets:tid(), map(), guild_id(), guild_state()) -> guild_state().+compact_push_state(MembersTab, Data, GuildId, UpdatedState) ->+ Sessions = maps:get(sessions, UpdatedState, #{}),+ #{+ id => maps:get(id, UpdatedState, GuildId),+ data => compact_push_data(Data),+ virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{}),+ members_ets => MembersTab,+ member_presence => maps:get(member_presence, UpdatedState, undefined),+ session_eligibility => build_push_session_eligibility(Sessions, UpdatedState),+ member_count => maps:get(member_count, UpdatedState, undefined)+ }.++-spec compact_push_data(map()) -> map().+compact_push_data(Data) ->+ maps:with(+ [+ <<"guild">>,+ <<"roles">>,+ <<"role_index">>,+ role_perms_cache,+ <<"channels">>,+ <<"channel_index">>,+ overwrite_perms_cache,+ members_ets+ ],+ Data+ ).++-spec spawn_compact_push(event_data(), guild_id(), guild_state()) -> ok.+spawn_compact_push(FinalData, GuildId, CompactState) ->+ spawn_push_worker(+ fun() ->+ collect_and_send_compact_push_notifications(FinalData, GuildId, CompactState)+ end,+ GuildId+ ).++-spec missing_members_table(event_data(), guild_id(), guild_state()) -> ok.+missing_members_table(FinalData, GuildId, UpdatedState) ->+ count_push_event(members_table_missing),+ logger:warning(+ "guild_push_members_table_unavailable: guild_id=~p phase=spawn",+ [GuildId]+ ),+ spawn_legacy_push(FinalData, GuildId, UpdatedState).++-spec spawn_legacy_push(event_data(), guild_id(), guild_state()) -> ok.+spawn_legacy_push(FinalData, GuildId, UpdatedState) ->+ LegacyState = legacy_push_state(GuildId, UpdatedState),+ spawn_push_worker(+ fun() ->+ collect_and_send_push_notifications(FinalData, GuildId, LegacyState)+ end,+ GuildId+ ).++-spec legacy_push_state(guild_id(), guild_state()) -> guild_state().+legacy_push_state(GuildId, UpdatedState) ->+ #{id => maps:get(id, UpdatedState, GuildId),data => maps:get(data, UpdatedState, #{}),sessions => maps:get(sessions, UpdatedState, #{}),- virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{})- },- Pid = spawn(fun() ->- collect_and_send_push_notifications(FinalData, GuildId, PushState)- end),- put(push_inflight, Pid),+ member_presence => maps:get(member_presence, UpdatedState, undefined),+ virtual_channel_access => maps:get(virtual_channel_access, UpdatedState, #{}),+ member_count => maps:get(member_count, UpdatedState, undefined)+ }.++-spec spawn_push_worker(fun(() -> ok), guild_id()) -> ok.+spawn_push_worker(Worker, GuildId) ->+ Counted = fun() -> run_counted_push_worker(Worker) end,+ case try_spawn_push_worker(Counted, GuildId) of+ {ok, Pid} ->+ put(push_inflight, Pid),+ note_push_worker(Pid),+ count_push_event(worker_started);+ error ->+ count_push_event(spawn_failed)+ end.++-spec run_counted_push_worker(fun(() -> ok)) -> ok.+run_counted_push_worker(Worker) ->+ try Worker() of+ _ -> count_push_event(worker_completed)+ catch+ _Class:_Reason -> count_push_event(worker_failed)+ end.++-spec try_spawn_push_worker(fun(() -> ok), guild_id()) -> {ok, pid()} | error.+try_spawn_push_worker(Worker, GuildId) ->+ try+ {ok, spawn(fun() -> run_deprioritised_push_worker(Worker) end)}+ catch+ error:system_limit ->+ logger:warning(+ "guild_push_worker_spawn_failed: guild_id=~p reason=system_limit",+ [GuildId]+ ),+ error+ end.++-spec run_deprioritised_push_worker(fun(() -> ok)) -> ok.+run_deprioritised_push_worker(Worker) ->+ ok = apply_push_worker_priority(),+ Worker().++-spec apply_push_worker_priority() -> ok.+apply_push_worker_priority() ->+ _ = erlang:process_flag(priority, low),+ ok.++-spec large_guild_meta(guild_state()) -> map() | undefined.+large_guild_meta(State) ->+ case maps:get(member_count, State, undefined) of+ MemberCount when is_integer(MemberCount), MemberCount >= 0 ->+ #{member_count => MemberCount, features => guild_features(State)};+ _ ->+ undefined+ end.++-spec guild_features(guild_state()) -> [binary()].+guild_features(State) ->+ Data = maps:get(data, State, #{}),+ Guild = maps:get(<<"guild">>, Data, #{}),+ case maps:get(<<"features">>, Guild, []) of+ Features when is_list(Features) -> Features;+ _ -> []+ end.++-spec collect_and_send_compact_push_notifications(event_data(), guild_id(), guild_state()) ->+ ok.+collect_and_send_compact_push_notifications(MessageData, GuildId, State) ->+ case guild_dispatch_config:should_send_push_notifications(State) of+ false -> ok;+ true -> send_compact_push_notifications(MessageData, GuildId, State)+ end.++-spec send_compact_push_notifications(event_data(), guild_id(), guild_state()) -> ok.+send_compact_push_notifications(MessageData, GuildId, State) ->+ ChannelIdBin = maps:get(<<"channel_id">>, MessageData, undefined),+ case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of+ undefined -> ok;+ ChannelId -> scan_and_send_compact_push(MessageData, GuildId, ChannelId, State)+ end.++-spec scan_and_send_compact_push(event_data(), guild_id(), integer(), guild_state()) -> ok.+scan_and_send_compact_push(MessageData, GuildId, ChannelId, State) ->+ Context = compact_scan_context(MessageData, ChannelId, State),+ MembersTab = maps:get(members_ets, State),+ case scan_push_members(MembersTab, Context, State) of+ {ok, #{eligible_user_ids := []}} ->+ ok;+ {ok, Scan} ->+ send_compact_scanned_push(MessageData, GuildId, Scan, State);+ {error, table_unavailable} ->+ scan_table_unavailable(GuildId)+ end.++-spec scan_table_unavailable(guild_id()) -> ok.+scan_table_unavailable(GuildId) ->+ count_push_event(scan_table_unavailable),+ logger:warning(+ "guild_push_members_table_unavailable: guild_id=~p phase=scan",+ [GuildId]+ ),ok.+-spec compact_scan_context(event_data(), integer(), guild_state()) -> map().+compact_scan_context(MessageData, ChannelId, State) ->+ MentionRoles = maps:get(<<"mention_roles">>, MessageData, []),+ #{+ mention_everyone => maps:get(<<"mention_everyone">>, MessageData, false) =:= true,+ direct_mentions => direct_mention_id_set(MessageData),+ mention_roles => mention_role_id_set(MentionRoles),+ format_members => format_member_id_set(MessageData),+ channel_id => ChannelId,+ session_eligibility => maps:get(session_eligibility, State)+ }.++-spec scan_push_members(ets:tid(), map(), guild_state()) ->+ {ok, map()} | {error, table_unavailable}.+scan_push_members(MembersTab, Context, State) ->+ try+ {ok, fold_push_members(member_id_snapshot(MembersTab), MembersTab, Context, State)}+ catch+ error:badarg -> {error, table_unavailable}+ end.++-spec initial_scan_acc() -> map().+initial_scan_acc() ->+ #{eligible_user_ids => [], user_roles => #{}, format_members => #{}}.++-spec member_id_snapshot(ets:tid()) -> [user_id()].+member_id_snapshot(MembersTab) ->+ ets:select(MembersTab, [{{'$1', '_'}, [], ['$1']}]).++-spec fold_push_members([user_id()], ets:tid(), map(), guild_state()) -> map().+fold_push_members(UserIds, MembersTab, Context, State) ->+ lists:foldl(+ fun(UserId, Acc) -> scan_push_member_id(UserId, MembersTab, Context, State, Acc) end,+ initial_scan_acc(),+ UserIds+ ).++-spec scan_push_member_id(user_id(), ets:tid(), map(), guild_state(), map()) -> map().+scan_push_member_id(UserId, MembersTab, Context, State, Acc) ->+ case ets:lookup(MembersTab, UserId) of+ [Row] -> scan_push_member(Row, Context, State, Acc);+ _ -> Acc+ end.++-spec scan_push_member(term(), map(), guild_state(), map()) -> map().+scan_push_member({UserId, Member}, Context, State, Acc) when+ is_integer(UserId), UserId > 0, is_map(Member)+->+ Acc1 = maybe_collect_format_member(UserId, Member, Context, Acc),+ case scanned_member_is_eligible(UserId, Member, Context, State) of+ true -> add_scanned_member(UserId, Member, Acc1);+ false -> Acc1+ end;+scan_push_member(_Row, _Context, _State, Acc) ->+ Acc.++-spec scanned_member_is_eligible(user_id(), map(), map(), guild_state()) -> boolean().+scanned_member_is_eligible(UserId, Member, Context, State) ->+ is_push_candidate(UserId, Member, Context) andalso+ guild_permissions:can_view_channel(+ UserId, maps:get(channel_id, Context), Member, State+ ).++-spec add_scanned_member(user_id(), map(), map()) -> map().+add_scanned_member(UserId, Member, Acc) ->+ Eligible = maps:get(eligible_user_ids, Acc),+ UserRoles = maps:get(user_roles, Acc),+ Acc#{+ eligible_user_ids := [UserId | Eligible],+ user_roles := UserRoles#{UserId => extract_role_ids(Member)}+ }.++-spec maybe_collect_format_member(user_id(), map(), map(), map()) -> map().+maybe_collect_format_member(UserId, Member, Context, Acc) ->+ case maps:is_key(UserId, maps:get(format_members, Context)) of+ true ->+ Members = maps:get(format_members, Acc),+ Acc#{format_members := Members#{UserId => Member}};+ false ->+ Acc+ end.++-spec is_push_candidate(user_id(), map(), map()) -> boolean().+is_push_candidate(UserId, Member, Context) ->+ maps:get(UserId, maps:get(session_eligibility, Context), true) orelse+ (not mentions_respect_active_session() andalso+ is_mention_candidate(UserId, Member, Context)).++-spec is_mention_candidate(user_id(), map(), map()) -> boolean().+is_mention_candidate(UserId, Member, Context) ->+ maps:get(mention_everyone, Context) orelse+ maps:is_key(UserId, maps:get(direct_mentions, Context)) orelse+ member_has_mentioned_role(Member, maps:get(mention_roles, Context)).++-spec mentions_respect_active_session() -> boolean().+mentions_respect_active_session() ->+ application:get_env(fluxer_gateway, push_mentions_respect_active_session, true) =:= true.++-spec send_compact_scanned_push(event_data(), guild_id(), map(), guild_state()) -> ok.+send_compact_scanned_push(MessageData, GuildId, Scan, State) ->+ FormatData = compact_format_data(maps:get(data, State), maps:get(format_members, Scan)),+ send_push_to_eligible_users(+ MessageData,+ GuildId,+ lists:reverse(maps:get(eligible_user_ids, Scan)),+ maps:get(user_roles, Scan),+ maps:get(session_eligibility, State),+ FormatData,+ large_guild_meta(State)+ ).++-spec compact_format_data(map(), map()) -> map().+compact_format_data(Data, FormatMembers) ->+ Data#{+ <<"members">> => FormatMembers,+ members_normalized => FormatMembers,+ members_sorted_ids => lists:sort(maps:keys(FormatMembers)),+ <<"member_role_index">> =>+ guild_data_index_members:build_member_role_index(FormatMembers)+ }.++-spec direct_mention_id_set(event_data()) -> map().+direct_mention_id_set(MessageData) ->+ lists:foldl(+ fun add_direct_mention_id/2,+ #{},+ maps:get(<<"mentions">>, MessageData, [])+ ).++-spec add_direct_mention_id(term(), map()) -> map().+add_direct_mention_id(Mention, Acc) ->+ case direct_mention_user_id(Mention) of+ {true, UserId} -> Acc#{UserId => true};+ false -> Acc+ end.++-spec format_member_id_set(event_data()) -> map().+format_member_id_set(MessageData) ->+ AuthorSet = author_format_id_set(maps:get(<<"author">>, MessageData, undefined)),+ collect_format_mention_ids(+ maps:get(<<"mentions">>, MessageData, []),+ ?MAX_FORMAT_MEMBERS,+ AuthorSet+ ).++-spec author_format_id_set(term()) -> map().+author_format_id_set(Author) when is_map(Author) ->+ case direct_mention_user_id(Author) of+ {true, UserId} -> #{UserId => true};+ false -> #{}+ end;+author_format_id_set(_Author) ->+ #{}.++-spec collect_format_mention_ids([term()], non_neg_integer(), map()) -> map().+collect_format_mention_ids(_Mentions, 0, Acc) ->+ Acc;+collect_format_mention_ids([], _Remaining, Acc) ->+ Acc;+collect_format_mention_ids([Mention | Rest], Remaining, Acc) ->+ case direct_mention_user_id(Mention) of+ {true, UserId} -> collect_format_mention_id(UserId, Rest, Remaining, Acc);+ false -> collect_format_mention_ids(Rest, Remaining, Acc)+ end.++-spec collect_format_mention_id(user_id(), [term()], non_neg_integer(), map()) -> map().+collect_format_mention_id(UserId, Rest, Remaining, Acc) ->+ case maps:is_key(UserId, Acc) of+ true -> collect_format_mention_ids(Rest, Remaining, Acc);+ false -> collect_format_mention_ids(Rest, Remaining - 1, Acc#{UserId => true})+ end.+-spec collect_and_send_push_notifications(event_data(), guild_id(), guild_state()) -> ok.collect_and_send_push_notifications(MessageData, GuildId, State) ->case guild_dispatch_config:should_send_push_notifications(State) of@@ -64,7 +547,7 @@ send_push_notifications(MessageData, GuildId, State) ->Data = maps:get(data, State),Members = guild_data_index:member_map(Data),Sessions = maps:get(sessions, State, #{}),- SessionEligibility = build_push_session_eligibility(Sessions),+ SessionEligibility = build_push_session_eligibility(Sessions, State),CandidateUserIds = push_candidate_user_ids(Members, SessionEligibility, MessageData),ChannelIdBin = maps:get(<<"channel_id">>, MessageData, undefined),case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of@@ -114,7 +597,8 @@ send_to_eligible(EligibleUserIds,UserRolesMap,SessionEligibility,- Data+ Data,+ large_guild_meta(State))end.@@ -128,8 +612,9 @@ push_candidate_user_ids(Members, SessionEligibility, MessageData) ->).-spec push_candidate_user_ids(term(), map(), map(), event_data()) -> [user_id()].-push_candidate_user_ids(true, Members, _SessionEligibility, _MessageData) ->- maps:keys(Members);+push_candidate_user_ids(true, Members, SessionEligibility, _MessageData) ->+ {CandidateUserIds, _Seen} = base_candidate_acc(Members, SessionEligibility),+ lists:reverse(CandidateUserIds);push_candidate_user_ids(_MentionEveryone, Members, SessionEligibility, MessageData) ->BaseAcc = base_candidate_acc(Members, SessionEligibility),{CandidateUserIds, _Seen} =@@ -276,6 +761,72 @@ view_to_filtermap(UserId, ChannelId, Member, State) ->false -> falseend.+-spec build_push_session_eligibility(map(), guild_state()) -> #{user_id() => boolean()}.+build_push_session_eligibility(Sessions, State) ->+ case presence_eligibility_enabled() of+ true -> build_push_presence_eligibility(Sessions, State);+ false -> build_push_session_eligibility(Sessions)+ end.++-spec presence_eligibility_enabled() -> boolean().+presence_eligibility_enabled() ->+ application:get_env(fluxer_gateway, push_presence_eligibility, true) =:= true.++-spec build_push_presence_eligibility(map(), guild_state()) -> #{user_id() => boolean()}.+build_push_presence_eligibility(Sessions, State) ->+ Presences = maps:get(member_presence, State, undefined),+ maps:fold(+ fun(_Sid, Session, Acc) ->+ accumulate_presence_eligibility(Session, Presences, Acc)+ end,+ #{},+ Sessions+ ).++-spec accumulate_presence_eligibility(map(), term(), #{user_id() => boolean()}) ->+ #{user_id() => boolean()}.+accumulate_presence_eligibility(Session, Presences, Acc) ->+ case maps:get(user_id, Session, undefined) of+ UserId when is_integer(UserId) ->+ Acc#{+ UserId =>+ maps:get(UserId, Acc, true) andalso+ not actively_engaged(UserId, Presences)+ };+ _ ->+ Acc+ end.++-spec actively_engaged(user_id(), term()) -> boolean().+actively_engaged(_UserId, undefined) ->+ false;+actively_engaged(UserId, Presences) ->+ case lookup_presence_safe(UserId, Presences) of+ undefined -> false;+ Presence -> presence_is_active(Presence)+ end.++-spec lookup_presence_safe(user_id(), term()) -> map() | undefined.+lookup_presence_safe(UserId, Presences) ->+ try guild_state_member:lookup_presence(Presences, UserId) of+ Presence when is_map(Presence) -> Presence;+ _ -> undefined+ catch+ error:badarg -> undefined+ end.++-spec presence_is_active(map()) -> boolean().+presence_is_active(Presence) ->+ Status = maps:get(<<"status">>, Presence, <<"offline">>),+ Afk = maps:get(<<"afk">>, Presence, false) =:= true,+ Mobile = maps:get(<<"mobile">>, Presence, false) =:= true,+ is_engaged_status(Status) andalso not Afk andalso not Mobile.++-spec is_engaged_status(term()) -> boolean().+is_engaged_status(<<"online">>) -> true;+is_engaged_status(<<"dnd">>) -> true;+is_engaged_status(_Status) -> false.+-spec build_push_session_eligibility(map()) -> #{user_id() => boolean()}.build_push_session_eligibility(Sessions) ->maps:fold(@@ -299,22 +850,23 @@ accumulate_session_eligibility(Session, Acc) ->Accend.--spec send_push_to_eligible_users(event_data(), guild_id(), [user_id()], map(), map(), map()) ->- ok.+-spec send_push_to_eligible_users(+ event_data(), guild_id(), [user_id()], map(), map(), map(), map() | undefined+) -> ok.send_push_to_eligible_users(MessageData,GuildId,EligibleUserIds,UserRolesMap,ConnectedUsers,- Data+ Data,+ LargeGuildMeta) ->AuthorIdBin = maps:get(<<"id">>, maps:get(<<"author">>, MessageData, #{}), undefined),case guild_dispatch_decorate:parse_snowflake(<<"author.id">>, AuthorIdBin) ofundefined ->ok;AuthorId ->- Guild = maps:get(<<"guild">>, Data),ChannelIdBin = maps:get(<<"channel_id">>, MessageData),ChannelName = find_channel_name(ChannelIdBin, Data),RoleNames = build_role_names_map(Data),@@ -324,11 +876,11 @@ send_push_to_eligible_users(EligibleUserIds,UserRolesMap,ConnectedUsers,- Guild,ChannelName,RoleNames,Data,- AuthorId+ AuthorId,+ LargeGuildMeta)end.@@ -338,11 +890,11 @@ send_push_to_eligible_users([user_id()],map(),map(),- map(),binary(),map(),map(),- integer()+ integer(),+ map() | undefined) -> ok.do_send_push(MessageData,@@ -350,12 +902,13 @@ do_send_push(EligibleUserIds,UserRolesMap,ConnectedUsers,- Guild,ChannelName,RoleNames,Data,- AuthorId+ AuthorId,+ LargeGuildMeta) ->+ Guild = maps:get(<<"guild">>, Data),DefaultMessageNotifications = maps:get(<<"default_message_notifications">>, Guild, 0),GuildName = maps:get(<<"name">>, Guild, <<"Unknown">>),push:handle_message_create(#{@@ -372,9 +925,19 @@ do_send_push(MessageData, GuildId, RoleNames, Data),user_roles => UserRolesMap,- connected_users => ConnectedUsers+ connected_users => ConnectedUsers,+ guild_member_count => meta_member_count(LargeGuildMeta),+ guild_features => meta_features(LargeGuildMeta)}).+-spec meta_member_count(map() | undefined) -> non_neg_integer() | undefined.+meta_member_count(#{member_count := MemberCount}) -> MemberCount;+meta_member_count(_Meta) -> undefined.++-spec meta_features(map() | undefined) -> [binary()] | undefined.+meta_features(#{features := Features}) -> Features;+meta_features(_Meta) -> undefined.+-spec find_channel_name(binary(), map()) -> binary().find_channel_name(ChannelIdBin, Data) ->case guild_dispatch_decorate:parse_snowflake(<<"channel_id">>, ChannelIdBin) of@@ -444,6 +1007,134 @@ collect_role_id(Role, Acc) ->-ifdef(TEST).-include_lib("eunit/include/eunit.hrl").+presence_eligibility_suppresses_active_desktop_test() ->+ Tab = ets:new(test_member_presence, [set, public]),+ try+ ets:insert(+ Tab, {1, #{<<"status">> => <<"online">>, <<"mobile">> => false, <<"afk">> => false}}+ ),+ ?assertEqual(+ #{1 => false},+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{+ member_presence => Tab+ })+ )+ after+ ets:delete(Tab)+ end.++presence_eligibility_allows_mobile_only_session_test() ->+ Tab = ets:new(test_member_presence, [set, public]),+ try+ ets:insert(+ Tab, {1, #{<<"status">> => <<"online">>, <<"mobile">> => true, <<"afk">> => false}}+ ),+ ?assertEqual(+ #{1 => true},+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{+ member_presence => Tab+ })+ )+ after+ ets:delete(Tab)+ end.++presence_eligibility_allows_idle_and_afk_test() ->+ Tab = ets:new(test_member_presence, [set, public]),+ try+ ets:insert(+ Tab, {1, #{<<"status">> => <<"idle">>, <<"mobile">> => false, <<"afk">> => false}}+ ),+ ets:insert(+ Tab, {2, #{<<"status">> => <<"online">>, <<"mobile">> => false, <<"afk">> => true}}+ ),+ ?assertEqual(+ #{1 => true, 2 => true},+ build_push_presence_eligibility(+ #{<<"s1">> => #{user_id => 1}, <<"s2">> => #{user_id => 2}},+ #{member_presence => Tab}+ )+ )+ after+ ets:delete(Tab)+ end.++presence_eligibility_allows_unknown_presence_test() ->+ Tab = ets:new(test_member_presence, [set, public]),+ try+ ?assertEqual(+ #{7 => true},+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 7}}, #{+ member_presence => Tab+ })+ )+ after+ ets:delete(Tab)+ end.++presence_eligibility_allows_when_presence_table_missing_test() ->+ ?assertEqual(+ #{1 => true},+ build_push_presence_eligibility(#{<<"s1">> => #{user_id => 1}}, #{})+ ).++legacy_session_eligibility_suppresses_every_real_session_test() ->+ RealSession = #{+ session_id => <<"s1">>,+ user_id => 1,+ pid => self(),+ active_guilds => sets:new(),+ bot => false,+ is_staff => false,+ pending_connect => false,+ viewable_channels => #{}+ },+ ?assertEqual(#{1 => false}, build_push_session_eligibility(#{<<"s1">> => RealSession})).++is_push_candidate_suppresses_mention_for_active_user_test() ->+ Context = #{+ session_eligibility => #{1 => false},+ mention_everyone => true,+ direct_mentions => #{1 => true},+ mention_roles => #{}+ },+ ?assertEqual(false, is_push_candidate(1, #{}, Context)).++is_push_candidate_allows_eligible_user_test() ->+ Context = #{+ session_eligibility => #{1 => true},+ mention_everyone => false,+ direct_mentions => #{},+ mention_roles => #{}+ },+ ?assertEqual(true, is_push_candidate(1, #{}, Context)).++is_push_candidate_allows_sessionless_user_test() ->+ Context = #{+ session_eligibility => #{},+ mention_everyone => false,+ direct_mentions => #{},+ mention_roles => #{}+ },+ ?assertEqual(true, is_push_candidate(9, #{}, Context)).++push_candidate_user_ids_everyone_respects_active_sessions_test() ->+ Members = #{1 => #{}, 2 => #{}, 3 => #{}},+ ?assertEqual(+ [],+ push_candidate_user_ids(+ Members, #{1 => false, 2 => false, 3 => false}, #{<<"mention_everyone">> => true}+ )+ ),+ ?assertEqual(+ [1, 3],+ lists:sort(+ push_candidate_user_ids(+ Members, #{1 => true, 2 => false}, #{<<"mention_everyone">> => true}+ )+ )+ ).+build_push_session_eligibility_test() ->Sessions = #{<<"s1">> => #{user_id => 1, mobile => false, afk => true},@@ -477,14 +1168,6 @@ push_candidate_user_ids_includes_connected_mentioned_users_test() ->CandidateUserIds = push_candidate_user_ids(Members, SessionEligibility, MessageData),?assertEqual([1, 3], lists:sort(CandidateUserIds)).-push_candidate_user_ids_includes_everyone_even_when_connected_test() ->- Members = #{1 => #{}, 2 => #{}, 3 => #{}},- SessionEligibility = #{1 => false, 2 => false, 3 => false},- CandidateUserIds = push_candidate_user_ids(- Members, SessionEligibility, #{<<"mention_everyone">> => true}- ),- ?assertEqual([1, 2, 3], lists:sort(CandidateUserIds)).-push_candidate_user_ids_deduplicates_mentions_test() ->Members = #{1 => #{<<"roles">> => [<<"10">>]}, 2 => #{<<"roles">> => []}},SessionEligibility = #{1 => false, 2 => false},@@ -557,13 +1240,18 @@ send_push_to_eligible_users_uses_full_data_for_channel_name_test() ->}},?assertEqual(- ok, send_push_to_eligible_users(MessageData, 10, [1], #{1 => []}, #{}, Data)+ ok,+ send_push_to_eligible_users(+ MessageData, 10, [1], #{1 => []}, #{}, Data, undefined+ )),receive{push_params, Params} ->?assertEqual(<<"general">>, maps:get(channel_name, Params)),?assertEqual(<<"Test Guild">>, maps:get(guild_name, Params)),- ?assertEqual(#{200 => <<"Alerts">>}, maps:get(role_names, Params))+ ?assertEqual(#{200 => <<"Alerts">>}, maps:get(role_names, Params)),+ ?assertEqual(undefined, maps:get(guild_member_count, Params)),+ ?assertEqual(undefined, maps:get(guild_features, Params))after 1000 ->?assert(false)end,@@ -602,4 +1290,518 @@ extract_role_ids_missing_key_test() ->Member = #{},?assertEqual([], extract_role_ids(Member)).+meta_accessors_default_to_undefined_test() ->+ ?assertEqual(undefined, meta_member_count(undefined)),+ ?assertEqual(undefined, meta_features(undefined)),+ ?assertEqual(5, meta_member_count(#{member_count => 5, features => []})),+ ?assertEqual([], meta_features(#{member_count => 5, features => []})).++compact_push_state_drops_members_and_keeps_member_count_test() ->+ Tab = ets:new(test_members, [set, public]),+ try+ Data = compact_test_data(Tab),+ UpdatedState = #{+ id => 7,+ data => Data,+ sessions => #{<<"s1">> => #{user_id => 1}},+ virtual_channel_access => #{},+ member_count => 49435+ },+ Compact = compact_push_state(Tab, Data, 7, UpdatedState),+ CompactData = maps:get(data, Compact),+ ?assertEqual(49435, maps:get(member_count, Compact)),+ ?assertEqual(Tab, maps:get(members_ets, Compact)),+ ?assertNot(maps:is_key(sessions, Compact)),+ ?assertEqual(#{1 => true}, maps:get(session_eligibility, Compact)),+ ?assertNot(maps:is_key(<<"members">>, CompactData)),+ ?assertNot(maps:is_key(members_normalized, CompactData)),+ ?assertNot(maps:is_key(<<"member_role_index">>, CompactData)),+ ?assertEqual(+ #{member_count => 49435, features => [<<"COMMUNITY">>]},+ large_guild_meta(Compact)+ )+ after+ ets:delete(Tab)+ end.++legacy_push_state_carries_member_count_test() ->+ UpdatedState = #{id => 7, data => #{}, sessions => #{}, member_count => 1234},+ Legacy = legacy_push_state(7, UpdatedState),+ ?assertEqual(1234, maps:get(member_count, Legacy)),+ ?assertEqual(#{}, maps:get(sessions, Legacy)),+ ?assertEqual(#{member_count => 1234, features => []}, large_guild_meta(Legacy)).++compact_scan_collects_eligible_users_and_format_members_test() ->+ Tab = ets:new(test_members, [set, public]),+ try+ true = ets:insert(Tab, {1, #{<<"roles">> => []}}),+ true = ets:insert(Tab, {2, #{<<"roles">> => []}}),+ true = ets:insert(Tab, {3, #{<<"roles">> => [<<"200">>]}}),+ State = compact_scan_state(Tab),+ MessageData = #{<<"mentions">> => [#{<<"id">> => <<"3">>}]},+ Context = compact_scan_context(MessageData, 10, State),+ {ok, Scan} = scan_push_members(Tab, Context, State),+ ?assertEqual([1], lists:sort(maps:get(eligible_user_ids, Scan))),+ ?assertEqual(#{3 => #{<<"roles">> => [<<"200">>]}}, maps:get(format_members, Scan)),+ ?assertNot(maps:is_key(3, maps:get(user_roles, Scan))),+ ?assertEqual([], maps:get(1, maps:get(user_roles, Scan)))+ after+ ets:delete(Tab)+ end.++compact_scan_includes_mentioned_user_when_bypass_is_re_enabled_test() ->+ Tab = ets:new(test_members, [set, public]),+ application:set_env(fluxer_gateway, push_mentions_respect_active_session, false),+ try+ true = ets:insert(Tab, {1, #{<<"roles">> => []}}),+ true = ets:insert(Tab, {3, #{<<"roles">> => [<<"200">>]}}),+ State = compact_scan_state(Tab),+ MessageData = #{<<"mentions">> => [#{<<"id">> => <<"3">>}]},+ Context = compact_scan_context(MessageData, 10, State),+ {ok, Scan} = scan_push_members(Tab, Context, State),+ ?assertEqual([1, 3], lists:sort(maps:get(eligible_user_ids, Scan)))+ after+ application:unset_env(fluxer_gateway, push_mentions_respect_active_session),+ ets:delete(Tab)+ end.++scan_push_members_reports_unavailable_table_test() ->+ Tab = ets:new(test_members, [set, public]),+ State = compact_scan_state(Tab),+ Context = compact_scan_context(#{}, 10, State),+ true = ets:delete(Tab),+ ?assertEqual({error, table_unavailable}, scan_push_members(Tab, Context, State)).++collect_format_mention_ids_is_bounded_test() ->+ Mentions = [#{<<"id">> => integer_to_binary(Id)} || Id <- lists:seq(1, 80)],+ Result = collect_format_mention_ids(Mentions, ?MAX_FORMAT_MEMBERS, #{}),+ ?assertEqual(?MAX_FORMAT_MEMBERS, map_size(Result)).++format_member_id_set_includes_author_test() ->+ MessageData = #{+ <<"author">> => #{<<"id">> => <<"99">>},+ <<"mentions">> => [#{<<"id">> => <<"1">>}, #{<<"id">> => <<"1">>}]+ },+ ?assertEqual(#{1 => true, 99 => true}, format_member_id_set(MessageData)).++spawn_push_without_members_table_falls_back_to_legacy_push_test() ->+ reset_push_worker_state(),+ try+ ?assertEqual(ok, spawn_push(#{}, 7, #{id => 7, data => #{}, sessions => #{}})),+ ?assert(is_pid(get(push_inflight)))+ after+ reset_push_worker_state()+ end.++compact_push_carries_large_guild_metadata_test() ->+ Self = self(),+ Tab = ets:new(test_members, [set, public]),+ ok = meck:new(push, [passthrough, no_link]),+ try+ ok = meck:expect(push, handle_message_create, fun(Params) ->+ Self ! {push_params, Params},+ ok+ end),+ State = compact_scan_state(Tab),+ Scan = #{+ eligible_user_ids => [1],+ user_roles => #{1 => []},+ format_members => #{1 => #{<<"nick">> => <<"one">>}}+ },+ MessageData = #{+ <<"channel_id">> => <<"10">>,+ <<"author">> => #{<<"id">> => <<"42">>}+ },+ ?assertEqual(ok, send_compact_scanned_push(MessageData, 7, Scan, State)),+ receive+ {push_params, Params} ->+ ?assertEqual(49435, maps:get(guild_member_count, Params)),+ ?assertEqual([<<"COMMUNITY">>], maps:get(guild_features, Params)),+ ?assertEqual([1], maps:get(user_ids, Params)),+ ?assertEqual(<<"general">>, maps:get(channel_name, Params))+ after 1000 ->+ ?assert(false)+ end,+ ?assert(meck:validate(push))+ after+ meck:unload(push),+ ets:delete(Tab)+ end.++compact_format_data_restricts_member_map_test() ->+ Data = #{<<"guild">> => #{}, <<"members">> => #{1 => #{}, 2 => #{}, 3 => #{}}},+ FormatMembers = #{2 => #{<<"roles">> => [<<"200">>]}},+ Result = compact_format_data(Data, FormatMembers),+ ?assertEqual(FormatMembers, maps:get(<<"members">>, Result)),+ ?assertEqual(FormatMembers, maps:get(members_normalized, Result)),+ ?assertEqual([2], maps:get(members_sorted_ids, Result)),+ ?assertEqual(#{200 => #{2 => true}}, maps:get(<<"member_role_index">>, Result)).++inflight_pid_without_a_slot_still_counts_against_the_limit_test() ->+ reset_push_worker_state(),+ Blocker = blocking_push_worker(),+ put(push_inflight, Blocker),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1),+ try+ ?assertEqual([Blocker], worker_pids(live_push_workers())),+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertEqual(Blocker, get(push_inflight)),+ ?assertEqual([Blocker], worker_pids(get(push_inflight_workers)))+ after+ stop_push_worker(Blocker),+ reset_push_worker_state()+ end.++unknown_worker_entries_are_dropped_test() ->+ reset_push_worker_state(),+ Blocker = blocking_push_worker(),+ put(push_inflight_workers, [Blocker, {Blocker, 0}, not_a_worker]),+ try+ ?assertEqual([], live_push_workers())+ after+ stop_push_worker(Blocker),+ reset_push_worker_state()+ end.++remote_worker_pid_is_dropped_without_badarg_test() ->+ reset_push_worker_state(),+ Remote = remote_test_pid(),+ put(push_inflight_workers, [{1, Remote, erlang:monotonic_time(millisecond)}]),+ try+ ?assertNotEqual(node(), node(Remote)),+ ?assertError(badarg, erlang:is_process_alive(Remote)),+ ?assertEqual(false, local_process_alive(Remote)),+ ?assertEqual([], live_push_workers())+ after+ reset_push_worker_state()+ end.++reused_worker_pid_consumes_only_one_slot_test() ->+ reset_push_worker_state(),+ Blocker = blocking_push_worker(),+ Now = erlang:monotonic_time(millisecond),+ put(push_inflight_workers, [{2, Blocker, Now}, {1, Blocker, Now - 10}]),+ Before = read_push_counter(slot_deduped),+ try+ ?assertEqual([{2, Blocker, Now}], live_push_workers()),+ ?assertEqual(Before + 1, read_push_counter(slot_deduped))+ after+ stop_push_worker(Blocker),+ reset_push_worker_state()+ end.++stale_push_worker_slot_is_reclaimed_and_counted_test() ->+ reset_push_worker_state(),+ Stale = blocking_push_worker(),+ Aged = erlang:monotonic_time(millisecond) - ?PUSH_WORKER_MAX_AGE_MS - 1,+ put(push_inflight_workers, [{1, Stale, Aged}]),+ Before = read_push_counter(slot_reclaimed),+ try+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertEqual(Before + 1, read_push_counter(slot_reclaimed)),+ ?assertEqual(1, length(get(push_inflight_workers))),+ ?assertNot(lists:member(Stale, worker_pids(get(push_inflight_workers))))+ after+ stop_push_worker(Stale),+ reset_push_worker_state()+ end.++push_drop_is_readable_from_named_ets_table_test() ->+ reset_push_worker_state(),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1),+ Blocker = blocking_push_worker(),+ put(push_inflight_workers, [{1, Blocker, erlang:monotonic_time(millisecond)}]),+ Before = read_push_counter(dropped_at_limit),+ try+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertEqual(Before + 1, read_push_counter(dropped_at_limit)),+ ?assertMatch([{dropped_at_limit, _}], ets:lookup(?PUSH_COUNTERS, dropped_at_limit)),+ ?assertEqual(+ lists:sort(?PUSH_COUNTER_KEYS),+ lists:sort([Key || {Key, _} <- ets:tab2list(?PUSH_COUNTERS)])+ )+ after+ stop_push_worker(Blocker),+ reset_push_worker_state()+ end.++started_push_worker_is_counted_test() ->+ reset_push_worker_state(),+ Before = read_push_counter(worker_started),+ try+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertEqual(Before + 1, read_push_counter(worker_started)),+ ?assert(is_pid(get(push_inflight))),+ ?assertEqual(1, length(get(push_inflight_workers)))+ after+ reset_push_worker_state()+ end.++worker_completion_is_counted_test() ->+ reset_push_worker_state(),+ Before = read_push_counter(worker_completed),+ BeforeFailed = read_push_counter(worker_failed),+ try+ ?assertEqual(ok, run_counted_push_worker(fun() -> ok end)),+ ?assertEqual(Before + 1, read_push_counter(worker_completed)),+ ?assertEqual(BeforeFailed, read_push_counter(worker_failed))+ after+ reset_push_worker_state()+ end.++worker_crash_is_counted_as_failure_not_completion_test() ->+ reset_push_worker_state(),+ Before = read_push_counter(worker_failed),+ BeforeCompleted = read_push_counter(worker_completed),+ try+ ?assertEqual(ok, run_counted_push_worker(fun() -> error(boom) end)),+ ?assertEqual(Before + 1, read_push_counter(worker_failed)),+ ?assertEqual(BeforeCompleted, read_push_counter(worker_completed))+ after+ reset_push_worker_state()+ end.++spawned_worker_reports_started_and_completed_test() ->+ reset_push_worker_state(),+ Self = self(),+ Started = read_push_counter(worker_started),+ Completed = read_push_counter(worker_completed),+ try+ ?assertEqual(+ ok,+ spawn_push_worker(+ fun() ->+ Self ! {ran, self()},+ ok+ end,+ 7+ )+ ),+ Pid = get(push_inflight),+ receive+ {ran, Pid} -> ok+ after 1000 -> ?assert(false)+ end,+ wait_for_counter(worker_completed, Completed + 1),+ ?assertEqual(Started + 1, read_push_counter(worker_started)),+ ?assertEqual(Completed + 1, read_push_counter(worker_completed)),+ Messages = element(2, process_info(self(), messages)),+ ?assertEqual([], [M || {'DOWN', _, process, P, _} = M <- Messages, P =:= Pid])+ after+ reset_push_worker_state()+ end.++missing_members_table_is_counted_test() ->+ reset_push_worker_state(),+ Before = read_push_counter(members_table_missing),+ try+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertEqual(Before + 1, read_push_counter(members_table_missing))+ after+ reset_push_worker_state()+ end.++scan_table_unavailable_is_counted_test() ->+ reset_push_worker_state(),+ Tab = ets:new(test_members, [set, public]),+ State = compact_scan_state(Tab),+ true = ets:delete(Tab),+ Before = read_push_counter(scan_table_unavailable),+ try+ MessageData = #{<<"channel_id">> => <<"10">>},+ ?assertEqual(ok, send_compact_push_notifications(MessageData, 7, State)),+ ?assertEqual(Before + 1, read_push_counter(scan_table_unavailable))+ after+ reset_push_worker_state()+ end.++bounded_scan_leaves_the_members_table_unfixed_test() ->+ reset_push_worker_state(),+ Tab = ets:new(test_members, [set, public]),+ try+ true = ets:insert(Tab, {1, #{<<"roles">> => []}}),+ true = ets:insert(Tab, {2, #{<<"roles">> => []}}),+ State = compact_scan_state(Tab),+ Context = compact_scan_context(#{}, 10, State),+ ?assertEqual(false, ets:info(Tab, fixed)),+ {ok, Scan} = scan_push_members(Tab, Context, State),+ ?assertEqual([1], lists:sort(maps:get(eligible_user_ids, Scan))),+ ?assertEqual(false, ets:info(Tab, fixed)),+ ?assertEqual(false, ets:info(Tab, safe_fixed_monotonic_time))+ after+ ets:delete(Tab),+ reset_push_worker_state()+ end.++bounded_push_concurrency_spawns_second_worker_under_limit_test() ->+ reset_push_worker_state(),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 3),+ Blocker = blocking_push_worker(),+ put(push_inflight, Blocker),+ put(push_inflight_workers, [{1, Blocker, erlang:monotonic_time(millisecond)}]),+ try+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertNotEqual(Blocker, get(push_inflight)),+ ?assertEqual(2, length(get(push_inflight_workers))),+ ?assert(lists:member(Blocker, worker_pids(get(push_inflight_workers))))+ after+ stop_push_worker(Blocker),+ reset_push_worker_state()+ end.++bounded_push_concurrency_counts_drops_at_limit_test() ->+ reset_push_worker_state(),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 2),+ First = blocking_push_worker(),+ Second = blocking_push_worker(),+ Now = erlang:monotonic_time(millisecond),+ put(push_inflight, Second),+ put(push_inflight_workers, [{2, Second, Now}, {1, First, Now}]),+ Before = read_push_counter(dropped_at_limit),+ try+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertEqual(Second, get(push_inflight)),+ ?assertEqual([Second, First], worker_pids(get(push_inflight_workers))),+ ?assertEqual(Before + 1, read_push_counter(dropped_at_limit)),+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertEqual(Before + 2, read_push_counter(dropped_at_limit)),+ ?assertEqual([Second, First], worker_pids(get(push_inflight_workers)))+ after+ stop_push_worker(First),+ stop_push_worker(Second),+ reset_push_worker_state()+ end.++bounded_push_concurrency_reaps_finished_workers_test() ->+ reset_push_worker_state(),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1),+ Finished = blocking_push_worker(),+ stop_push_worker(Finished),+ put(push_inflight, Finished),+ put(push_inflight_workers, [{1, Finished, erlang:monotonic_time(millisecond)}]),+ try+ ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())),+ ?assertNot(lists:member(Finished, worker_pids(get(push_inflight_workers))))+ after+ reset_push_worker_state()+ end.++bounded_worker_list_stays_bounded_by_the_limit_test() ->+ reset_push_worker_state(),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 2),+ Blockers = [blocking_push_worker() || _ <- lists:seq(1, 6)],+ Now = erlang:monotonic_time(millisecond),+ put(push_inflight_workers, [+ {Gen, Pid, Now}+ || {Gen, Pid} <- lists:zip(lists:seq(1, 6), Blockers)+ ]),+ try+ lists:foreach(+ fun(_) -> ?assertEqual(ok, maybe_spawn_push(#{}, 7, legacy_test_state())) end,+ lists:seq(1, 20)+ ),+ ?assert(length(get(push_inflight_workers)) =< 6)+ after+ lists:foreach(fun stop_push_worker/1, Blockers),+ reset_push_worker_state()+ end.++push_concurrency_limit_defaults_and_clamps_test() ->+ reset_push_worker_state(),+ try+ ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 0),+ ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, not_a_number),+ ?assertEqual(?DEFAULT_PUSH_CONCURRENCY, push_concurrency_limit()),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 1000),+ ?assertEqual(?MAX_PUSH_CONCURRENCY, push_concurrency_limit()),+ ok = application:set_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY, 6),+ ?assertEqual(6, push_concurrency_limit())+ after+ reset_push_worker_state()+ end.++legacy_test_state() ->+ #{id => 7, data => #{}, sessions => #{}}.++worker_pids(Workers) ->+ [Pid || {_Gen, Pid, _StartedMs} <- Workers].++wait_for_counter(Key, Target) ->+ wait_for_counter(Key, Target, 200).++wait_for_counter(Key, Target, 0) ->+ ?assertEqual(Target, read_push_counter(Key));+wait_for_counter(Key, Target, Retries) ->+ case read_push_counter(Key) >= Target of+ true ->+ ok;+ false ->+ timer:sleep(5),+ wait_for_counter(Key, Target, Retries - 1)+ end.++remote_test_pid() ->+ binary_to_term(<<131, 88, 119, 12, "fake@nowhere", 1:32, 0:32, 1:32>>).++read_push_counter(Key) ->+ try ets:lookup(?PUSH_COUNTERS, Key) of+ [{Key, Value}] when is_integer(Value) -> Value;+ _ -> 0+ catch+ error:badarg -> 0+ end.++blocking_push_worker() ->+ spawn(fun() ->+ receive+ stop -> ok+ end+ end).++stop_push_worker(Pid) ->+ Ref = monitor(process, Pid),+ Pid ! stop,+ receive+ {'DOWN', Ref, process, Pid, _Reason} -> ok+ after 5000 -> error(push_worker_did_not_stop)+ end.++reset_push_worker_state() ->+ erase(push_inflight),+ erase(push_inflight_workers),+ ok = application:unset_env(fluxer_gateway, ?CONCURRENCY_LIMIT_KEY).++compact_test_data(Tab) ->+ #{+ <<"guild">> => #{+ <<"id">> => <<"7">>,+ <<"owner_id">> => <<"999">>,+ <<"features">> => [<<"COMMUNITY">>],+ <<"default_message_notifications">> => 0+ },+ <<"roles">> => [],+ <<"role_index">> => #{},+ <<"channels">> => [#{<<"id">> => <<"10">>, <<"name">> => <<"general">>}],+ <<"channel_index">> => #{10 => #{<<"id">> => <<"10">>, <<"name">> => <<"general">>}},+ <<"members">> => #{1 => #{}, 2 => #{}, 3 => #{}},+ members_normalized => #{1 => #{}, 2 => #{}, 3 => #{}},+ <<"member_role_index">> => #{},+ members_ets => Tab+ }.++compact_scan_state(Tab) ->+ #{+ id => 7,+ data => compact_push_data(compact_test_data(Tab)),+ virtual_channel_access => #{+ 1 => sets:from_list([10]),+ 3 => sets:from_list([10])+ },+ members_ets => Tab,+ session_eligibility => #{1 => true, 2 => true, 3 => false},+ member_count => 49435+ }.+-endif.diff --git a/fluxer_gateway/src/guild/guild_dispatch_send.erl b/fluxer_gateway/src/guild/guild_dispatch_send.erlindex c3b535513..bf3344ebc 100644--- a/fluxer_gateway/src/guild/guild_dispatch_send.erl+++ b/fluxer_gateway/src/guild/guild_dispatch_send.erl@@ -10,6 +10,8 @@filter_visible_channels/4]).+-define(MAX_BULK_ENCODE_GROUPS, 1024).+-type event() :: atom().-type event_data() :: map().-type guild_state() :: map().@@ -17,6 +19,9 @@-type guild_id() :: integer().-type user_id() :: integer().-type session_pair() :: {session_id(), map()}.+-type bulk_group_key() :: [term()].+-type bulk_groups() :: #{bulk_group_key() => {[map()], [pid()]}}.+-type bulk_acc() :: {bulk_groups(), non_neg_integer()}.-export_type([event/0, event_data/0, guild_state/0, user_id/0, session_pair/0]).-spec dispatch_to_sessions([session_pair()], event(), event_data(), guild_state()) ->@@ -45,44 +50,76 @@ dispatch_bulk_update(FilteredSessions, Event, FinalData, UpdatedState) ->}|| Ch <- BulkChannels],- SuccessCount = lists:foldl(+ {Groups, Dispatched} = lists:foldl(fun({_Sid, SessionData}, Acc) ->- dispatch_bulk_to_one_session_indexed(+ collect_bulk_recipient(SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc)end,- 0,+ {#{}, 0},FilteredSessions),+ SuccessCount = dispatch_bulk_groups(Groups, Event, FinalData, GuildId, Dispatched),normalize_success(SuccessCount).--spec dispatch_bulk_to_one_session_indexed(+-spec collect_bulk_recipient(map(),event(),event_data(),[{integer() | undefined, map()}],guild_id(),guild_state(),- non_neg_integer()-) -> non_neg_integer().-dispatch_bulk_to_one_session_indexed(- SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc-) ->+ bulk_acc()+) -> bulk_acc().+collect_bulk_recipient(SessionData, Event, FinalData, IndexedChannels, GuildId, State, Acc) ->Pid = maps:get(pid, SessionData),- case- session_passive:should_receive_event(- Event, FinalData, GuildId, SessionData, UpdatedState- )- of+ Eligible =+ is_pid(Pid) andalso+ session_passive:should_receive_event(Event, FinalData, GuildId, SessionData, State),+ case Eligible offalse ->Acc;true ->- FilteredChannels = filter_indexed_for_session(- SessionData, IndexedChannels, UpdatedState+ Filtered = filter_indexed_for_session(SessionData, IndexedChannels, State),+ add_bulk_recipient(Pid, Filtered, Event, FinalData, GuildId, Acc)+ end.++-spec add_bulk_recipient(pid(), [map()], event(), event_data(), guild_id(), bulk_acc()) ->+ bulk_acc().+add_bulk_recipient(_Pid, [], _Event, _FinalData, _GuildId, Acc) ->+ Acc;+add_bulk_recipient(Pid, FilteredChannels, Event, FinalData, GuildId, {Groups, Dispatched}) ->+ Key = bulk_group_key(FilteredChannels),+ case Groups of+ #{Key := {Channels, Pids}} ->+ {Groups#{Key := {Channels, [Pid | Pids]}}, Dispatched};+ _ when map_size(Groups) >= ?MAX_BULK_ENCODE_GROUPS ->+ Sent = dispatch_bulk_to_pid(+ Pid, Event, FinalData, FilteredChannels, GuildId, Dispatched),- dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc)+ {Groups, Sent};+ _ ->+ {Groups#{Key => {FilteredChannels, [Pid]}}, Dispatched}end.+-spec bulk_group_key([map()]) -> bulk_group_key().+bulk_group_key(FilteredChannels) ->+ [maps:get(<<"id">>, Ch, undefined) || Ch <- FilteredChannels].++-spec dispatch_bulk_groups(+ bulk_groups(), event(), event_data(), guild_id(), non_neg_integer()+) -> non_neg_integer().+dispatch_bulk_groups(Groups, Event, FinalData, GuildId, Dispatched) ->+ maps:fold(+ fun(_Key, {FilteredChannels, Pids}, Acc) ->+ dispatch_bulk_to_pids(+ lists:reverse(Pids), Event, FinalData, FilteredChannels, GuildId, Acc+ )+ end,+ Dispatched,+ Groups+ ).+-spec filter_indexed_for_session(map(), [{integer() | undefined, map()}], guild_state()) ->[map()].filter_indexed_for_session(SessionData, IndexedChannels, UpdatedState) ->@@ -128,12 +165,7 @@ is_channel_visible(Channel, UserId, Member, State) ->dispatch_bulk_to_pid(_, _, _, [], _GuildId, Acc) ->Acc;dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) when is_pid(Pid) ->- CustomData = FinalData#{<<"channels">> => FilteredChannels},- EncodedData =- {pre_encoded,- iolist_to_binary(- json:encode(guild_data_wire:payload(CustomData), fun json:encode_value/2)- )},+ EncodedData = encode_bulk_payload(FinalData, FilteredChannels),trygateway_dispatch_relay:dispatch(Pid, Event, EncodedData, GuildId),Acc + 1@@ -143,6 +175,28 @@ dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc) whendispatch_bulk_to_pid(_, _, _, _, _GuildId, Acc) ->Acc.+-spec dispatch_bulk_to_pids(+ [pid()], event(), event_data(), [map()], guild_id(), non_neg_integer()+) -> non_neg_integer().+dispatch_bulk_to_pids([], _Event, _FinalData, _FilteredChannels, _GuildId, Acc) ->+ Acc;+dispatch_bulk_to_pids(Pids, Event, FinalData, FilteredChannels, GuildId, Acc) ->+ EncodedData = encode_bulk_payload(FinalData, FilteredChannels),+ try+ gateway_dispatch_relay:dispatch_many(Pids, Event, EncodedData, GuildId),+ Acc + length(Pids)+ catch+ _:_ -> Acc+ end.++-spec encode_bulk_payload(event_data(), [map()]) -> {pre_encoded, binary()}.+encode_bulk_payload(FinalData, FilteredChannels) ->+ CustomData = FinalData#{<<"channels">> => FilteredChannels},+ {pre_encoded,+ iolist_to_binary(+ json:encode(guild_data_wire:payload(CustomData), fun json:encode_value/2)+ )}.+-spec dispatch_standard([session_pair()], event(), event_data(), guild_id(), guild_state()) ->non_neg_integer().dispatch_standard(FilteredSessions, Event, FinalData, GuildId, State) ->@@ -205,6 +259,8 @@ normalize_dispatched([_ | _]) -> 1.-ifdef(TEST).-include_lib("eunit/include/eunit.hrl").+-define(OVERFLOW_CHANNEL_BITS, 11).+normalize_success_test() ->?assertEqual(1, normalize_success(5)),?assertEqual(1, normalize_success(1)),@@ -287,6 +343,150 @@ passive_channel_update_bulk_dispatches_visible_channels_test() ->maps:get(<<"channels">>, Payload)).+grouped_bulk_encoding_matches_ungrouped_test() ->+ flush_dispatches(),+ Data = bulk_channel_data(),+ State = passive_dispatch_state(),+ Sessions = [bulk_session_pair(<<"a">>, #{100 => true})],+ ?assertEqual(1, reference_dispatch_bulk_update(Sessions, channel_update_bulk, Data, State)),+ Ungrouped = receive_pre_encoded_binary(channel_update_bulk),+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),+ ?assertEqual(Ungrouped, receive_pre_encoded_binary(channel_update_bulk)).++reference_dispatch_bulk_update(FilteredSessions, Event, FinalData, UpdatedState) ->+ GuildId = maps:get(id, UpdatedState),+ BulkChannels = maps:get(<<"channels">>, FinalData, []),+ IndexedChannels = [+ {+ guild_dispatch_decorate:parse_snowflake(+ <<"id">>,+ maps:get(<<"id">>, Ch, undefined)+ ),+ Ch+ }+ || Ch <- BulkChannels+ ],+ SuccessCount = lists:foldl(+ fun({_Sid, SessionData}, Acc) ->+ reference_dispatch_bulk_to_one_session_indexed(+ SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc+ )+ end,+ 0,+ FilteredSessions+ ),+ normalize_success(SuccessCount).++reference_dispatch_bulk_to_one_session_indexed(+ SessionData, Event, FinalData, IndexedChannels, GuildId, UpdatedState, Acc+) ->+ Pid = maps:get(pid, SessionData),+ case+ session_passive:should_receive_event(+ Event, FinalData, GuildId, SessionData, UpdatedState+ )+ of+ false ->+ Acc;+ true ->+ FilteredChannels = filter_indexed_for_session(+ SessionData, IndexedChannels, UpdatedState+ ),+ dispatch_bulk_to_pid(Pid, Event, FinalData, FilteredChannels, GuildId, Acc)+ end.++grouped_bulk_encodes_once_per_visible_set_test() ->+ flush_dispatches(),+ Data = bulk_channel_data(),+ State = passive_dispatch_state(),+ Sessions = [+ bulk_session_pair(<<"a">>, #{100 => true}),+ bulk_session_pair(<<"b">>, #{100 => true}),+ bulk_session_pair(<<"c">>, #{200 => true})+ ],+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),+ Bins = [receive_pre_encoded_binary(channel_update_bulk) || _ <- lists:seq(1, 3)],+ ?assertEqual(2, length(lists:usort(Bins))),+ Sets = [maps:get(<<"channels">>, json:decode(Bin)) || Bin <- Bins],+ ?assertEqual(2, length([S || S <- Sets, S =:= [bulk_channel(<<"100">>, <<"first">>)]])),+ ?assertEqual(1, length([S || S <- Sets, S =:= [bulk_channel(<<"200">>, <<"second">>)]])).++grouped_bulk_skips_sessions_without_visible_channels_test() ->+ flush_dispatches(),+ Data = bulk_channel_data(),+ State = passive_dispatch_state(),+ Sessions = [+ bulk_session_pair(<<"a">>, #{}),+ bulk_session_pair(<<"b">>, #{200 => true})+ ],+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),+ ?assertEqual(+ [bulk_channel(<<"200">>, <<"second">>)],+ maps:get(<<"channels">>, receive_pre_encoded_payload(channel_update_bulk))+ ),+ ?assertEqual(ok, assert_no_further_dispatch()).++grouped_bulk_dispatches_overflow_recipients_without_retaining_them_test() ->+ flush_dispatches(),+ Total = ?MAX_BULK_ENCODE_GROUPS + 1,+ State = passive_dispatch_state(),+ Data = #{+ <<"guild_id">> => <<"42">>,+ <<"channels">> => [+ bulk_channel(integer_to_binary(Id), <<"c">>)+ || Id <- lists:seq(1, ?OVERFLOW_CHANNEL_BITS)+ ]+ },+ Sessions = [overflow_session_pair(N) || N <- lists:seq(1, Total)],+ ?assertEqual(1, dispatch_to_sessions(Sessions, channel_update_bulk, Data, State)),+ Received = collect_bulk_channel_ids(Total, []),+ Expected = [overflow_channel_ids(N) || N <- lists:seq(1, Total)],+ ?assertEqual(lists:sort(Expected), lists:sort(Received)),+ ?assertEqual(ok, assert_no_further_dispatch()).++overflow_session_pair(N) ->+ Base = passive_session_data(),+ Sid = integer_to_binary(N),+ Viewable = maps:from_list([{Id, true} || Id <- overflow_visible_ids(N)]),+ {Sid, Base#{session_id => Sid, viewable_channels => Viewable}}.++overflow_visible_ids(N) ->+ [Id || Id <- lists:seq(1, ?OVERFLOW_CHANNEL_BITS), (N bsr (Id - 1)) band 1 =:= 1].++overflow_channel_ids(N) ->+ [integer_to_binary(Id) || Id <- overflow_visible_ids(N)].++collect_bulk_channel_ids(0, Acc) ->+ Acc;+collect_bulk_channel_ids(N, Acc) ->+ Payload = receive_pre_encoded_payload(channel_update_bulk),+ Ids = [maps:get(<<"id">>, Ch) || Ch <- maps:get(<<"channels">>, Payload)],+ collect_bulk_channel_ids(N - 1, [Ids | Acc]).++bulk_channel_data() ->+ #{+ <<"guild_id">> => <<"42">>,+ <<"channels">> => [+ bulk_channel(<<"100">>, <<"first">>),+ bulk_channel(<<"200">>, <<"second">>)+ ]+ }.++bulk_channel(Id, Name) ->+ #{<<"id">> => Id, <<"name">> => Name}.++bulk_session_pair(Sid, ViewableChannels) ->+ Base = passive_session_data(),+ {Sid, Base#{session_id => Sid, viewable_channels => ViewableChannels}}.++assert_no_further_dispatch() ->+ receive+ {'$gen_cast', {dispatch, Event, _Payload}} ->+ ?assert(false, {unexpected_dispatch, Event})+ after 0 ->+ ok+ end.+standard_dispatch_without_eligible_sessions_test() ->flush_dispatches(),Session = (passive_session_data())#{pid => undefined},@@ -364,9 +564,12 @@ passive_dispatch_state() ->}.receive_pre_encoded_payload(Event) ->+ json:decode(receive_pre_encoded_binary(Event)).++receive_pre_encoded_binary(Event) ->receive{'$gen_cast', {dispatch, Event, {pre_encoded, Bin}}} ->- json:decode(Bin)+ Binafter 1000 ->?assert(false, {dispatch_not_received, Event})end.diff --git a/fluxer_gateway/src/guild/guild_init.erl b/fluxer_gateway/src/guild/guild_init.erlindex d1612c538..6640624ec 100644--- a/fluxer_gateway/src/guild/guild_init.erl+++ b/fluxer_gateway/src/guild/guild_init.erl@@ -13,7 +13,11 @@handle_reload/2]).+-define(MAX_RELOAD_CARRY_WARN_MEMBERS, 100000).+-type guild_state() :: map().+-type user_id() :: integer().+-type member() :: map().-export_type([guild_state/0]).@@ -122,7 +126,8 @@ handle_reload(NewData, State) ->{ReloadVoiceStates, ReloadData} = extract_voice_states_from_data(NewData, ExistingVoiceStates),- NormalizedNewData = guild_data_index:normalize_map(ReloadData),+ NormalizedNewData0 = guild_data_index:normalize_map(ReloadData),+ NormalizedNewData = carry_members_table(OldData, NormalizedNewData0),NewState0 = State#{voice_states => ReloadVoiceStates, data => NormalizedNewData},NewState1 = guild_availability:handle_unavailability_transition(State, NewState0),NewState2 = guild_sessions:refresh_all_viewable_channels(NewState1),@@ -139,6 +144,89 @@ handle_reload(NewData, State) ->ok = guild_maintenance:maybe_put_permission_cache(NewState),{reply, ok, NewState}.+-spec carry_members_table(term(), term()) -> term().+carry_members_table(OldData, NewData) when is_map(OldData), is_map(NewData) ->+ carry_healthy_members_table(OldData, NewData);+carry_members_table(_OldData, NewData) ->+ NewData.++-spec carry_healthy_members_table(map(), map()) -> map().+carry_healthy_members_table(OldData, NewData) ->+ case members_ets_table(OldData) of+ Tab when is_reference(Tab) -> carry_live_members_table(Tab, OldData, NewData);+ undefined -> maps:remove(members_ets, NewData)+ end.++-spec members_ets_table(map()) -> ets:tid() | undefined.+members_ets_table(#{members_ets := Tab}) ->+ Tab;+members_ets_table(_Data) ->+ undefined.++-spec carry_live_members_table(ets:tid(), map(), map()) -> map().+carry_live_members_table(Tab, OldData, NewData) ->+ case guild_members_table_repair:members_table_healthy(Tab) of+ true -> apply_members_table_delta(Tab, OldData, NewData);+ false -> maps:remove(members_ets, NewData)+ end.++-spec apply_members_table_delta(ets:tid(), map(), map()) -> map().+apply_members_table_delta(Tab, OldData, NewData) ->+ OldMap = guild_data_index_members:member_map(OldData),+ NewMap = guild_data_index_members:member_map(NewData),+ maybe_warn_large_carry(map_size(NewMap)),+ try+ ok = insert_changed_members(Tab, OldMap, NewMap),+ ok = delete_removed_members(Tab, OldMap, NewMap),+ NewData#{members_ets => Tab}+ catch+ error:badarg ->+ logger:warning(+ "guild_reload_members_table_carry_failed: members=~p", [map_size(NewMap)]+ ),+ maps:remove(members_ets, NewData)+ end.++-spec insert_changed_members(ets:tid(), map(), map()) -> ok.+insert_changed_members(Tab, OldMap, NewMap) ->+ maps:foreach(+ fun(UserId, Member) -> insert_changed_member(Tab, OldMap, UserId, Member) end,+ NewMap+ ).++-spec insert_changed_member(ets:tid(), map(), user_id(), member()) -> ok.+insert_changed_member(Tab, OldMap, UserId, Member) ->+ case maps:get(UserId, OldMap, undefined) of+ Member -> ok;+ _ -> insert_member_row(Tab, UserId, Member)+ end.++-spec insert_member_row(ets:tid(), user_id(), member()) -> ok.+insert_member_row(Tab, UserId, Member) ->+ true = ets:insert(Tab, {UserId, Member}),+ ok.++-spec delete_removed_members(ets:tid(), map(), map()) -> ok.+delete_removed_members(Tab, OldMap, NewMap) ->+ OldIds = sets:from_list(maps:keys(OldMap)),+ NewIds = sets:from_list(maps:keys(NewMap)),+ RemovedIds = sets:to_list(sets:subtract(OldIds, NewIds)),+ lists:foreach(fun(UserId) -> delete_member_row(Tab, UserId) end, RemovedIds).++-spec delete_member_row(ets:tid(), user_id()) -> ok.+delete_member_row(Tab, UserId) ->+ true = ets:delete(Tab, UserId),+ ok.++-spec maybe_warn_large_carry(non_neg_integer()) -> ok.+maybe_warn_large_carry(Size) when Size > ?MAX_RELOAD_CARRY_WARN_MEMBERS ->+ logger:warning(+ "guild_reload_members_table_carry_large: members=~p threshold=~p",+ [Size, ?MAX_RELOAD_CARRY_WARN_MEMBERS]+ );+maybe_warn_large_carry(_Size) ->+ ok.+-spec collect_active_pid(term(), map(), [pid()]) -> [pid()].collect_active_pid(_Sid, S, Acc) ->case maps:get(pending_connect, S, false) of@@ -168,3 +256,92 @@ maybe_index_voice_state(_, Acc) ->-spec guild_id(guild_state()) -> integer() | undefined.guild_id(State) ->snowflake_id:parse_optional(maps:get(id, State, undefined)).++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++carry_members_table_carries_healthy_tid_test() ->+ Tab = ets:new(carry_members, [set, public]),+ try+ OldMap = #{1 => carry_member(1), 2 => carry_member(2)},+ NewMap = #{1 => carry_member(1), 3 => carry_member(3)},+ seed_carry_table(Tab, OldMap),+ OldData = carry_data(OldMap, Tab),+ Carried = carry_members_table(OldData, carry_data(NewMap, undefined)),+ ?assertEqual(Tab, maps:get(members_ets, Carried)),+ ?assertEqual([1, 3], carry_table_ids(Tab))+ after+ ets:delete(Tab)+ end.++carry_members_table_skips_dead_tid_test() ->+ Tab = ets:new(dead_carry_members, [set, public]),+ true = ets:delete(Tab),+ OldData = carry_data(#{1 => carry_member(1)}, Tab),+ NewData = carry_data(#{1 => carry_member(1)}, undefined),+ ?assertEqual(NewData, carry_members_table(OldData, NewData)).++carry_members_table_without_old_tid_leaves_new_data_test() ->+ OldData = carry_data(#{1 => carry_member(1)}, undefined),+ NewData = carry_data(#{2 => carry_member(2)}, undefined),+ ?assertEqual(NewData, carry_members_table(OldData, NewData)).++carry_members_table_delta_failure_falls_back_to_new_data_test() ->+ Tab = ets:new(failing_members, [set, public]),+ true = ets:delete(Tab),+ OldData = carry_data(#{}, undefined),+ NewData = carry_data(#{1 => carry_member(1)}, undefined),+ ?assertEqual(NewData, apply_members_table_delta(Tab, OldData, NewData)).++carry_delta_never_empties_table_test() ->+ Tab = ets:new(delta_members, [set, public]),+ try+ OldMap = #{1 => carry_member(1), 2 => carry_member(2), 3 => carry_member(3)},+ NewMap = #{+ 1 => carry_member(1),+ 2 => carry_member(2, <<"changed">>),+ 4 => carry_member(4)+ },+ seed_carry_table(Tab, OldMap),+ ?assertEqual(3, ets:info(Tab, size)),+ ok = insert_changed_members(Tab, OldMap, NewMap),+ ?assertEqual(4, ets:info(Tab, size)),+ ?assertEqual([{1, carry_member(1)}], ets:lookup(Tab, 1)),+ ok = delete_removed_members(Tab, OldMap, NewMap),+ ?assertEqual(3, ets:info(Tab, size)),+ ?assertEqual([{1, carry_member(1)}], ets:lookup(Tab, 1)),+ ?assertEqual([1, 2, 4], carry_table_ids(Tab)),+ ?assertEqual([{2, carry_member(2, <<"changed">>)}], ets:lookup(Tab, 2))+ after+ ets:delete(Tab)+ end.++carry_delta_skips_unchanged_members_test() ->+ Tab = ets:new(unchanged_members, [set, public]),+ try+ MemberMap = #{1 => carry_member(1)},+ true = ets:insert(Tab, {1, sentinel}),+ ok = insert_changed_members(Tab, MemberMap, MemberMap),+ ?assertEqual([{1, sentinel}], ets:tab2list(Tab))+ after+ ets:delete(Tab)+ end.++carry_data(MemberMap, undefined) ->+ #{<<"members">> => MemberMap, members_normalized => MemberMap};+carry_data(MemberMap, Tab) ->+ #{<<"members">> => MemberMap, members_normalized => MemberMap, members_ets => Tab}.++seed_carry_table(Tab, MemberMap) ->+ maps:foreach(fun(UserId, Member) -> ets:insert(Tab, {UserId, Member}) end, MemberMap).++carry_member(UserId) ->+ #{<<"user">> => #{<<"id">> => UserId}}.++carry_table_ids(Tab) ->+ lists:sort([Id || {Id, _} <- ets:tab2list(Tab)]).++carry_member(UserId, Nick) ->+ #{<<"user">> => #{<<"id">> => UserId}, <<"nick">> => Nick}.++-endif.diff --git a/fluxer_gateway/src/guild/guild_maintenance.erl b/fluxer_gateway/src/guild/guild_maintenance.erlindex e654c44e0..67e36d156 100644--- a/fluxer_gateway/src/guild/guild_maintenance.erl+++ b/fluxer_gateway/src/guild/guild_maintenance.erl@@ -22,15 +22,18 @@-type session_id() :: binary().-type subscription_state() :: guild_subscriptions:subscription_state().-type guild_state() :: map().+-type viewable_memo() :: #{exceptions := sets:set(user_id()), cache := #{term() => map()}}.+-type prune_acc() :: {subscription_state(), #{user_id() => non_neg_integer()}, viewable_memo()}.-spec update_counts(guild_state()) -> guild_state().update_counts(State) ->- Data = maps:get(data, State, #{}),+ State0 = guild_members_table_repair:maybe_repair(State),+ Data = maps:get(data, State0, #{}),MemberCount = guild_data_index:member_count(Data),- OnlineCount = guild_member_list:get_online_count(State),- PublicOnlineCount = guild_public_online:compute_count(State),- ok = maybe_put_guild_count_cache(State, MemberCount, PublicOnlineCount),- State#{+ OnlineCount = guild_member_list:get_online_count(State0),+ PublicOnlineCount = guild_public_online:compute_count(State0),+ ok = maybe_put_guild_count_cache(State0, MemberCount, PublicOnlineCount),+ State0#{member_count => MemberCount,online_count => OnlineCount,public_online_count => PublicOnlineCount@@ -160,30 +163,36 @@ require_subscription_state(MemberSubs) when is_map(MemberSubs) ->-spec prune_member_subscription_map(subscription_state(), map(), guild_state()) ->{subscription_state(), #{user_id() => non_neg_integer()}}.+prune_member_subscription_map(MemberSubs, _Sessions, _State) when map_size(MemberSubs) =:= 0 ->+ {MemberSubs, #{}};prune_member_subscription_map(MemberSubs, Sessions, State) ->- maps:fold(+ {NewMemberSubs, Counts, _Memo} = maps:fold(fun(MemberId, Subscribers, Acc) ->prune_member_subscribers(MemberId, Subscribers, Sessions, State, Acc)end,- {MemberSubs, #{}},+ {MemberSubs, #{}, new_viewable_memo(State)},MemberSubs- ).+ ),+ {NewMemberSubs, Counts}.-spec prune_member_subscribers(term(),sets:set(session_id()),map(),guild_state(),- {subscription_state(), #{user_id() => non_neg_integer()}}-) -> {subscription_state(), #{user_id() => non_neg_integer()}}.-prune_member_subscribers(MemberId, Subscribers, Sessions, State, Acc) when+ prune_acc()+) -> prune_acc().+prune_member_subscribers(MemberId, Subscribers, Sessions, State, {Subs, Counts, Memo}) whenis_integer(MemberId)->- MemberViewable = member_viewable_channel_map(MemberId, State),+ {MemberViewable, Memo1} = memoised_member_viewable_channel_map(MemberId, State, Memo),{KeptSubscribers, RemovedCount} = prune_subscriber_set(Subscribers, Sessions, MemberViewable, State),- update_pruned_member_subscription(MemberId, KeptSubscribers, RemovedCount, Acc);+ {NewSubs, NewCounts} = update_pruned_member_subscription(+ MemberId, KeptSubscribers, RemovedCount, {Subs, Counts}+ ),+ {NewSubs, NewCounts, Memo1};prune_member_subscribers(_MemberId, _Subscribers, _Sessions, _State, Acc) ->Acc.@@ -204,24 +213,152 @@ member_viewable_channel_map(MemberId, State) ->guild_visibility:get_user_viewable_channels(MemberId, State)).+%% One pass reads a fixed state, and get_user_viewable_channels/2 reads nothing from a+%% member except its raw roles term, so two members holding the same term see the same+%% channels. The user-specific inputs are the guild owner, virtual channel access and+%% user-type permission overwrites, and every user named by one of those bypasses the+%% memo. A term is still computed the first time the pass meets it, so a term whose+%% computation raises raises on the same member it does without the memo.+-spec new_viewable_memo(guild_state()) -> viewable_memo().+new_viewable_memo(State) ->+ #{exceptions => viewable_exceptions(State), cache => #{}}.++-spec memoised_member_viewable_channel_map(user_id(), guild_state(), viewable_memo()) ->+ {map(), viewable_memo()}.+memoised_member_viewable_channel_map(MemberId, State, Memo) ->+ #{exceptions := Exceptions, cache := Cache} = Memo,+ case memo_key(MemberId, Exceptions, State) of+ bypass ->+ {member_viewable_channel_map(MemberId, State), Memo};+ {ok, RawRoles} ->+ cached_member_viewable_channel_map(MemberId, RawRoles, State, Cache, Memo)+ end.++-spec cached_member_viewable_channel_map(+ user_id(), term(), guild_state(), #{term() => map()}, viewable_memo()+) -> {map(), viewable_memo()}.+cached_member_viewable_channel_map(MemberId, RawRoles, State, Cache, Memo) ->+ case maps:find(RawRoles, Cache) of+ {ok, Viewable} ->+ {Viewable, Memo};+ error ->+ Viewable = member_viewable_channel_map(MemberId, State),+ {Viewable, Memo#{cache := Cache#{RawRoles => Viewable}}}+ end.++-spec memo_key(user_id(), sets:set(user_id()), guild_state()) -> {ok, term()} | bypass.+memo_key(MemberId, Exceptions, State) ->+ case sets:is_element(MemberId, Exceptions) of+ true -> bypass;+ false -> member_roles_key(guild_permissions:find_member_by_user_id(MemberId, State))+ end.++-spec member_roles_key(map() | undefined) -> {ok, term()} | bypass.+member_roles_key(Member) when is_map(Member) ->+ {ok, maps:get(<<"roles">>, Member, [])};+member_roles_key(_Member) ->+ bypass.++-spec viewable_exceptions(guild_state()) -> sets:set(user_id()).+viewable_exceptions(State) ->+ Data = map_utils:ensure_map(map_utils:get_safe(State, data, #{})),+ Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})),+ Owner = owner_ids(maps:get(<<"owner_id">>, Guild, undefined)),+ Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))),+ sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(Data)).++%% parse_maybe/1, not parse_optional/1: an id this rejects makes the permission check+%% itself raise before it can match a user, so leaving it out keeps the memo exact.+-spec owner_ids(term()) -> [user_id()].+owner_ids(OwnerIdRaw) ->+ case snowflake_id:parse_maybe(OwnerIdRaw) of+ OwnerId when is_integer(OwnerId) -> [OwnerId];+ _ -> []+ end.++%% Both places a permission check can take overwrites from, and role targets alongside+%% user targets. Role ids can never collide with user ids, so the superset only costs a+%% few members their memo entry.+-spec overwrite_target_ids(map()) -> [user_id()].+overwrite_target_ids(Data) ->+ Cached = maps:fold(+ fun(_ChannelId, Entries, Acc) -> cached_target_ids(Entries, Acc) end,+ [],+ map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{}))+ ),+ maps:fold(+ fun(_ChannelId, Channel, Acc) -> channel_target_ids(Channel, Acc) end,+ Cached,+ guild_data_index:channel_index(Data)+ ).++-spec cached_target_ids(term(), [user_id()]) -> [user_id()].+cached_target_ids(Entries, Acc) when is_list(Entries) ->+ [Id || {Id, _Type, _Allow, _Deny} <- Entries, is_integer(Id)] ++ Acc;+cached_target_ids(_Entries, Acc) ->+ Acc.++-spec channel_target_ids(term(), [user_id()]) -> [user_id()].+channel_target_ids(Channel, Acc) when is_map(Channel) ->+ lists:foldl(+ fun overwrite_target_id/2,+ Acc,+ map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, []))+ );+channel_target_ids(_Channel, Acc) ->+ Acc.++-spec overwrite_target_id(term(), [user_id()]) -> [user_id()].+overwrite_target_id(Overwrite, Acc) when is_map(Overwrite) ->+ case snowflake_id:parse_maybe(maps:get(<<"id">>, Overwrite, undefined)) of+ Id when is_integer(Id) -> [Id | Acc];+ _ -> Acc+ end;+overwrite_target_id(_Overwrite, Acc) ->+ Acc.++%% update_pruned_member_subscription/4 drops the whole result when nothing was removed,+%% so the kept set is only ever built when Removed is non-empty, and then by the same+%% fold over the same subscribers as before.-spec prune_subscriber_set(sets:set(session_id()), map(), map(), guild_state()) ->{sets:set(session_id()), non_neg_integer()}.prune_subscriber_set(Subscribers, Sessions, MemberViewable, State) ->- sets:fold(+ Removed = sets:fold(fun(SessionId, Acc) ->- prune_subscriber(SessionId, Sessions, MemberViewable, State, Acc)+ collect_removed_subscriber(SessionId, Sessions, MemberViewable, State, Acc)end,- {sets:new(), 0},+ #{},Subscribers- ).+ ),+ keep_remaining_subscribers(Subscribers, Removed).--spec prune_subscriber(- session_id(), map(), map(), guild_state(), {sets:set(session_id()), non_neg_integer()}-) -> {sets:set(session_id()), non_neg_integer()}.-prune_subscriber(SessionId, Sessions, MemberViewable, State, {Kept, RemovedCount}) ->+-spec collect_removed_subscriber(+ session_id(), map(), map(), guild_state(), #{session_id() => true}+) -> #{session_id() => true}.+collect_removed_subscriber(SessionId, Sessions, MemberViewable, State, Removed) ->case subscriber_can_still_view_member(SessionId, Sessions, MemberViewable, State) of- true -> {sets:add_element(SessionId, Kept), RemovedCount};- false -> {Kept, RemovedCount + 1}+ true -> Removed;+ false -> Removed#{SessionId => true}+ end.++-spec keep_remaining_subscribers(sets:set(session_id()), #{session_id() => true}) ->+ {sets:set(session_id()), non_neg_integer()}.+keep_remaining_subscribers(Subscribers, Removed) when map_size(Removed) =:= 0 ->+ {Subscribers, 0};+keep_remaining_subscribers(Subscribers, Removed) ->+ Kept = sets:fold(+ fun(SessionId, Acc) -> add_kept_subscriber(SessionId, Removed, Acc) end,+ sets:new(),+ Subscribers+ ),+ {Kept, map_size(Removed)}.++-spec add_kept_subscriber(session_id(), #{session_id() => true}, sets:set(session_id())) ->+ sets:set(session_id()).+add_kept_subscriber(SessionId, Removed, Kept) ->+ case maps:is_key(SessionId, Removed) of+ true -> Kept;+ false -> sets:add_element(SessionId, Kept)end.-spec subscriber_can_still_view_member(session_id(), map(), map(), guild_state()) ->@@ -399,8 +536,8 @@ prune_test_state() ->30 => prune_member(30, [OtherRole])},<<"channels">> => [- prune_channel(ChannelA, ViewerRole),- prune_channel(ChannelB, OtherRole)+ prune_channel(ChannelA, [prune_overwrite(ViewerRole, 0)]),+ prune_channel(ChannelB, [prune_overwrite(OtherRole, 0)])]}}.@@ -417,20 +554,186 @@ prune_member(UserId, RoleIds) -><<"roles">> => [integer_to_binary(RoleId) || RoleId <- RoleIds]}.-prune_channel(ChannelId, RoleId) ->- ViewPerm = constants:view_channel_permission(),+update_counts_repairs_missing_members_table_test() ->+ ok = application:set_env(fluxer_gateway, guild_members_table_self_heal, true),+ try+ Result = update_counts(repair_counts_state()),+ Tab = maps:get(members_ets, maps:get(data, Result)),+ ?assert(is_reference(Tab)),+ ?assertNot(maps:is_key(members_ets_repair, Result)),+ ?assertEqual([1, 2], lists:sort([Id || {Id, _} <- ets:tab2list(Tab)])),+ ?assertEqual(2, maps:get(member_count, Result)),+ ets:delete(Tab)+ after+ application:unset_env(fluxer_gateway, guild_members_table_self_heal)+ end.++repair_counts_state() ->+ MemberMap = #{+ 1 => #{<<"user">> => #{<<"id">> => 1}},+ 2 => #{<<"user">> => #{<<"id">> => 2}}+ },+ #{+ id => 42,+ disable_guild_count_cache_updates => true,+ data => #{+ <<"channels">> => [],+ <<"members">> => MemberMap,+ members_normalized => MemberMap+ }+ }.++prune_channel(ChannelId, Overwrites) ->#{<<"id">> => integer_to_binary(ChannelId),- <<"permission_overwrites">> => [- #{- <<"id">> => integer_to_binary(RoleId),- <<"type">> => 0,- <<"allow">> => integer_to_binary(ViewPerm),- <<"deny">> => <<"0">>- }- ]+ <<"permission_overwrites">> => Overwrites+ }.++prune_overwrite(TargetId, Type) ->+ #{+ <<"id">> => integer_to_binary(TargetId),+ <<"type">> => Type,+ <<"allow">> => integer_to_binary(constants:view_channel_permission()),+ <<"deny">> => <<"0">>+ }.++%% Members 30, 31, 99 and 4242 all carry the same roles term and reach only channel 600+%% through it, but 99 holds virtual access to channel 500 and 4242 is a user-overwrite+%% target on 500, so neither may be answered from that term. Member 7 owns the guild,+%% 777 is subscribed without being a member, and session s3 has no cached viewable map.+memo_prune_state() ->+ GuildId = 42,+ ViewerRole = 1000,+ OtherRole = 2000,+ Members = maps:from_list(+ [{7, prune_member(7, [])}] +++ [{Id, prune_member(Id, [ViewerRole])} || Id <- [10, 20, 21]] +++ [{Id, prune_member(Id, [OtherRole])} || Id <- [30, 31, 99, 4242]]+ ),+ #{+ id => GuildId,+ virtual_channel_access => #{99 => sets:from_list([500])},+ sessions => #{+ <<"s1">> => #{user_id => 10, viewable_channels => #{500 => true}},+ <<"s2">> => #{user_id => 30, viewable_channels => #{600 => true}},+ <<"s3">> => #{user_id => 10}+ },+ member_subscriptions => memo_prune_subscriptions(),+ presence_subscriptions => maps:from_list(+ [{Id, 3} || Id <- [7, 20, 21, 30, 31, 99, 4242, 777]]+ ),+ data => #{+ <<"guild">> => #{<<"owner_id">> => <<"7">>},+ <<"roles">> => [+ prune_role(GuildId, 0),+ prune_role(ViewerRole, 0),+ prune_role(OtherRole, 0)+ ],+ <<"members">> => Members,+ <<"channels">> => [+ prune_channel(500, [prune_overwrite(ViewerRole, 0), prune_overwrite(4242, 1)]),+ prune_channel(600, [prune_overwrite(OtherRole, 0)])+ ]+ }}.+memo_prune_subscriptions() ->+ lists:foldl(+ fun({SessionId, MemberIds}, Acc) ->+ guild_subscriptions:update_subscriptions(SessionId, MemberIds, Acc)+ end,+ guild_subscriptions:init_state(),+ [+ {<<"s1">>, [20, 21, 30, 31, 7, 99, 4242, 777]},+ {<<"s2">>, [20, 30]},+ {<<"s3">>, [20, 30]},+ {<<"missing">>, [31]}+ ]+ ).++%% prune_member_subscription_map/3 as it read before the memo: every subscribed member+%% materialises its own viewable channel map and every subscriber set is rebuilt.+reference_prune_member_subscription_map(MemberSubs, Sessions, State) ->+ maps:fold(+ fun(MemberId, Subscribers, Acc) ->+ reference_prune_member(MemberId, Subscribers, Sessions, State, Acc)+ end,+ {MemberSubs, #{}},+ MemberSubs+ ).++reference_prune_member(MemberId, Subscribers, Sessions, State, Acc) when is_integer(MemberId) ->+ MemberViewable = member_viewable_channel_map(MemberId, State),+ {Kept, RemovedCount} = reference_prune_subscriber_set(+ Subscribers, Sessions, MemberViewable, State+ ),+ update_pruned_member_subscription(MemberId, Kept, RemovedCount, Acc);+reference_prune_member(_MemberId, _Subscribers, _Sessions, _State, Acc) ->+ Acc.++reference_prune_subscriber_set(Subscribers, Sessions, MemberViewable, State) ->+ sets:fold(+ fun(SessionId, Acc) ->+ reference_prune_subscriber(SessionId, Sessions, MemberViewable, State, Acc)+ end,+ {sets:new(), 0},+ Subscribers+ ).++reference_prune_subscriber(SessionId, Sessions, MemberViewable, State, {Kept, RemovedCount}) ->+ case subscriber_can_still_view_member(SessionId, Sessions, MemberViewable, State) of+ true -> {sets:add_element(SessionId, Kept), RemovedCount};+ false -> {Kept, RemovedCount + 1}+ end.++prune_member_subscription_map_matches_unmemoised_reference_test() ->+ State = memo_prune_state(),+ MemberSubs = member_subscriptions(State),+ Sessions = maps:get(sessions, State),+ ?assertEqual(+ reference_prune_member_subscription_map(MemberSubs, Sessions, State),+ prune_member_subscription_map(MemberSubs, Sessions, State)+ ).++prune_keeps_exception_members_a_session_can_still_see_test() ->+ Result = maybe_prune_invalid_member_subscriptions(guild_role_update, memo_prune_state()),+ MemberSubs = maps:get(member_subscriptions, Result),+ Presence = #{7 => 3, 20 => 2, 21 => 3, 30 => 1, 31 => 1, 99 => 3, 4242 => 3, 777 => 2},+ ?assertEqual(+ [[<<"s1">>], [<<"s1">>], [<<"s1">>], [], []],+ [+ lists:sort(guild_subscriptions:get_subscribed_sessions(Id, MemberSubs))+ || Id <- [7, 99, 4242, 31, 777]+ ]+ ),+ ?assertEqual(Presence, maps:get(presence_subscriptions, Result)).++memo_reuses_one_entry_per_roles_term_test() ->+ State = memo_prune_state(),+ {First, Memo1} = memoised_member_viewable_channel_map(20, State, new_viewable_memo(State)),+ {Second, Memo2} = memoised_member_viewable_channel_map(21, State, Memo1),+ ?assertEqual(member_viewable_channel_map(21, State), Second),+ ?assertEqual(First, Second),+ ?assertEqual([1, 1], [map_size(maps:get(cache, M)) || M <- [Memo1, Memo2]]).++memo_never_answers_owner_virtual_or_overwrite_targets_from_the_cache_test() ->+ State = memo_prune_state(),+ {_, Memo} = memoised_member_viewable_channel_map(30, State, new_viewable_memo(State)),+ Answers = [memoised_member_viewable_channel_map(Id, State, Memo) || Id <- [7, 99, 4242]],+ ?assertEqual(+ [member_viewable_channel_map(Id, State) || Id <- [7, 99, 4242]],+ [Viewable || {Viewable, _} <- Answers]+ ),+ ?assertEqual([Memo, Memo, Memo], [M || {_, M} <- Answers]).++prune_subscriber_set_returns_the_original_set_when_nothing_is_removed_test() ->+ State = memo_prune_state(),+ Subscribers = sets:from_list([<<"s1">>, <<"s3">>]),+ Viewable = member_viewable_channel_map(20, State),+ ?assertEqual(+ {Subscribers, 0},+ prune_subscriber_set(Subscribers, maps:get(sessions, State), Viewable, State)+ ).permission_cache_skips_nick_only_member_update_test() ->UserId = 4242,OldState = permission_cache_state(diff --git a/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl b/fluxer_gateway/src/guild/guild_member_list_channel_engine.erlindex 8fb67b034..f18f3fec0 100644--- a/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl+++ b/fluxer_gateway/src/guild/guild_member_list_channel_engine.erl@@ -23,6 +23,8 @@-type guild_state() :: map().-type list_id() :: binary().+-type visibility_memo() :: disabled | #{exceptions := sets:set(integer()), cache := map()}.+-type role_cache() :: #{term() => {[integer()], [integer()]}}.-type engine_ref() :: ets:table().-export_type([guild_state/0, list_id/0, engine_ref/0]).@@ -244,13 +246,21 @@ build_inputs(ChannelId, State) ->MemberMap = guild_data_index:member_map(Data),ConnectedUserIds = guild_member_list_common:connected_session_user_ids(State),PresenceTab = maps:get(member_presence, State),- Tuples = maps:fold(- fun(UserId, Member, Acc) ->+ {Tuples, _Memo, _RoleCache} = maps:fold(+ fun(UserId, Member, {Acc, Memo, RoleCache}) ->maybe_prepare_visible_member_tuple(- UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc+ UserId,+ Member,+ ChannelId,+ State,+ PresenceTab,+ ConnectedUserIds,+ Acc,+ Memo,+ RoleCache)end,- [],+ {[], new_visibility_memo(ChannelId, Data, State), #{}},MemberMap),Roles = map_utils:ensure_list(maps:get(<<"roles">>, Data, [])),@@ -264,30 +274,165 @@ build_inputs(ChannelId, State) ->{Tuples, HoistedRoleIds}.-spec maybe_prepare_visible_member_tuple(- integer(), map(), pos_integer(), guild_state(), ets:tid() | map(), sets:set(integer()), [- guild_member_list_store:member_tuple()- ]-) -> [guild_member_list_store:member_tuple()].+ integer(),+ map(),+ pos_integer(),+ guild_state(),+ ets:tid() | map(),+ sets:set(integer()),+ [guild_member_list_store:member_tuple()],+ visibility_memo(),+ role_cache()+) -> {[guild_member_list_store:member_tuple()], visibility_memo(), role_cache()}.maybe_prepare_visible_member_tuple(- UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc+ UserId, Member, ChannelId, State, PresenceTab, ConnectedUserIds, Acc, Memo, RoleCache) ->- case can_view(UserId, ChannelId, Member, State) of- true -> prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, Acc);- false -> Acc+ {RoleIds, SortedRoleIds, RoleCache1} = member_role_ids(Member, RoleCache),+ {Visible, Memo1} = memoised_can_view(UserId, ChannelId, Member, SortedRoleIds, State, Memo),+ case Visible of+ true ->+ {+ prepare_member_tuple(+ UserId, Member, PresenceTab, ConnectedUserIds, RoleIds, Acc+ ),+ Memo1,+ RoleCache1+ };+ false ->+ {Acc, Memo1, RoleCache1}+ end.++%% extract_role_ids/1 and the sort that forms the visibility memo key are both+%% pure functions of the member's raw roles term, so caching on that term+%% collapses them to the number of distinct role combinations.+-spec member_role_ids(map(), role_cache()) -> {[integer()], [integer()], role_cache()}.+member_role_ids(Member, RoleCache) ->+ RawRoles = maps:get(<<"roles">>, Member, []),+ case maps:find(RawRoles, RoleCache) of+ {ok, {RoleIds, SortedRoleIds}} ->+ {RoleIds, SortedRoleIds, RoleCache};+ error ->+ RoleIds = guild_member_list_store:extract_role_ids(Member),+ SortedRoleIds = lists:sort(RoleIds),+ {RoleIds, SortedRoleIds, RoleCache#{RawRoles => {RoleIds, SortedRoleIds}}}end.--spec prepare_member_tuple(integer(), map(), ets:tid() | map(), sets:set(integer()), [- guild_member_list_store:member_tuple()-]) -> [guild_member_list_store:member_tuple()].-prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, Acc) ->+%% can_view_channel/4 reads nothing from the member except its role ids, so for a+%% non-category channel its result is a function of the role set. The only+%% user-specific inputs are the guild owner, virtual channel access, and the+%% channel's own permission overwrites, and those users bypass the memo.+-spec new_visibility_memo(pos_integer(), map(), guild_state()) -> visibility_memo().+new_visibility_memo(ChannelId, Data, State) ->+ case memoisable_channel(ChannelId, Data) of+ true ->+ #{+ exceptions => visibility_exceptions(ChannelId, Data, State),+ cache => #{}+ };+ false ->+ disabled+ end.++-spec memoised_can_view(+ integer(), pos_integer(), map(), [integer()], guild_state(), visibility_memo()+) -> {boolean(), visibility_memo()}.+memoised_can_view(UserId, ChannelId, Member, _SortedRoleIds, State, disabled) ->+ {can_view(UserId, ChannelId, Member, State), disabled};+memoised_can_view(UserId, ChannelId, Member, SortedRoleIds, State, Memo) ->+ #{exceptions := Exceptions, cache := Cache} = Memo,+ case sets:is_element(UserId, Exceptions) of+ true ->+ {can_view(UserId, ChannelId, Member, State), Memo};+ false ->+ case maps:find(SortedRoleIds, Cache) of+ {ok, Visible} ->+ {Visible, Memo};+ error ->+ Visible = can_view(UserId, ChannelId, Member, State),+ {Visible, Memo#{cache := Cache#{SortedRoleIds => Visible}}}+ end+ end.++%% Category visibility recurses into child channels, each with its own+%% overwrites, so the exception set is not local to this channel. Do not memoise.+-spec memoisable_channel(pos_integer(), map()) -> boolean().+memoisable_channel(ChannelId, Data) ->+ case maps:get(ChannelId, guild_data_index:channel_index(Data), undefined) of+ Channel when is_map(Channel) -> maps:get(<<"type">>, Channel, undefined) =/= 4;+ _ -> false+ end.++-spec visibility_exceptions(pos_integer(), map(), guild_state()) -> sets:set(integer()).+visibility_exceptions(ChannelId, Data, State) ->+ Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})),+ Owner =+ case snowflake_id:parse_optional(maps:get(<<"owner_id">>, Guild, undefined)) of+ OwnerId when is_integer(OwnerId) -> [OwnerId];+ _ -> []+ end,+ Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))),+ sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(ChannelId, Data)).++%% Every overwrite target on the channel, role and user alike. Role ids can never+%% collide with user ids, so the superset costs only a few direct computations.+-spec overwrite_target_ids(pos_integer(), map()) -> [integer()].+overwrite_target_ids(ChannelId, Data) ->+ Cache = map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{})),+ case maps:get(ChannelId, Cache, undefined) of+ Cached when is_list(Cached) ->+ [Id || {Id, _Type, _Allow, _Deny} <- Cached, is_integer(Id)];+ _ ->+ channel_overwrite_target_ids(ChannelId, Data)+ end.++-spec channel_overwrite_target_ids(pos_integer(), map()) -> [integer()].+channel_overwrite_target_ids(ChannelId, Data) ->+ case maps:get(ChannelId, guild_data_index:channel_index(Data), undefined) of+ Channel when is_map(Channel) ->+ lists:filtermap(+ fun(Overwrite) -> overwrite_target_id(Overwrite) end,+ map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, []))+ );+ _ ->+ []+ end.++-spec overwrite_target_id(term()) -> {true, integer()} | false.+overwrite_target_id(Overwrite) when is_map(Overwrite) ->+ case snowflake_id:parse_optional(maps:get(<<"id">>, Overwrite, undefined)) of+ Id when is_integer(Id) -> {true, Id};+ _ -> false+ end;+overwrite_target_id(_Overwrite) ->+ false.++-spec prepare_member_tuple(+ integer(),+ map(),+ ets:tid() | map(),+ sets:set(integer()),+ [integer()],+ [guild_member_list_store:member_tuple()]+) -> [guild_member_list_store:member_tuple()].+prepare_member_tuple(UserId, Member, PresenceTab, ConnectedUserIds, RoleIds, Acc) ->DisplayName = guild_member_list_common:get_member_display_name(Member),SortKey = guild_member_list_common:casefold_binary(DisplayName),- RoleIds = guild_member_list_store:extract_role_ids(Member),+ IsOnline = member_is_online(UserId, PresenceTab, ConnectedUserIds),+ [{UserId, SortKey, RoleIds, IsOnline} | Acc].++%% Presence is consulted only to decide IsOnline, and IsOnline is false for anyone+%% without a connected session, so the ETS read and its default-presence+%% allocation are skipped for every member who is not connected.+-spec member_is_online(integer(), ets:tid() | map(), sets:set(integer())) -> boolean().+member_is_online(UserId, PresenceTab, ConnectedUserIds) ->+ sets:is_element(UserId, ConnectedUserIds) andalso+ presence_status_is_online(UserId, PresenceTab).++-spec presence_status_is_online(integer(), ets:tid() | map()) -> boolean().+presence_status_is_online(UserId, PresenceTab) ->Presence = guild_state_member:lookup_presence(PresenceTab, UserId),Status = maps:get(<<"status">>, Presence, <<"offline">>),- IsConnected = sets:is_element(UserId, ConnectedUserIds),- IsOnline = IsConnected andalso Status =/= <<"offline">> andalso Status =/= <<"invisible">>,- [{UserId, SortKey, RoleIds, IsOnline} | Acc].+ Status =/= <<"offline">> andalso Status =/= <<"invisible">>.-spec can_view(integer(), pos_integer(), map(), guild_state()) -> boolean().can_view(UserId, ChannelId, Member, State) when is_integer(UserId), UserId > 0 ->@@ -346,4 +491,91 @@ sync_online_noop_without_engines_test() ->ok, sync_online(1, true, #{data => #{<<"guild">> => #{<<"features">> => []}}})).+memo_channel_state() ->+ #{+ data => #{+ <<"guild">> => #{<<"features">> => [], <<"owner_id">> => <<"7">>},+ <<"channel_index">> => #{+ 500 => #{+ <<"id">> => <<"500">>,+ <<"type">> => 0,+ <<"permission_overwrites">> => [#{<<"id">> => <<"42">>, <<"type">> => 1}]+ },+ 600 => #{<<"id">> => <<"600">>, <<"type">> => 4}+ }+ },+ virtual_channel_access => #{99 => sets:from_list([500])}+ }.++memoisable_channel_skips_categories_test() ->+ Data = maps:get(data, memo_channel_state()),+ ?assert(memoisable_channel(500, Data)),+ ?assertNot(memoisable_channel(600, Data)),+ ?assertNot(memoisable_channel(777, Data)).++visibility_exceptions_cover_owner_virtual_and_overwrites_test() ->+ State = memo_channel_state(),+ Exceptions = visibility_exceptions(500, maps:get(data, State), State),+ ?assert(sets:is_element(7, Exceptions)),+ ?assert(sets:is_element(99, Exceptions)),+ ?assert(sets:is_element(42, Exceptions)),+ ?assertNot(sets:is_element(1234, Exceptions)).++new_visibility_memo_is_disabled_for_categories_test() ->+ State = memo_channel_state(),+ ?assertEqual(disabled, new_visibility_memo(600, maps:get(data, State), State)),+ ?assertNotEqual(disabled, new_visibility_memo(500, maps:get(data, State), State)).++memoised_can_view_caches_one_entry_per_role_set_test() ->+ State = memo_channel_state(),+ Memo0 = new_visibility_memo(500, maps:get(data, State), State),+ {First, Memo1} = memoised_can_view(+ 1000, 500, #{<<"roles">> => [2, 1]}, lists:sort([2, 1]), State, Memo0+ ),+ ?assertEqual(1, map_size(maps:get(cache, Memo1))),+ {Second, Memo2} = memoised_can_view(+ 1001, 500, #{<<"roles">> => [1, 2]}, lists:sort([1, 2]), State, Memo1+ ),+ ?assertEqual(First, Second),+ ?assertEqual(1, map_size(maps:get(cache, Memo2))).++memoised_can_view_bypasses_exception_users_test() ->+ State = memo_channel_state(),+ Memo0 = new_visibility_memo(500, maps:get(data, State), State),+ {_, Memo1} = memoised_can_view(7, 500, #{<<"roles">> => []}, [], State, Memo0),+ ?assertEqual(0, map_size(maps:get(cache, Memo1))),+ {_, Memo2} = memoised_can_view(42, 500, #{<<"roles">> => []}, [], State, Memo1),+ ?assertEqual(0, map_size(maps:get(cache, Memo2))).++memoised_can_view_disabled_never_caches_test() ->+ State = memo_channel_state(),+ ?assertMatch(+ {_, disabled},+ memoised_can_view(1000, 600, #{<<"roles">> => [1]}, [1], State, disabled)+ ).++member_role_ids_caches_and_sorts_test() ->+ Member = #{<<"roles">> => [3, 1, 2]},+ {RoleIds, Sorted, Cache1} = member_role_ids(Member, #{}),+ ?assertEqual([1, 2, 3], Sorted),+ ?assertEqual(lists:sort(RoleIds), Sorted),+ ?assertEqual(1, map_size(Cache1)),+ {RoleIds2, Sorted2, Cache2} = member_role_ids(#{<<"roles">> => [3, 1, 2]}, Cache1),+ ?assertEqual({RoleIds, Sorted}, {RoleIds2, Sorted2}),+ ?assertEqual(1, map_size(Cache2)),+ {_, _, Cache3} = member_role_ids(#{<<"roles">> => [9]}, Cache2),+ ?assertEqual(2, map_size(Cache3)).++offline_member_skips_the_presence_lookup_test() ->+ Presences = #{7 => #{<<"status">> => <<"online">>}},+ Connected = sets:from_list([7]),+ ?assert(member_is_online(7, Presences, Connected)),+ ?assertNot(member_is_online(8, Presences, Connected)),+ ?assertNot(member_is_online(7, Presences, sets:new())).++invisible_member_is_not_online_test() ->+ Connected = sets:from_list([7]),+ ?assertNot(member_is_online(7, #{7 => #{<<"status">> => <<"invisible">>}}, Connected)),+ ?assertNot(member_is_online(7, #{7 => #{<<"status">> => <<"offline">>}}, Connected)).+-endif.diff --git a/fluxer_gateway/src/guild/guild_member_list_common.erl b/fluxer_gateway/src/guild/guild_member_list_common.erlindex 6da4efe9f..75800e79e 100644--- a/fluxer_gateway/src/guild/guild_member_list_common.erl+++ b/fluxer_gateway/src/guild/guild_member_list_common.erl@@ -65,11 +65,45 @@ get_member_sort_key(Member) ->-spec casefold_binary(term()) -> binary().casefold_binary(Value) ->Bin = normalize_name(Value),- case unicode_chardata_to_binary(string:casefold(Bin)) of- {ok, Folded} -> Folded;- error -> Bin+ case ascii_casefold(Bin) of+ {ok, Folded} ->+ Folded;+ not_ascii ->+ case unicode_chardata_to_binary(string:casefold(Bin)) of+ {ok, Folded} -> Folded;+ error -> Bin+ endend.+%% Case folding a pure-ASCII binary is exactly A-Z -> a-z, so a byte pass is+%% equivalent to string:casefold/1 and skips two full unicode traversals per+%% member. Anything with a byte above 127 falls through to the unicode path.+-spec ascii_casefold(binary()) -> {ok, binary()} | not_ascii.+ascii_casefold(Bin) ->+ case scan_ascii(Bin, false) of+ not_ascii -> not_ascii;+ false -> {ok, Bin};+ true -> {ok, lower_ascii(Bin, <<>>)}+ end.++-spec scan_ascii(binary(), boolean()) -> boolean() | not_ascii.+scan_ascii(<<>>, HasUpper) ->+ HasUpper;+scan_ascii(<<C, Rest/binary>>, _HasUpper) when C >= $A, C =< $Z ->+ scan_ascii(Rest, true);+scan_ascii(<<C, Rest/binary>>, HasUpper) when C < 128 ->+ scan_ascii(Rest, HasUpper);+scan_ascii(_Bin, _HasUpper) ->+ not_ascii.++-spec lower_ascii(binary(), binary()) -> binary().+lower_ascii(<<>>, Acc) ->+ Acc;+lower_ascii(<<C, Rest/binary>>, Acc) when C >= $A, C =< $Z ->+ lower_ascii(Rest, <<Acc/binary, (C + 32)>>);+lower_ascii(<<C, Rest/binary>>, Acc) ->+ lower_ascii(Rest, <<Acc/binary, C>>).+-spec deep_merge_member(map(), map()) -> map().deep_merge_member(CurrentMember, MemberUpdate) ->User0 = maps:get(<<"user">>, CurrentMember, #{}),@@ -153,3 +187,43 @@ default_presence() -> guild_member_list_connected:default_presence().-spec connected_session_user_ids(map()) -> sets:set(integer()).connected_session_user_ids(S) -> guild_member_list_connected:connected_session_user_ids(S).++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++casefold_reference(Bin) ->+ {ok, Folded} = unicode_chardata_to_binary(string:casefold(Bin)),+ Folded.++ascii_casefold_matches_unicode_casefold_test() ->+ lists:foreach(+ fun(Bin) ->+ ?assertEqual({Bin, casefold_reference(Bin)}, {Bin, casefold_binary(Bin)})+ end,+ [+ <<>>,+ <<"a">>,+ <<"A">>,+ <<"Zed">>,+ <<"already lower">>,+ <<"MiXeD CaSe 123">>,+ <<"punct!@#$%^&*()_+-=[]{}">>,+ <<"0123456789">>,+ <<"~", 127>>+ ]+ ).++ascii_casefold_defers_to_unicode_for_non_ascii_test() ->+ ?assertEqual(not_ascii, ascii_casefold(<<"Ärger"/utf8>>)),+ lists:foreach(+ fun(Bin) ->+ ?assertEqual({Bin, casefold_reference(Bin)}, {Bin, casefold_binary(Bin)})+ end,+ [<<"Ärger"/utf8>>, <<"ÅSTRÖM"/utf8>>, <<"日本語"/utf8>>, <<"ß"/utf8>>]+ ).++ascii_casefold_returns_input_untouched_when_already_folded_test() ->+ Bin = <<"no uppercase here">>,+ ?assertEqual({ok, Bin}, ascii_casefold(Bin)).++-endif.diff --git a/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl b/fluxer_gateway/src/guild/guild_member_list_read_cursor.erlindex 73a4f6ce6..91503d829 100644--- a/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl+++ b/fluxer_gateway/src/guild/guild_member_list_read_cursor.erl@@ -25,7 +25,7 @@ members_cursor(Limit, AfterId, State) ->Data = maps:get(data, State, #{}),MemberMap = guild_data_index:member_map(Data),Total = map_size(MemberMap),- SortedIds = lists:sort(maps:keys(MemberMap)),+ SortedIds = guild_data_index_members:sorted_member_ids(Data, MemberMap),FilteredIds = filter_ids_after(SortedIds, AfterId),ResponseMembers = take_members(FilteredIds, Limit, MemberMap),{reply, #{members => ResponseMembers, total => Total}, State}.@@ -33,8 +33,61 @@ members_cursor(Limit, AfterId, State) ->-spec take_members([integer()], integer(), map()) -> [map()].take_members(_Ids, Limit, _MemberMap) when Limit =< 0 -> [];take_members(Ids, Limit, MemberMap) ->- [maps:get(Id, MemberMap) || Id <- lists:sublist(Ids, Limit)].+ [Member || Id <- lists:sublist(Ids, Limit), {ok, Member} <- [maps:find(Id, MemberMap)]].-spec filter_ids_after([integer()], integer() | undefined) -> [integer()].filter_ids_after(Ids, undefined) -> Ids;filter_ids_after(Ids, AfterId) -> lists:dropwhile(fun(Id) -> Id =< AfterId end, Ids).++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++get_members_cursor_matches_full_sort_test() ->+ lists:foreach(fun assert_cursor_page_matches/1, cursor_cases()).++assert_cursor_page_matches({State, Limit, After}) ->+ {reply, Reply, _} = get_members_cursor(cursor_request(Limit, After), State),+ ?assertEqual(legacy_members_cursor(Limit, After, State), Reply).++legacy_members_cursor(Limit, After, State) ->+ MemberMap = guild_data_index:member_map(maps:get(data, State, #{})),+ AfterId = snowflake_id:parse_optional(After),+ Ids = filter_ids_after(lists:sort(maps:keys(MemberMap)), AfterId),+ #{members => legacy_take_members(Ids, Limit, MemberMap), total => map_size(MemberMap)}.++legacy_take_members(_Ids, Limit, _MemberMap) when Limit =< 0 -> [];+legacy_take_members(Ids, Limit, MemberMap) ->+ [maps:get(Id, MemberMap) || Id <- lists:sublist(Ids, Limit)].++cursor_request(Limit, undefined) -> #{<<"limit">> => Limit};+cursor_request(Limit, After) -> #{<<"limit">> => Limit, <<"after">> => After}.++cursor_cases() ->+ [+ {State, Limit, After}+ || State <- cursor_states(),+ Limit <- [0, 1, 2, 99],+ After <- [undefined, 1, 5, 12, 39, 40, 41, 99]+ ].++cursor_states() ->+ Members = cursor_members(cursor_user_ids(40)),+ Cached = guild_data_index:normalize_data(#{<<"members">> => Members}),+ [+ #{data => Cached},+ #{data => guild_data_index:put_member(cursor_member(500), Cached)},+ #{data => #{<<"members">> => Members}},+ #{data => guild_data_index:normalize_data(#{<<"members">> => []})},+ #{}+ ].++cursor_members(UserIds) ->+ [cursor_member(UserId) || UserId <- UserIds].++cursor_user_ids(Count) ->+ lists:sort(fun(A, B) -> erlang:phash2(A) =< erlang:phash2(B) end, lists:seq(1, Count)).++cursor_member(UserId) ->+ #{<<"user">> => #{<<"id">> => integer_to_binary(UserId)}}.++-endif.diff --git a/fluxer_gateway/src/guild/guild_member_list_write.erl b/fluxer_gateway/src/guild/guild_member_list_write.erlindex 1dfdc0a46..867e786bf 100644--- a/fluxer_gateway/src/guild/guild_member_list_write.erl+++ b/fluxer_gateway/src/guild/guild_member_list_write.erl@@ -20,6 +20,10 @@-type list_id() :: binary().-type user_id() :: integer().-type channel_id() :: integer().+-type engine_ref() :: ets:table().+-type absence() :: {absent, engine_ref()} | present.++-define(MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT, 1000000000).-export_type([guild_state/0, list_id/0, user_id/0, channel_id/0]).@@ -180,14 +184,14 @@ broadcast_channel_engine_connection_change(UserId, State, SubsTab) ->guild_state().fold_connection_change_lists(GuildId, UserId, State, SubsTab) ->Sessions = maps:get(sessions, State, #{}),- guild_member_list_subs:fold_list_ids(+ lists:foldl(fun(ListId, AccState) ->- sync_connection_change_for_list(- GuildId, UserId, ListId, SubsTab, Sessions, AccState+ sync_connection_change_for_subscribed_list(+ GuildId, UserId, ListId, Sessions, AccState)end,State,- SubsTab+ guild_member_list_subs:list_ids(SubsTab)).-spec dispatch_user_change_to_subscribed_lists(@@ -200,29 +204,38 @@ fold_connection_change_lists(GuildId, UserId, State, SubsTab) ->dispatch_user_change_to_subscribed_lists(UserId, OldMember, NewMember, SubsTab, State) ->- guild_member_list_subs:fold_list_ids(+ lists:foldl(fun(ListId, AccState) ->- dispatch_user_change_to_list(- UserId, OldMember, NewMember, ListId, SubsTab, AccState+ dispatch_user_change_to_subscribed_list(+ UserId, OldMember, NewMember, ListId, AccState)end,State,- SubsTab+ guild_member_list_subs:list_ids(SubsTab)).--spec dispatch_user_change_to_list(+-spec dispatch_user_change_to_subscribed_list(user_id(),map() | undefined,map() | undefined,list_id(),- ets:table(),guild_state()) -> guild_state().-dispatch_user_change_to_list(- UserId, OldMember, NewMember, ListId, SubsTab, State-) ->+dispatch_user_change_to_subscribed_list(UserId, OldMember, NewMember, ListId, State) ->+ Absence = user_change_absence(UserId, ListId, OldMember, NewMember, State),State1 = apply_user_change_to_channel_store(UserId, ListId, OldMember, NewMember, State),- queue_list_sync_if_subscribed(ListId, SubsTab, State1).+ case sync_body_unchanged(UserId, ListId, Absence, State1) of+ true -> record_member_list_sync_skipped_absent(State1);+ false -> guild_member_list_sync_batch:queue_list_sync(ListId, State1)+ end.++-spec user_change_absence(+ user_id(), list_id(), map() | undefined, map() | undefined, guild_state()+) -> absence().+user_change_absence(UserId, ListId, Member, Member, State) when is_map(Member) ->+ list_member_absence(UserId, ListId, State);+user_change_absence(_UserId, _ListId, _OldMember, _NewMember, _State) ->+ present.-spec apply_user_change_to_channel_store(user_id(), list_id(), map() | undefined, map() | undefined, guild_state()@@ -249,18 +262,13 @@ apply_channel_member_change(UserId, ListId, OldMember, NewMember, State) whenapply_channel_member_change(_UserId, _ListId, _OldMember, _NewMember, _State) ->ok.--spec sync_connection_change_for_list(- integer(),- user_id(),- list_id(),- ets:table(),- map(),- guild_state()+-spec sync_connection_change_for_subscribed_list(+ integer(), user_id(), list_id(), map(), guild_state()) -> guild_state().-sync_connection_change_for_list(_GuildId, _UserId, ListId, SubsTab, _Sessions, State) ->+sync_connection_change_for_subscribed_list(_GuildId, UserId, ListId, _Sessions, State) ->case guild_member_list_channel_engine:is_engine_list(ListId, State) oftrue ->- queue_list_sync_if_subscribed(ListId, SubsTab, State);+ queue_connection_list_sync_unless_absent(UserId, ListId, State);false ->Stateend.@@ -280,12 +288,10 @@ broadcast_list_by_id(_GuildId, _ChannelId, ListId, State) ->-spec rebuild_subscribed_channel_lists(guild_state(), ets:table()) -> guild_state().rebuild_subscribed_channel_lists(State, SubsTab) ->- guild_member_list_subs:fold_lists(- fun(ListId, _Subs, AccState) ->- rebuild_channel_store(ListId, AccState)- end,+ lists:foldl(+ fun rebuild_channel_store/2,State,- SubsTab+ guild_member_list_subs:list_ids(SubsTab)).-spec rebuild_channel_store(list_id(), guild_state()) -> guild_state().@@ -295,13 +301,104 @@ rebuild_channel_store(ListId, State) ->false -> Stateend.--spec queue_list_sync_if_subscribed(list_id(), ets:table(), guild_state()) -> guild_state().-queue_list_sync_if_subscribed(ListId, SubsTab, State) ->- case guild_member_list_subs:has_subs(ListId, SubsTab) of- false -> State;- true -> guild_member_list_sync_batch:queue_list_sync(ListId, State)+-spec queue_connection_list_sync_unless_absent(user_id(), list_id(), guild_state()) ->+ guild_state().+queue_connection_list_sync_unless_absent(UserId, ListId, State) ->+ case list_member_absence(UserId, ListId, State) of+ {absent, _Ref} -> record_member_list_sync_skipped_absent(State);+ present -> queue_connection_list_sync(ListId, State)end.+-spec queue_connection_list_sync(list_id(), guild_state()) -> guild_state().+queue_connection_list_sync(ListId, State) ->+ case maps:get(pending_member_list_sync_batch, State, undefined) of+ #{pending_list_ids := PendingListIds} = Batch when is_map(PendingListIds) ->+ State#{+ pending_member_list_sync_batch => Batch#{+ pending_list_ids => PendingListIds#{ListId => true}+ }+ };+ _ ->+ TimerRef = erlang:send_after(+ connection_sync_delay_ms(State), self(), flush_member_list_sync_batch+ ),+ State#{+ pending_member_list_sync_batch => #{+ pending_list_ids => #{ListId => true},+ timer_ref => TimerRef+ }+ }+ end.++-spec connection_sync_delay_ms(guild_state()) -> pos_integer().+connection_sync_delay_ms(State) ->+ case State of+ #{member_count := MemberCount} when is_integer(MemberCount), MemberCount >= 5000 ->+ positive_env(member_list_large_connection_sync_delay_ms, 1000);+ _ ->+ positive_env(member_list_connection_sync_delay_ms, 250)+ end.++-spec positive_env(atom(), pos_integer()) -> pos_integer().+positive_env(Key, Default) ->+ case application:get_env(fluxer_gateway, Key, Default) of+ Value when is_integer(Value), Value >= 1 -> Value;+ _ -> Default+ end.++-spec record_member_list_sync_skipped_absent(guild_state()) -> guild_state().+record_member_list_sync_skipped_absent(State) ->+ case maps:get(member_list_sync_skipped_absent, State, 0) of+ Count when+ is_integer(Count), Count >= 0, Count < ?MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT+ ->+ State#{member_list_sync_skipped_absent => Count + 1};+ ?MAX_MEMBER_LIST_SYNC_SKIPPED_ABSENT ->+ State;+ Invalid ->+ erlang:error({invalid_member_list_sync_skipped_absent, Invalid})+ end.++-spec sync_body_unchanged(user_id(), list_id(), absence(), guild_state()) -> boolean().+sync_body_unchanged(_UserId, _ListId, present, _State) ->+ false;+sync_body_unchanged(UserId, ListId, {absent, Ref}, State) ->+ list_store_ref(ListId, State) =:= Ref andalso member_row_absent(UserId, Ref).++-spec list_member_absence(user_id(), list_id(), guild_state()) -> absence().+list_member_absence(UserId, ListId, State) ->+ absence_for_store(UserId, list_store_ref(ListId, State)).++-spec absence_for_store(user_id(), engine_ref() | undefined) -> absence().+absence_for_store(_UserId, undefined) ->+ present;+absence_for_store(UserId, Ref) ->+ case member_row_absent(UserId, Ref) of+ true -> {absent, Ref};+ false -> present+ end.++-spec member_row_absent(user_id(), engine_ref()) -> boolean().+member_row_absent(UserId, Ref) when is_integer(UserId), UserId > 0 ->+ try guild_member_list_engine:is_member_online(Ref, UserId) of+ not_present -> true;+ _Online -> false+ catch+ _:_ -> false+ end;+member_row_absent(_UserId, _Ref) ->+ false.++-spec list_store_ref(list_id(), guild_state()) -> engine_ref() | undefined.+list_store_ref(<<"0">>, State) ->+ case maps:get(member_list_engine, State, undefined) of+ undefined -> undefined;+ Ref when is_reference(Ref); is_atom(Ref) -> Ref;+ _Other -> undefined+ end;+list_store_ref(ListId, State) ->+ guild_member_list_channel_engine:ref(ListId, State).+-spec flush_pending_member_list_syncs(guild_state()) -> guild_state().flush_pending_member_list_syncs(State) ->guild_member_list_sync_batch:flush_pending_syncs(State).@@ -316,12 +413,247 @@ apply_user_change_to_channel_store_updates_engine_test() ->State1 = apply_user_change_to_channel_store(2, <<"123">>, undefined, Member, State),?assert(maps:is_key(<<"123">>, maps:get(channel_member_list_engines, State1))).-sync_connection_change_ignores_zero_list_test() ->- SubsTab = guild_member_list_subs:new(),- ?assertEqual(- #{},- sync_connection_change_for_list(1, 2, <<"0">>, SubsTab, #{}, #{})- ),- guild_member_list_subs:destroy(SubsTab).+absence_is_absent_for_unloaded_member_test() ->+ Ref = guild_member_list_engine:new(),+ State = engine_state(Ref),+ try+ ?assertEqual({absent, Ref}, list_member_absence(7, <<"500">>, State))+ after+ guild_member_list_engine:destroy(Ref)+ end.++absence_is_present_for_loaded_member_test() ->+ Ref = guild_member_list_engine:new(),+ State = engine_state(Ref),+ try+ ok = guild_member_list_engine:bulk_load(Ref, [{7, <<"seven">>, [], true}], []),+ ?assertEqual(present, list_member_absence(7, <<"500">>, State)),+ ?assertEqual({absent, Ref}, list_member_absence(8, <<"500">>, State))+ after+ guild_member_list_engine:destroy(Ref)+ end.++absence_is_present_without_store_test() ->+ ?assertEqual(present, list_member_absence(7, <<"500">>, #{})),+ ?assertEqual(present, list_member_absence(7, <<"0">>, #{})),+ ?assertEqual(present, list_member_absence(0, <<"500">>, #{})).++default_list_absence_reads_default_engine_test() ->+ Ref = guild_member_list_engine:new(),+ State = #{member_list_engine => Ref},+ try+ ?assertEqual({absent, Ref}, list_member_absence(7, <<"0">>, State)),+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),+ ?assertEqual(present, list_member_absence(7, <<"0">>, State))+ after+ guild_member_list_engine:destroy(Ref)+ end.++sync_body_unchanged_requires_stable_ref_test() ->+ Ref = guild_member_list_engine:new(),+ Other = guild_member_list_engine:new(),+ State = engine_state(Ref),+ try+ ?assert(sync_body_unchanged(7, <<"500">>, {absent, Ref}, State)),+ ?assertNot(sync_body_unchanged(7, <<"500">>, present, State)),+ ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Other}, State)),+ ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Ref}, #{}))+ after+ guild_member_list_engine:destroy(Ref),+ guild_member_list_engine:destroy(Other)+ end.++sync_body_unchanged_false_when_member_reappears_test() ->+ Ref = guild_member_list_engine:new(),+ State = engine_state(Ref),+ try+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),+ ?assertNot(sync_body_unchanged(7, <<"500">>, {absent, Ref}, State))+ after+ guild_member_list_engine:destroy(Ref)+ end.++user_change_absence_forces_sync_on_member_change_test() ->+ Ref = guild_member_list_engine:new(),+ State = engine_state(Ref),+ Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []},+ try+ ?assertEqual({absent, Ref}, user_change_absence(7, <<"500">>, Member, Member, State)),+ ?assertEqual(present, user_change_absence(7, <<"500">>, Member, undefined, State)),+ ?assertEqual(present, user_change_absence(7, <<"500">>, undefined, Member, State)),+ ?assertEqual(+ present,+ user_change_absence(7, <<"500">>, Member, Member#{<<"nick">> => <<"x">>}, State)+ )+ after+ guild_member_list_engine:destroy(Ref)+ end.++subscribed_list_sync_is_skipped_only_for_absent_member_test() ->+ Ref = guild_member_list_engine:new(),+ Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []},+ State = (engine_state(Ref))#{pending_member_list_sync_batch => #{pending_list_ids => #{}}},+ try+ with_batch_window(false, fun() ->+ ?assert(+ maps:is_key(+ pending_member_list_sync_batch,+ dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)+ )+ ),+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),+ ?assertNot(+ maps:is_key(+ pending_member_list_sync_batch,+ dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)+ )+ )+ end)+ after+ guild_member_list_engine:destroy(Ref)+ end.++list_id_fold_matches_fold_lists_test() ->+ Tab = guild_member_list_subs:new(),+ try+ guild_member_list_subs:subscribe(<<"s1">>, <<"500">>, [{0, 99}], Tab),+ guild_member_list_subs:subscribe(<<"s2">>, <<"500">>, [{0, 9}], Tab),+ guild_member_list_subs:subscribe(<<"s1">>, <<"600">>, [{0, 99}], Tab),+ Sizes = guild_member_list_subs:fold_lists(+ fun(ListId, ListSubs, Acc) -> Acc#{ListId => map_size(ListSubs)} end,+ #{},+ Tab+ ),+ ?assertEqual(lists:sort(maps:keys(Sizes)), guild_member_list_subs:list_ids(Tab)),+ ?assertEqual([], [ListId || {ListId, 0} <- maps:to_list(Sizes)])+ after+ guild_member_list_subs:destroy(Tab)+ end.++connection_sync_delay_defaults_test() ->+ ?assertEqual(250, connection_sync_delay_ms(#{})),+ ?assertEqual(250, connection_sync_delay_ms(#{member_count => 4999})),+ ?assertEqual(250, connection_sync_delay_ms(#{member_count => undefined})),+ ?assertEqual(1000, connection_sync_delay_ms(#{member_count => 5000})),+ ?assertEqual(1000, connection_sync_delay_ms(#{member_count => 60000})).++connection_sync_delay_reads_env_test() ->+ with_env(member_list_connection_sync_delay_ms, 40, fun() ->+ ?assertEqual(40, connection_sync_delay_ms(#{member_count => 10}))+ end),+ with_env(member_list_large_connection_sync_delay_ms, 90, fun() ->+ ?assertEqual(90, connection_sync_delay_ms(#{member_count => 5000}))+ end).++connection_sync_delay_rejects_invalid_env_test() ->+ lists:foreach(+ fun(Invalid) ->+ with_env(member_list_connection_sync_delay_ms, Invalid, fun() ->+ ?assertEqual(250, connection_sync_delay_ms(#{}))+ end)+ end,+ [0, -1, undefined, 250.0, <<"250">>]+ ).++connection_list_sync_arms_one_timer_per_window_test() ->+ State1 = queue_connection_list_sync(<<"500">>, #{}),+ Batch1 = maps:get(pending_member_list_sync_batch, State1),+ TimerRef = maps:get(timer_ref, Batch1),+ try+ ?assert(is_reference(TimerRef)),+ ?assertEqual(#{<<"500">> => true}, maps:get(pending_list_ids, Batch1)),+ State2 = queue_connection_list_sync(<<"600">>, State1),+ Batch2 = maps:get(pending_member_list_sync_batch, State2),+ ?assertEqual(TimerRef, maps:get(timer_ref, Batch2)),+ ?assertEqual(+ #{<<"500">> => true, <<"600">> => true}, maps:get(pending_list_ids, Batch2)+ ),+ State3 = queue_connection_list_sync(<<"500">>, State2),+ ?assertEqual(Batch2, maps:get(pending_member_list_sync_batch, State3))+ after+ _ = erlang:cancel_timer(TimerRef)+ end.++connection_change_sync_is_debounced_test() ->+ Ref = guild_member_list_engine:new(),+ try+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),+ Next = sync_connection_change_for_subscribed_list(+ 1, 7, <<"500">>, #{}, engine_state(Ref)+ ),+ Batch = maps:get(pending_member_list_sync_batch, Next),+ ?assertEqual(#{<<"500">> => true}, maps:get(pending_list_ids, Batch)),+ _ = erlang:cancel_timer(maps:get(timer_ref, Batch))+ after+ guild_member_list_engine:destroy(Ref)+ end.++connection_change_skip_absent_arms_no_timer_test() ->+ Ref = guild_member_list_engine:new(),+ try+ Next = sync_connection_change_for_subscribed_list(+ 1, 7, <<"500">>, #{}, engine_state(Ref)+ ),+ ?assertNot(maps:is_key(pending_member_list_sync_batch, Next)),+ ?assertEqual(1, maps:get(member_list_sync_skipped_absent, Next))+ after+ guild_member_list_engine:destroy(Ref)+ end.++connection_change_skip_absent_keeps_pending_batch_test() ->+ Ref = guild_member_list_engine:new(),+ TimerRef = erlang:send_after(60000, self(), flush_member_list_sync_batch),+ Batch = #{pending_list_ids => #{<<"600">> => true}, timer_ref => TimerRef},+ State = (engine_state(Ref))#{pending_member_list_sync_batch => Batch},+ try+ Next = sync_connection_change_for_subscribed_list(1, 7, <<"500">>, #{}, State),+ ?assertEqual(Batch, maps:get(pending_member_list_sync_batch, Next))+ after+ _ = erlang:cancel_timer(TimerRef),+ guild_member_list_engine:destroy(Ref)+ end.++member_update_sync_stays_immediate_test() ->+ Ref = guild_member_list_engine:new(),+ TimerRef = erlang:send_after(60000, self(), flush_member_list_sync_batch),+ Member = #{<<"user">> => #{<<"id">> => <<"7">>}, <<"roles">> => []},+ State = (engine_state(Ref))#{+ pending_member_list_sync_batch => #{+ pending_list_ids => #{<<"600">> => true},+ timer_ref => TimerRef+ }+ },+ try+ ok = guild_member_list_engine:add_member(Ref, 7, <<"seven">>, [], true),+ ?assertNot(+ maps:is_key(+ pending_member_list_sync_batch,+ dispatch_user_change_to_subscribed_list(7, Member, Member, <<"500">>, State)+ )+ )+ after+ _ = erlang:cancel_timer(TimerRef),+ guild_member_list_engine:destroy(Ref)+ end.++engine_state(Ref) ->+ #{channel_member_list_engines => #{<<"500">> => Ref}, member_presence => #{}}.++with_batch_window(Value, Fun) ->+ with_env(guild_member_list_sync_batch_window, Value, Fun).++with_env(Key, Value, Fun) ->+ Previous = application:get_env(fluxer_gateway, Key),+ application:set_env(fluxer_gateway, Key, Value),+ try+ Fun()+ after+ restore_env(Key, Previous)+ end.++restore_env(Key, {ok, Previous}) ->+ application:set_env(fluxer_gateway, Key, Previous);+restore_env(Key, undefined) ->+ application:unset_env(fluxer_gateway, Key).-endif.diff --git a/fluxer_gateway/src/guild/guild_members_table_repair.erl b/fluxer_gateway/src/guild/guild_members_table_repair.erlnew file mode 100644index 000000000..daa508360--- /dev/null+++ b/fluxer_gateway/src/guild/guild_members_table_repair.erl@@ -0,0 +1,542 @@+%% SPDX-License-Identifier: AGPL-3.0-or-later++-module(guild_members_table_repair).+-typing([eqwalizer]).++-export([+ maybe_repair/1,+ members_table_healthy/1+]).++-define(REPAIR_KEY, members_ets_repair).+-define(ATTEMPTS_KEY, members_ets_repair_attempts).+-define(MAX_MEMBER_ETS_REPAIR_CHUNK, 5000).+-define(MAX_MEMBER_ETS_REPAIR_MEMBERS, 250000).+-define(MAX_MEMBER_ETS_REPAIR_TICKS, 200).+-define(MAX_MEMBER_ETS_REPAIR_ATTEMPTS, 3).+-define(MAX_MEMBER_ETS_REPAIR_MAILBOX, 1000).++-type guild_state() :: map().+-type user_id() :: integer().+-type member() :: map().+-type member_map() :: #{user_id() => member()}.+-type pass_result() :: continue | complete | {error, table_unavailable}.++-export_type([guild_state/0]).++-spec maybe_repair(guild_state()) -> guild_state().+maybe_repair(State) ->+ case self_heal_enabled() of+ true -> repair_enabled(State);+ false -> release_pending_repair(State)+ end.++-spec repair_enabled(guild_state()) -> guild_state().+repair_enabled(State) ->+ case members_table_healthy(members_table(State)) of+ true -> release_pending_repair(State);+ false -> repair_unhealthy(State)+ end.++-spec self_heal_enabled() -> boolean().+self_heal_enabled() ->+ application:get_env(fluxer_gateway, guild_members_table_self_heal, true) =:= true.++-spec members_table_healthy(term()) -> boolean().+members_table_healthy(Tab) when is_reference(Tab) ->+ table_owner(Tab) =:= self();+members_table_healthy(_Tab) ->+ false.++-spec repair_unhealthy(guild_state()) -> guild_state().+repair_unhealthy(State) ->+ case mailbox_within_bound(message_queue_len()) of+ true -> repair_pass(State);+ false -> State+ end.++-spec repair_pass(guild_state()) -> guild_state().+repair_pass(State) ->+ case maps:get(?REPAIR_KEY, State, undefined) of+ #{tab := Tab, ticks := Ticks} when is_reference(Tab), is_integer(Ticks) ->+ continue_repair(Tab, Ticks, State);+ _ ->+ start_repair(maps:remove(?REPAIR_KEY, State))+ end.++-spec start_repair(guild_state()) -> guild_state().+start_repair(State) ->+ case attempts(State) < ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS of+ true -> start_repair_within_bounds(State);+ false -> State+ end.++-spec start_repair_within_bounds(guild_state()) -> guild_state().+start_repair_within_bounds(State) ->+ Size = map_size(member_map(State)),+ case Size =< ?MAX_MEMBER_ETS_REPAIR_MEMBERS of+ true -> create_staging_table(State);+ false -> refuse_oversized(Size, State)+ end.++-spec refuse_oversized(non_neg_integer(), guild_state()) -> guild_state().+refuse_oversized(Size, State) ->+ logger:warning(+ "guild_members_table_repair_refused: guild_id=~p members=~p max=~p",+ [guild_id(State), Size, ?MAX_MEMBER_ETS_REPAIR_MEMBERS]+ ),+ State#{?ATTEMPTS_KEY => ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS}.++-spec create_staging_table(guild_state()) -> guild_state().+create_staging_table(State) ->+ case new_staging_table() of+ Tab when is_reference(Tab) -> continue_repair(Tab, 0, State);+ undefined -> record_start_failure(State)+ end.++-spec continue_repair(ets:tid(), non_neg_integer(), guild_state()) -> guild_state().+continue_repair(Tab, Ticks, State) when Ticks >= ?MAX_MEMBER_ETS_REPAIR_TICKS ->+ abandon_repair(Tab, ticks_exhausted, State);+continue_repair(Tab, Ticks, State) ->+ case run_pass(Tab, member_map(State)) of+ continue -> State#{?REPAIR_KEY => #{tab => Tab, ticks => Ticks + 1}};+ complete -> publish_repair(Tab, State);+ {error, table_unavailable} -> abandon_repair(Tab, table_unavailable, State)+ end.++-spec publish_repair(ets:tid(), guild_state()) -> guild_state().+publish_repair(Tab, State) ->+ case maps:get(data, State, #{}) of+ Data when is_map(Data) -> publish_into_data(Tab, Data, State);+ _ -> abandon_repair(Tab, invalid_guild_data, State)+ end.++-spec publish_into_data(ets:tid(), map(), guild_state()) -> guild_state().+publish_into_data(Tab, Data, State) ->+ logger:info(+ "guild_members_table_repair_completed: guild_id=~p members=~p",+ [guild_id(State), map_size(member_map(State))]+ ),+ Published = State#{data => Data#{members_ets => Tab}},+ maps:remove(?ATTEMPTS_KEY, maps:remove(?REPAIR_KEY, Published)).++-spec abandon_repair(ets:tid(), atom(), guild_state()) -> guild_state().+abandon_repair(Tab, Reason, State) ->+ ok = delete_staging_table(Tab),+ logger:warning(+ "guild_members_table_repair_abandoned: guild_id=~p reason=~p attempts=~p",+ [guild_id(State), Reason, attempts(State) + 1]+ ),+ bump_attempts(maps:remove(?REPAIR_KEY, State)).++-spec record_start_failure(guild_state()) -> guild_state().+record_start_failure(State) ->+ logger:warning(+ "guild_members_table_repair_start_failed: guild_id=~p attempts=~p",+ [guild_id(State), attempts(State) + 1]+ ),+ bump_attempts(State).++-spec run_pass(ets:tid(), member_map()) -> pass_result().+run_pass(Tab, MemberMap) ->+ run_pass(Tab, MemberMap, ?MAX_MEMBER_ETS_REPAIR_CHUNK).++-spec run_pass(ets:tid(), member_map(), pos_integer()) -> pass_result().+run_pass(Tab, MemberMap, Budget) ->+ try+ apply_pass(Tab, MemberMap, Budget)+ catch+ error:badarg -> {error, table_unavailable}+ end.++-spec apply_pass(ets:tid(), member_map(), pos_integer()) -> continue | complete.+apply_pass(Tab, MemberMap, Budget) ->+ case collect_missing_rows(Tab, MemberMap, Budget) of+ [] -> drop_extra_rows(Tab, MemberMap, Budget);+ Rows -> insert_missing_rows(Tab, MemberMap, Rows, Budget)+ end.++-spec insert_missing_rows(ets:tid(), member_map(), [{user_id(), member()}], pos_integer()) ->+ continue | complete.+insert_missing_rows(Tab, MemberMap, Rows, Budget) ->+ true = ets:insert(Tab, Rows),+ case length(Rows) < Budget of+ true -> drop_extra_rows(Tab, MemberMap, Budget);+ false -> continue+ end.++-spec drop_extra_rows(ets:tid(), member_map(), pos_integer()) -> continue | complete.+drop_extra_rows(Tab, MemberMap, Budget) ->+ case collect_extra_ids(Tab, MemberMap, Budget) of+ [] -> complete;+ Ids -> delete_extra_ids(Tab, Ids, Budget)+ end.++-spec delete_extra_ids(ets:tid(), [user_id()], pos_integer()) -> continue | complete.+delete_extra_ids(Tab, Ids, Budget) ->+ lists:foreach(fun(UserId) -> ets:delete(Tab, UserId) end, Ids),+ case length(Ids) < Budget of+ true -> complete;+ false -> continue+ end.++-spec collect_missing_rows(ets:tid(), member_map(), pos_integer()) ->+ [{user_id(), member()}].+collect_missing_rows(Tab, MemberMap, Budget) ->+ collect_missing_iter(maps:next(maps:iterator(MemberMap)), Tab, Budget, []).++-spec collect_missing_iter(term(), ets:tid(), non_neg_integer(), [{user_id(), member()}]) ->+ [{user_id(), member()}].+collect_missing_iter(_Next, _Tab, 0, Acc) ->+ Acc;+collect_missing_iter(none, _Tab, _Budget, Acc) ->+ Acc;+collect_missing_iter({UserId, Member, Iter}, Tab, Budget, Acc) ->+ case ets:member(Tab, UserId) of+ true ->+ collect_missing_iter(maps:next(Iter), Tab, Budget, Acc);+ false ->+ collect_missing_iter(maps:next(Iter), Tab, Budget - 1, [{UserId, Member} | Acc])+ end.++-spec collect_extra_ids(ets:tid(), member_map(), pos_integer()) -> [user_id()].+collect_extra_ids(Tab, MemberMap, Budget) ->+ collect_extra_iter(ets:first(Tab), Tab, MemberMap, Budget, []).++-spec collect_extra_iter(term(), ets:tid(), member_map(), non_neg_integer(), [user_id()]) ->+ [user_id()].+collect_extra_iter(_Key, _Tab, _MemberMap, 0, Acc) ->+ Acc;+collect_extra_iter('$end_of_table', _Tab, _MemberMap, _Budget, Acc) ->+ Acc;+collect_extra_iter(Key, Tab, MemberMap, Budget, Acc) ->+ Next = ets:next(Tab, Key),+ case maps:is_key(Key, MemberMap) of+ true ->+ collect_extra_iter(Next, Tab, MemberMap, Budget, Acc);+ false ->+ collect_extra_iter(Next, Tab, MemberMap, Budget - 1, [Key | Acc])+ end.++-spec release_pending_repair(guild_state()) -> guild_state().+release_pending_repair(State) ->+ case maps:get(?REPAIR_KEY, State, undefined) of+ undefined -> State;+ #{tab := Tab} when is_reference(Tab) -> discard_staging_table(Tab, State);+ _ -> maps:remove(?REPAIR_KEY, State)+ end.++-spec discard_staging_table(ets:tid(), guild_state()) -> guild_state().+discard_staging_table(Tab, State) ->+ ok = delete_staging_table(Tab),+ maps:remove(?REPAIR_KEY, State).++-spec new_staging_table() -> ets:tid() | undefined.+new_staging_table() ->+ try ets:new(guild_members_data, [set, public, {read_concurrency, true}]) of+ Tab when is_reference(Tab) -> Tab;+ _Named -> undefined+ catch+ error:badarg -> undefined;+ error:system_limit -> undefined+ end.++-spec delete_staging_table(ets:tid()) -> ok.+delete_staging_table(Tab) ->+ try ets:delete(Tab) of+ _ -> ok+ catch+ error:badarg -> ok+ end.++-spec table_owner(term()) -> pid() | undefined.+table_owner(Tab) ->+ try ets:info(eqwalizer:dynamic_cast(Tab), owner) of+ Owner when is_pid(Owner) -> Owner;+ _ -> undefined+ catch+ error:badarg -> undefined+ end.++-spec members_table(guild_state()) -> ets:tid() | undefined.+members_table(State) ->+ members_ets_table(data_map(State)).++-spec members_ets_table(map()) -> ets:tid() | undefined.+members_ets_table(#{members_ets := Tab}) ->+ Tab;+members_ets_table(_Data) ->+ undefined.++-spec member_map(guild_state()) -> member_map().+member_map(State) ->+ guild_data_index_members:member_map(data_map(State)).++-spec data_map(guild_state()) -> map().+data_map(State) ->+ case maps:get(data, State, #{}) of+ Data when is_map(Data) -> Data;+ _ -> #{}+ end.++-spec guild_id(guild_state()) -> term().+guild_id(State) ->+ maps:get(id, State, undefined).++-spec attempts(guild_state()) -> non_neg_integer().+attempts(State) ->+ case maps:get(?ATTEMPTS_KEY, State, 0) of+ N when is_integer(N), N >= 0 -> N;+ _ -> 0+ end.++-spec bump_attempts(guild_state()) -> guild_state().+bump_attempts(State) ->+ State#{?ATTEMPTS_KEY => attempts(State) + 1}.++-spec message_queue_len() -> non_neg_integer().+message_queue_len() ->+ case erlang:process_info(self(), message_queue_len) of+ {message_queue_len, Len} when is_integer(Len), Len >= 0 -> Len;+ _ -> 0+ end.++-spec mailbox_within_bound(non_neg_integer()) -> boolean().+mailbox_within_bound(Len) ->+ Len < ?MAX_MEMBER_ETS_REPAIR_MAILBOX.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++healthy_table_is_left_untouched_test() ->+ Tab = ets:new(healthy_members, [set, public]),+ try+ State = repair_state_with_table(#{1 => test_member(1)}, Tab),+ ?assertEqual(State, maybe_repair(State)),+ ?assertEqual(0, ets:info(Tab, size))+ after+ ets:delete(Tab)+ end.++healthy_table_releases_pending_staging_table_test() ->+ Healthy = ets:new(healthy_members, [set, public]),+ try+ Staging = ets:new(rollback_members, [set, public]),+ State0 = with_repair_key(+ repair_state_with_table(#{1 => test_member(1)}, Healthy), Staging, 3+ ),+ State1 = maybe_repair(State0),+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),+ ?assertEqual(undefined, ets:info(Staging, size))+ after+ ets:delete(Healthy)+ end.++repair_publishes_table_matching_member_map_test() ->+ ok = drain_mailbox(),+ with_self_heal_enabled(fun() ->+ MemberMap = #{1 => test_member(1), 2 => test_member(2)},+ State1 = maybe_repair(repair_state(MemberMap)),+ Tab = published_table(State1),+ ?assert(is_reference(Tab)),+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),+ ?assertEqual(lists:sort(maps:to_list(MemberMap)), lists:sort(ets:tab2list(Tab))),+ ?assertEqual(State1, maybe_repair(State1)),+ ets:delete(Tab)+ end).++repair_replaces_dead_table_tid_test() ->+ ok = drain_mailbox(),+ with_self_heal_enabled(fun() ->+ Dead = ets:new(dead_members, [set, public]),+ true = ets:delete(Dead),+ State0 = repair_state_with_table(#{1 => test_member(1)}, Dead),+ State1 = maybe_repair(State0),+ Tab = published_table(State1),+ ?assert(is_reference(Tab)),+ ?assertNotEqual(Dead, Tab),+ ?assertEqual([{1, test_member(1)}], ets:tab2list(Tab)),+ ets:delete(Tab)+ end).++repair_replaces_table_owned_by_another_process_test() ->+ ok = drain_mailbox(),+ with_self_heal_enabled(fun() ->+ {Owner, Foreign} = foreign_members_table(),+ State1 = maybe_repair(repair_state_with_table(#{1 => test_member(1)}, Foreign)),+ Tab = published_table(State1),+ ?assert(is_reference(Tab)),+ ?assertNotEqual(Foreign, Tab),+ ?assertEqual(Owner, ets:info(Foreign, owner)),+ Owner ! stop,+ ets:delete(Tab)+ end).++repair_pass_is_bounded_by_chunk_test() ->+ Tab = ets:new(bounded_members, [set, public]),+ try+ MemberMap = maps:from_list([{N, test_member(N)} || N <- lists:seq(1, 5)]),+ ?assertEqual(continue, run_pass(Tab, MemberMap, 2)),+ ?assertEqual(2, ets:info(Tab, size)),+ ?assertEqual(continue, run_pass(Tab, MemberMap, 2)),+ ?assertEqual(4, ets:info(Tab, size)),+ ?assertEqual(complete, run_pass(Tab, MemberMap, 2)),+ ?assertEqual(5, ets:info(Tab, size))+ after+ ets:delete(Tab)+ end.++repair_picks_up_member_added_mid_build_test() ->+ Tab = ets:new(midbuild_members, [set, public]),+ try+ Map1 = #{1 => test_member(1), 2 => test_member(2)},+ ?assertEqual(continue, run_pass(Tab, Map1, 1)),+ Map2 = Map1#{3 => test_member(3)},+ ?assertEqual(continue, run_pass(Tab, Map2, 1)),+ ?assertEqual(continue, run_pass(Tab, Map2, 1)),+ ?assertEqual(complete, run_pass(Tab, Map2, 1)),+ ?assertEqual([1, 2, 3], table_ids(Tab))+ after+ ets:delete(Tab)+ end.++repair_deletes_member_removed_mid_build_test() ->+ Tab = ets:new(midremove_members, [set, public]),+ try+ Map1 = #{1 => test_member(1), 2 => test_member(2)},+ ?assertEqual(continue, run_pass(Tab, Map1, 2)),+ ?assertEqual(2, ets:info(Tab, size)),+ ?assertEqual(complete, run_pass(Tab, #{1 => test_member(1)}, 2)),+ ?assertEqual([{1, test_member(1)}], ets:tab2list(Tab))+ after+ ets:delete(Tab)+ end.++repair_abandons_after_tick_cap_test() ->+ ok = drain_mailbox(),+ with_self_heal_enabled(fun() ->+ Tab = ets:new(ticks_members, [set, public]),+ State0 = with_repair_key(+ repair_state(#{1 => test_member(1)}), Tab, ?MAX_MEMBER_ETS_REPAIR_TICKS+ ),+ State1 = maybe_repair(State0),+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),+ ?assertEqual(1, maps:get(?ATTEMPTS_KEY, State1)),+ ?assertEqual(undefined, ets:info(Tab, size)),+ ?assertEqual(undefined, published_table(State1))+ end).++repair_abandons_when_staging_table_is_dead_test() ->+ ok = drain_mailbox(),+ with_self_heal_enabled(fun() ->+ Tab = ets:new(dying_members, [set, public]),+ true = ets:delete(Tab),+ State0 = with_repair_key(repair_state(#{1 => test_member(1)}), Tab, 1),+ State1 = maybe_repair(State0),+ ?assertNot(maps:is_key(?REPAIR_KEY, State1)),+ ?assertEqual(1, maps:get(?ATTEMPTS_KEY, State1)),+ ?assertEqual(undefined, published_table(State1))+ end).++repair_stops_after_attempt_cap_test() ->+ State0 = (repair_state(#{1 => test_member(1)}))#{+ ?ATTEMPTS_KEY => ?MAX_MEMBER_ETS_REPAIR_ATTEMPTS+ },+ ?assertEqual(State0, maybe_repair(State0)).++oversized_member_map_is_refused_and_counted_test() ->+ State0 = repair_state(#{1 => test_member(1)}),+ State1 = refuse_oversized(?MAX_MEMBER_ETS_REPAIR_MEMBERS + 1, State0),+ ?assertEqual(?MAX_MEMBER_ETS_REPAIR_ATTEMPTS, maps:get(?ATTEMPTS_KEY, State1)),+ ?assertEqual(State1, start_repair(State1)).++mailbox_guard_bound_test() ->+ ?assert(mailbox_within_bound(0)),+ ?assert(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX - 1)),+ ?assertNot(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX)),+ ?assertNot(mailbox_within_bound(?MAX_MEMBER_ETS_REPAIR_MAILBOX + 1)).++repair_skips_pass_when_mailbox_is_deep_test() ->+ with_self_heal_enabled(fun() ->+ Self = self(),+ Sent = lists:seq(1, ?MAX_MEMBER_ETS_REPAIR_MAILBOX),+ lists:foreach(fun(N) -> Self ! {mailbox_guard, N} end, Sent),+ State = repair_state(#{1 => test_member(1)}),+ ?assertEqual(State, maybe_repair(State)),+ drain_mailbox_guard(?MAX_MEMBER_ETS_REPAIR_MAILBOX)+ end).++members_table_healthy_rejects_non_references_test() ->+ ?assertNot(members_table_healthy(undefined)),+ ?assertNot(members_table_healthy(not_a_table)),+ ?assertNot(members_table_healthy(make_ref())).++%% repair_unhealthy/1 bails out when the CURRENT process mailbox exceeds+%% ?MAX_MEMBER_ETS_REPAIR_MAILBOX, so a test that leaves the eunit process's+%% mailbox polluted by an earlier test silently exercises the bail-out branch+%% instead of the repair it means to assert.+drain_mailbox() ->+ receive+ _Any -> drain_mailbox()+ after 0 -> ok+ end.++%% Self-heal defaults to off (production enables it via config), so a test that+%% exercises the actual repair pass must arm it and always disarm it afterwards -+%% left on, it would make every other test's update_counts/1 call in this shared+%% eunit VM attempt a real repair too.+with_self_heal_enabled(Fun) ->+ ok = application:set_env(fluxer_gateway, guild_members_table_self_heal, true),+ try+ Fun()+ after+ application:unset_env(fluxer_gateway, guild_members_table_self_heal)+ end.++repair_state(MemberMap) ->+ #{+ id => 42,+ data => #{<<"members">> => MemberMap, members_normalized => MemberMap}+ }.++repair_state_with_table(MemberMap, Tab) ->+ State = repair_state(MemberMap),+ Data = maps:get(data, State),+ State#{data => Data#{members_ets => Tab}}.++with_repair_key(State, Tab, Ticks) ->+ State#{?REPAIR_KEY => #{tab => Tab, ticks => Ticks}}.++published_table(State) ->+ maps:get(members_ets, maps:get(data, State, #{}), undefined).++test_member(UserId) ->+ #{<<"user">> => #{<<"id">> => UserId}}.++table_ids(Tab) ->+ lists:sort([Id || {Id, _} <- ets:tab2list(Tab)]).++foreign_members_table() ->+ Self = self(),+ Owner = spawn(fun() -> foreign_table_owner(Self) end),+ receive+ {foreign_members_table, Tab} -> {Owner, Tab}+ after 5000 -> error(foreign_table_timeout)+ end.++foreign_table_owner(Parent) ->+ Parent ! {foreign_members_table, ets:new(foreign_members, [set, public])},+ receive+ stop -> ok+ after 30000 -> ok+ end.++drain_mailbox_guard(0) ->+ ok;+drain_mailbox_guard(N) ->+ receive+ {mailbox_guard, _} -> drain_mailbox_guard(N - 1)+ after 0 -> ok+ end.++-endif.diff --git a/fluxer_gateway/src/guild/guild_mutual_online.erl b/fluxer_gateway/src/guild/guild_mutual_online.erlindex e8855f13f..4c4350755 100644--- a/fluxer_gateway/src/guild/guild_mutual_online.erl+++ b/fluxer_gateway/src/guild/guild_mutual_online.erl@@ -13,6 +13,8 @@-type user_id() :: integer().-type guild_state() :: map().+-type viewable_index() :: #{user_id() => map()}.+-type index_ctx() :: {map(), viewable_index(), guild_state()}.-spec compute_count(user_id() | term(), guild_state()) -> non_neg_integer().compute_count(UserId, State) when is_integer(UserId), UserId > 0 ->@@ -37,7 +39,7 @@ slow_count(UserId, State) ->true ->self_online_count(UserId, State);false ->- count_mutually_visible(UserId, ViewerSet, State)+ count_mutually_visible_indexed(UserId, ViewerSet, State)end.-spec self_online_count(user_id(), guild_state()) -> non_neg_integer().@@ -47,38 +49,120 @@ self_online_count(UserId, State) ->false -> 0end.--spec count_mutually_visible(user_id(), sets:set(), guild_state()) -> non_neg_integer().-count_mutually_visible(UserId, ViewerSet, State) ->+-spec count_mutually_visible_indexed(user_id(), sets:set(), guild_state()) ->+ non_neg_integer().+count_mutually_visible_indexed(UserId, ViewerSet, State) ->Tab = maps:get(member_presence, State),+ Ctx = {viewer_channel_map(ViewerSet), build_viewable_index(State), State},ets:foldl(fun({OtherUserId, Presence}, Acc) ->- count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc)+ count_online_member_indexed(UserId, Ctx, OtherUserId, Presence, Acc)end,0,Tab).--spec count_online_member(- user_id(), sets:set(), guild_state(), term(), term(), non_neg_integer()+-spec count_online_member_indexed(+ user_id(), index_ctx(), term(), term(), non_neg_integer()) -> non_neg_integer().-count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) when+count_online_member_indexed(UserId, Ctx, OtherUserId, Presence, Acc) whenis_integer(OtherUserId), is_map(Presence), OtherUserId > 0->case is_online(Presence) offalse -> Acc;true when OtherUserId =:= UserId -> Acc + 1;- true -> count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc)+ true -> count_if_mutually_visible_indexed(OtherUserId, Ctx, Acc)end;-count_online_member(_UserId, _ViewerSet, _State, _OtherUserId, _Presence, Acc) ->+count_online_member_indexed(_UserId, _Ctx, _OtherUserId, _Presence, Acc) ->Acc.--spec count_if_mutually_visible(user_id(), sets:set(), guild_state(), non_neg_integer()) ->+-spec count_if_mutually_visible_indexed(user_id(), index_ctx(), non_neg_integer()) ->non_neg_integer().-count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) ->- OtherSet = guild_visibility:viewable_channel_set(OtherUserId, State),- case sets:is_empty(sets:intersection(ViewerSet, OtherSet)) of+count_if_mutually_visible_indexed(OtherUserId, Ctx, Acc) ->+ case shares_viewable_channel(OtherUserId, Ctx) of+ true -> Acc + 1;+ false -> Acc+ end.++-spec shares_viewable_channel(user_id(), index_ctx()) -> boolean().+shares_viewable_channel(OtherUserId, {ViewerMap, Index, State}) ->+ case maps:find(OtherUserId, Index) of+ {ok, OtherMap} -> maps_share_any_key(OtherMap, ViewerMap);+ error -> channel_list_shares_any(OtherUserId, ViewerMap, State)+ end.++-spec channel_list_shares_any(user_id(), map(), guild_state()) -> boolean().+channel_list_shares_any(OtherUserId, ViewerMap, State) ->+ Channels = guild_visibility:get_user_viewable_channels(OtherUserId, State),+ lists:any(fun(ChannelId) -> maps:is_key(ChannelId, ViewerMap) end, Channels).++-spec viewer_channel_map(sets:set()) -> map().+viewer_channel_map(ViewerSet) ->+ sets:fold(fun(ChannelId, Acc) -> Acc#{ChannelId => true} end, #{}, ViewerSet).++-spec build_viewable_index(guild_state()) -> viewable_index().+build_viewable_index(State) ->+ build_index_from_sessions(maps:get(sessions, State, #{})).++-spec build_index_from_sessions(term()) -> viewable_index().+build_index_from_sessions(Sessions) when is_map(Sessions) ->+ build_index_iter(maps:iterator(Sessions), #{});+build_index_from_sessions(_) ->+ #{}.++-spec build_index_iter(maps:iterator(), viewable_index()) -> viewable_index().+build_index_iter(Iterator, Acc) ->+ case maps:next(Iterator) of+ none ->+ Acc;+ {_, SessionData, Next} when is_map(SessionData) ->+ build_index_iter(Next, index_session(SessionData, Acc));+ {_, _, Next} ->+ build_index_iter(Next, Acc)+ end.++-spec index_session(map(), viewable_index()) -> viewable_index().+index_session(SessionData, Acc) ->+ SessionUserId = maps:get(user_id, SessionData, undefined),+ ViewableChannels = maps:get(viewable_channels, SessionData, undefined),+ index_session_entry(SessionUserId, ViewableChannels, Acc).++-spec index_session_entry(term(), term(), viewable_index()) -> viewable_index().+index_session_entry(UserId, ViewableChannels, Acc) when+ is_integer(UserId), is_map(ViewableChannels)+->+ put_first_session_map(UserId, ViewableChannels, Acc);+index_session_entry(_, _, Acc) ->+ Acc.++-spec put_first_session_map(user_id(), map(), viewable_index()) -> viewable_index().+put_first_session_map(UserId, ViewableChannels, Acc) ->+ case maps:is_key(UserId, Acc) oftrue -> Acc;- false -> Acc + 1+ false -> Acc#{UserId => ViewableChannels}+ end.++-spec maps_share_any_key(map(), map()) -> boolean().+maps_share_any_key(MapA, MapB) ->+ {Smaller, Larger} =+ case map_size(MapA) =< map_size(MapB) of+ true -> {MapA, MapB};+ false -> {MapB, MapA}+ end,+ maps_share_any_key_iter(maps:iterator(Smaller), Larger).++-spec maps_share_any_key_iter(maps:iterator(), map()) -> boolean().+maps_share_any_key_iter(Iterator, LargerMap) ->+ case maps:next(Iterator) of+ none -> false;+ {Key, _, NextIterator} -> key_matches_or_continue(Key, NextIterator, LargerMap)+ end.++-spec key_matches_or_continue(term(), maps:iterator(), map()) -> boolean().+key_matches_or_continue(Key, NextIterator, LargerMap) ->+ case maps:is_key(Key, LargerMap) of+ true -> true;+ false -> maps_share_any_key_iter(NextIterator, LargerMap)end.-spec is_self_online(user_id(), guild_state()) -> boolean().@@ -258,6 +342,69 @@ slow_path_returns_zero_when_viewer_offline_and_no_channels_test() ->},?assertEqual(0, compute_count(10, State)).+index_matches_scan_without_sessions_test() ->+ State = mutual_visibility_state(1, 5000),+ ViewerSet = guild_visibility:viewable_channel_set(10, State),+ Expected = reference_count_mutually_visible(10, ViewerSet, State),+ ?assertEqual(2, Expected),+ ?assertEqual(Expected, compute_count(10, State)).++index_matches_scan_with_cached_session_channels_test() ->+ Base = mutual_visibility_state(1, 5000),+ State = Base#{sessions => cached_viewable_sessions()},+ ViewerSet = guild_visibility:viewable_channel_set(10, State),+ Expected = reference_count_mutually_visible(10, ViewerSet, State),+ ?assertEqual(3, Expected),+ ?assertEqual(Expected, compute_count(10, State)).++index_counts_with_cached_session_channels_test() ->+ Base = mutual_visibility_state(1, 5000),+ State = Base#{sessions => cached_viewable_sessions()},+ ?assertEqual(3, compute_count(10, State)).++cached_viewable_sessions() ->+ #{+ <<"s20a">> => #{user_id => 20, viewable_channels => undefined},+ <<"s20b">> => #{user_id => 20, viewable_channels => #{100 => true}},+ <<"s40">> => #{user_id => 40, viewable_channels => #{101 => true}}+ }.++-spec reference_count_mutually_visible(user_id(), sets:set(), guild_state()) ->+ non_neg_integer().+reference_count_mutually_visible(UserId, ViewerSet, State) ->+ Tab = maps:get(member_presence, State),+ ets:foldl(+ fun({OtherUserId, Presence}, Acc) ->+ reference_count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc)+ end,+ 0,+ Tab+ ).++-spec reference_count_online_member(+ user_id(), sets:set(), guild_state(), term(), term(), non_neg_integer()+) -> non_neg_integer().+reference_count_online_member(UserId, ViewerSet, State, OtherUserId, Presence, Acc) when+ is_integer(OtherUserId), is_map(Presence), OtherUserId > 0+->+ case is_online(Presence) of+ false -> Acc;+ true when OtherUserId =:= UserId -> Acc + 1;+ true -> reference_count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc)+ end;+reference_count_online_member(_UserId, _ViewerSet, _State, _OtherUserId, _Presence, Acc) ->+ Acc.++-spec reference_count_if_mutually_visible(+ user_id(), sets:set(), guild_state(), non_neg_integer()+) -> non_neg_integer().+reference_count_if_mutually_visible(OtherUserId, ViewerSet, State, Acc) ->+ OtherSet = guild_visibility:viewable_channel_set(OtherUserId, State),+ case sets:is_empty(sets:intersection(ViewerSet, OtherSet)) of+ true -> Acc;+ false -> Acc + 1+ end.+make_presence_tab(Map) ->Tab = ets:new(test_member_presence, [set, public]),maps:foreach(fun(K, V) -> ets:insert(Tab, {K, V}) end, Map),diff --git a/fluxer_gateway/src/guild/guild_passive_sync.erl b/fluxer_gateway/src/guild/guild_passive_sync.erlindex 1de624378..1275b261b 100644--- a/fluxer_gateway/src/guild/guild_passive_sync.erl+++ b/fluxer_gateway/src/guild/guild_passive_sync.erl@@ -15,6 +15,7 @@-export_type([guild_state/0, channel_id/0, last_message_id/0, version/0, voice_state/0]).-define(PASSIVE_SYNC_INTERVAL, 30000).+-define(LARGE_GUILD_MEMBER_COUNT, 250).-type guild_state() :: map().-type channel_id() :: binary().@@ -30,40 +31,53 @@ schedule_passive_sync(State) ->-spec handle_passive_sync(guild_state()) -> {noreply, guild_state()}.handle_passive_sync(State) ->GuildId = maps:get(id, State),- Sessions = maps:get(sessions, State, #{}),- Data = maps:get(data, State, #{}),- MemberCount = maps:get(member_count, State, undefined),- VoiceStates = maps:get(voice_states, State, #{}),- _ = spawn(fun() ->- send_passive_updates(- GuildId, Sessions, passive_sync_state(State, Data, VoiceStates), MemberCount- )- end),+ ok = maybe_spawn_passive_updates(GuildId, State),_ = schedule_passive_sync(State),{noreply, State}.+%% send_passive_updates/4 keeps no session unless is_large_guild/1 holds, so for any other+%% guild the spawned child returns without touching a session, a payload or the registry.+-spec maybe_spawn_passive_updates(integer(), guild_state()) -> ok.+maybe_spawn_passive_updates(GuildId, State) ->+ case is_large_guild(maps:get(member_count, State, undefined)) of+ false -> ok;+ true -> spawn_passive_updates(GuildId, State)+ end.++-spec spawn_passive_updates(integer(), guild_state()) -> ok.+spawn_passive_updates(GuildId, State) ->+ _ = spawn(fun() -> send_passive_updates_for_state(GuildId, State) end),+ ok.+-spec send_passive_updates_to_sessions(guild_state()) -> guild_state().send_passive_updates_to_sessions(State) ->- GuildId = maps:get(id, State),+ ok = send_passive_updates_for_state(maps:get(id, State), State),+ State.++-spec send_passive_updates_for_state(integer(), guild_state()) -> ok.+send_passive_updates_for_state(GuildId, State) ->Sessions = maps:get(sessions, State, #{}),Data = maps:get(data, State, #{}),MemberCount = maps:get(member_count, State, undefined),VoiceStates = maps:get(voice_states, State, #{}),send_passive_updates(GuildId, Sessions, passive_sync_state(State, Data, VoiceStates), MemberCount- ),- State.+ ).-spec passive_sync_state(guild_state(), map(), map()) -> guild_state().passive_sync_state(State, Data, VoiceStates) ->State#{data => Data, voice_states => VoiceStates}.+-spec is_large_guild(term()) -> boolean().+is_large_guild(MemberCount) ->+ is_integer(MemberCount) andalso MemberCount > ?LARGE_GUILD_MEMBER_COUNT.+-spec send_passive_updates(integer(), map(), guild_state(), non_neg_integer() | undefined) ->ok.send_passive_updates(GuildId, Sessions, State, MemberCount) ->Data = maps:get(data, State, #{}),Channels = guild_data_index:channel_list(Data),- IsLargeGuild = is_integer(MemberCount) andalso MemberCount > 250,+ IsLargeGuild = is_large_guild(MemberCount),PassiveSessions = maps:filter(fun(_SessionId, SessionData) ->IsLargeGuild andalso session_passive:is_passive(GuildId, SessionData)@@ -327,3 +341,129 @@ snowflake_binary(FieldName, Value) ->{ok, Id} -> integer_to_binary(Id);{error, _, _} -> undefinedend.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++is_large_guild_matches_previous_inline_threshold_test() ->+ lists:foreach(+ fun(MemberCount) ->+ Expected = is_integer(MemberCount) andalso MemberCount > 250,+ ?assertEqual(Expected, is_large_guild(MemberCount))+ end,+ [undefined, 0, 1, 249, 250, 251, 1000000]+ ).++skipped_spawn_drops_no_session_the_old_filter_kept_test() ->+ GuildId = 4242,+ lists:foreach(+ fun({MemberCount, Sessions}) ->+ Kept = reference_passive_sessions(GuildId, Sessions, MemberCount),+ ?assert(is_large_guild(MemberCount) orelse map_size(Kept) =:= 0)+ end,+ passive_gate_cases()+ ),+ Large = reference_passive_sessions(GuildId, passive_test_sessions(), 251),+ ?assertEqual(1, map_size(Large)).++reference_passive_sessions(GuildId, Sessions, MemberCount) ->+ IsLargeGuild = is_integer(MemberCount) andalso MemberCount > 250,+ maps:filter(+ fun(_SessionId, SessionData) ->+ IsLargeGuild andalso session_passive:is_passive(GuildId, SessionData)+ end,+ Sessions+ ).++passive_gate_cases() ->+ Sessions = passive_test_sessions(),+ [+ {undefined, #{}},+ {undefined, Sessions},+ {0, Sessions},+ {250, Sessions},+ {251, Sessions},+ {251, #{}},+ {not_a_count, Sessions}+ ].++passive_test_sessions() ->+ passive_test_sessions(<<"passive-session">>).++passive_test_sessions(SessionId) ->+ #{+ SessionId => #{+ session_id => SessionId,+ user_id => 1130650140672000000,+ pid => self(),+ active_guilds => sets:new(),+ bot => false+ }+ }.++handle_passive_sync_stays_silent_below_threshold_test() ->+ flush_passive_dispatches(),+ ok = passive_sync_registry:init(),+ GuildId = 1427764661718740995,+ SessionId = <<"small-guild-session">>,+ State = passive_sync_test_state(GuildId, SessionId, 250),+ ?assertMatch({noreply, State}, handle_passive_sync(State)),+ ?assertEqual(no_dispatch, receive_passive_dispatch(200)),+ RegState = passive_sync_registry:lookup(SessionId, GuildId),+ ?assertEqual(#{}, maps:get(previous_passive_updates, RegState)).++handle_passive_sync_still_dispatches_above_threshold_test() ->+ flush_passive_dispatches(),+ ok = passive_sync_registry:init(),+ GuildId = 1427764661718740996,+ SessionId = <<"large-guild-session">>,+ ChannelId = passive_test_channel_id(),+ MessageId = passive_test_message_id(),+ State = passive_sync_test_state(GuildId, SessionId, 251),+ ?assertMatch({noreply, State}, handle_passive_sync(State)),+ Payload = receive_passive_dispatch(5000),+ ?assertEqual(+ #{integer_to_binary(ChannelId) => integer_to_binary(MessageId)},+ maps:get(<<"channels">>, Payload)+ ).++passive_sync_test_state(GuildId, SessionId, MemberCount) ->+ UserId = 1130650140672000000,+ Channel = #{+ <<"id">> => passive_test_channel_id(),+ <<"last_message_id">> => passive_test_message_id()+ },+ #{+ id => GuildId,+ member_count => MemberCount,+ sessions => passive_test_sessions(SessionId),+ data => #{+ <<"guild">> => #{<<"id">> => GuildId, <<"owner_id">> => UserId},+ <<"channels">> => [Channel],+ <<"members">> => [#{<<"user">> => #{<<"id">> => UserId}, <<"roles">> => []}],+ <<"roles">> => []+ },+ voice_states => #{}+ }.++passive_test_channel_id() ->+ 1497639278555484216.++passive_test_message_id() ->+ 1500000000000000000.++receive_passive_dispatch(Timeout) ->+ receive+ {'$gen_cast', {dispatch, passive_updates, Payload}} when is_map(Payload) ->+ Payload+ after Timeout ->+ no_dispatch+ end.++flush_passive_dispatches() ->+ case receive_passive_dispatch(0) of+ no_dispatch -> ok;+ _ -> flush_passive_dispatches()+ end.++-endif.diff --git a/fluxer_gateway/src/guild/guild_permission_cache.erl b/fluxer_gateway/src/guild/guild_permission_cache.erlindex a19a3a7ab..52ee39735 100644--- a/fluxer_gateway/src/guild/guild_permission_cache.erl+++ b/fluxer_gateway/src/guild/guild_permission_cache.erl@@ -30,6 +30,7 @@-export_type([guild_id/0, user_id/0, channel_id/0, guild_state/0, guild_data/0]).-define(TABLE, guild_permission_cache).+-define(STRIPPED_MEMBERS_MEMO, guild_permission_cache_stripped_members).-spec put_state(guild_state()) -> ok.put_state(State) when is_map(State) ->@@ -134,7 +135,7 @@ ensure_table() ->-spec strip_data(guild_data()) -> guild_data().strip_data(Data) when is_map(Data) ->Guild = strip_guild(maps:get(<<"guild">>, Data, #{})),- Members = strip_members(maps:get(<<"members">>, Data, #{})),+ Members = memoised_strip_members(maps:get(<<"members">>, Data, #{})),Roles = strip_roles(maps:get(<<"roles">>, Data, [])),Channels = strip_channels(maps:get(<<"channels">>, Data, [])),ChannelIndex = strip_channel_index(maps:get(<<"channel_index">>, Data, #{})),@@ -170,6 +171,54 @@ strip_guild(Guild) when is_map(Guild) ->strip_guild(_) ->#{}.+%% Keyed on the IDENTITY of the source term, never on its value: a hit means this process+%% already stripped that exact term, and strip_members/1 is pure, so the stored result is+%% that term's result. Every write to a member map yields a new term and so cannot hit, and+%% only a completed strip is stored, so a term whose strip raises still raises.+-spec memoised_strip_members(term()) -> map().+memoised_strip_members(Members) ->+ case strip_members_memo_enabled() of+ true -> strip_members_memoised(Members);+ false -> strip_members_unmemoised(Members)+ end.++-spec strip_members_unmemoised(term()) -> map().+strip_members_unmemoised(Members) ->+ _ = erlang:erase(?STRIPPED_MEMBERS_MEMO),+ strip_members(Members).++-spec strip_members_memoised(term()) -> map().+strip_members_memoised(Members) ->+ case erlang:get(?STRIPPED_MEMBERS_MEMO) of+ {Source, Stripped} when is_map(Stripped) ->+ reuse_stripped_members(Members, Source, Stripped);+ _ ->+ store_stripped_members(Members)+ end.++-spec reuse_stripped_members(term(), term(), map()) -> map().+reuse_stripped_members(Members, Source, Stripped) ->+ case erts_debug:same(Members, Source) of+ true -> Stripped;+ false -> store_stripped_members(Members)+ end.++%% Dropped before the new one is built so a member map that shares nothing with the+%% previous one, a wholesale rebuild, is never held twice on the guild heap at once.+-spec store_stripped_members(term()) -> map().+store_stripped_members(Members) ->+ _ = erlang:erase(?STRIPPED_MEMBERS_MEMO),+ Stripped = strip_members(Members),+ _ = erlang:put(?STRIPPED_MEMBERS_MEMO, {Members, Stripped}),+ Stripped.++-spec strip_members_memo_enabled() -> boolean().+strip_members_memo_enabled() ->+ case application:get_env(fluxer_gateway, permission_cache_strip_memo_enabled, true) of+ false -> false;+ _ -> true+ end.+-spec strip_members(map() | list() | term()) -> map().strip_members(Members) when is_map(Members) ->maps:map(fun(_UserId, Member) -> strip_member(Member) end, Members);@@ -191,22 +240,30 @@ strip_member_entry(_, Acc) ->strip_member(Member) when is_map(Member) ->User = maps:get(<<"user">>, Member, #{}),StrippedUser = strip_user(User),- Roles = snowflake_id:parse_list(maps:get(<<"roles">>, Member, [])),+ Roles = strip_role_ids(maps:get(<<"roles">>, Member, [])),Base = #{<<"user">> => StrippedUser,<<"roles">> => Roles},- copy_optional_member_fields([<<"communication_disabled_until">>], Member, Base);+ copy_if_present(<<"communication_disabled_until">>, Member, Base);strip_member(_) ->#{}.--spec copy_optional_member_fields([binary()], map(), map()) -> map().-copy_optional_member_fields(Keys, Source, Acc) ->- lists:foldl(- fun(Key, Current) -> copy_if_present(Key, Source, Current) end,- Acc,- Keys- ).+%% snowflake_id:parse/1 returns a positive integer unchanged, so a roles list that already+%% holds only those is its own parse_list/1 result and does not need to be rebuilt.+-spec strip_role_ids(term()) -> list().+strip_role_ids(Roles) when is_list(Roles) ->+ case all_role_ids_parsed(Roles) of+ true -> Roles;+ false -> snowflake_id:parse_list(Roles)+ end;+strip_role_ids(Roles) ->+ snowflake_id:parse_list(Roles).++-spec all_role_ids_parsed(term()) -> boolean().+all_role_ids_parsed([Id | Rest]) when is_integer(Id), Id > 0 -> all_role_ids_parsed(Rest);+all_role_ids_parsed([]) -> true;+all_role_ids_parsed(_) -> false.-spec copy_if_present(binary(), map(), map()) -> map().copy_if_present(Key, Source, Current) ->@@ -327,4 +384,263 @@ strip_member_preserves_communication_disabled_until_test() ->ok = delete(GuildId)end.+strip_member_matches_reference_implementation_test() ->+ lists:foreach(+ fun(Member) ->+ ?assertEqual(reference_strip_member(Member), strip_member(Member))+ end,+ strip_member_cases()+ ).++strip_member_cases() ->+ Until = <<"2026-05-09T22:00:00.000Z">>,+ User = #{<<"id">> => 5, <<"username">> => <<"dropped">>},+ [+ #{<<"user">> => User, <<"roles">> => [7, 9]},+ #{<<"user">> => User, <<"roles">> => [<<"7">>, 9]},+ #{<<"user">> => User, <<"roles">> => [<<"7">>, <<"9">>]},+ #{<<"user">> => User, <<"roles">> => []},+ #{<<"user">> => User},+ #{<<"roles">> => [7]},+ #{<<"user">> => User, <<"roles">> => [7], <<"communication_disabled_until">> => Until},+ #{<<"user">> => User, <<"roles">> => [7], <<"nick">> => <<"dropped">>},+ #{<<"user">> => #{}, <<"roles">> => undefined},+ #{<<"user">> => not_a_map, <<"roles">> => null},+ not_a_member+ ].++reference_strip_member(Member) when is_map(Member) ->+ StrippedUser = strip_user(maps:get(<<"user">>, Member, #{})),+ Roles = snowflake_id:parse_list(maps:get(<<"roles">>, Member, [])),+ Base = #{<<"user">> => StrippedUser, <<"roles">> => Roles},+ lists:foldl(+ fun(Key, Current) -> copy_if_present(Key, Member, Current) end,+ Base,+ [<<"communication_disabled_until">>]+ );+reference_strip_member(_) ->+ #{}.++strip_member_rejects_unparsed_role_ids_test() ->+ ?assertError({invalid_snowflake, 0}, strip_member(#{<<"roles">> => [0]})),+ ?assertError({invalid_snowflake, -3}, strip_member(#{<<"roles">> => [-3]})),+ ?assertError({invalid_snowflake, <<"0">>}, strip_member(#{<<"roles">> => [<<"0">>]})),+ ?assertError({invalid_snowflake, bad}, strip_member(#{<<"roles">> => [7, bad]})),+ ?assertError({invalid_snowflake_list, oops}, strip_member(#{<<"roles">> => oops})).++strip_members_keeps_already_parsed_role_ids_test() ->+ GuildId = 905,+ UserId = 906,+ Member = #{<<"user">> => #{<<"id">> => UserId}, <<"roles">> => [42, 43]},+ Data = #{+ <<"guild">> => #{<<"owner_id">> => <<"1">>},+ <<"roles">> => [],+ <<"members">> => #{UserId => Member},+ <<"channels">> => []+ },+ ok = put_data(GuildId, Data),+ try+ {ok, #{} = MemberData} = get_member(GuildId, UserId),+ ?assertEqual([42, 43], maps:get(<<"roles">>, MemberData)),+ ?assertEqual(#{<<"id">> => UserId}, maps:get(<<"user">>, MemberData))+ after+ ok = delete(GuildId)+ end.++memo_reset() ->+ erlang:erase(?STRIPPED_MEMBERS_MEMO).++memo_member(Id, Roles) ->+ #{+ <<"user">> => #{<<"id">> => integer_to_binary(Id), <<"username">> => <<"dropped">>},+ <<"roles">> => Roles,+ <<"nick">> => <<"dropped">>+ }.++memo_members(Ids) ->+ maps:from_list([{Id, memo_member(Id, [<<"42">>])} || Id <- Ids]).++%% strip_members/1 as it read before the memo, over the pre-shortcut strip_member/1.+reference_strip_members(Members) when is_map(Members) ->+ maps:map(fun(_UserId, Member) -> reference_strip_member(Member) end, Members);+reference_strip_members(Members) when is_list(Members) ->+ lists:foldl(fun reference_strip_member_entry/2, #{}, Members);+reference_strip_members(_) ->+ #{}.++reference_strip_member_entry(Member, Acc) when is_map(Member) ->+ case get_member_user_id(Member) of+ undefined -> Acc;+ UserId -> Acc#{UserId => reference_strip_member(Member)}+ end;+reference_strip_member_entry(_, Acc) ->+ Acc.++assert_memo_matches_reference(Members) ->+ Expected = reference_strip_members(Members),+ ?assertEqual(Expected, memoised_strip_members(Members)),+ ?assertEqual(Expected, memoised_strip_members(Members)).++memo_members_cases() ->+ [+ #{},+ memo_members([1]),+ memo_members([1, 2, 3]),+ #{7 => memo_member(7, [])},+ [memo_member(4, [<<"42">>]), memo_member(5, [7, 9])],+ [memo_member(6, []), not_a_member],+ [],+ not_a_member_map,+ undefined+ ].++memoised_strip_members_matches_reference_test() ->+ memo_reset(),+ try+ lists:foreach(fun assert_memo_matches_reference/1, memo_members_cases())+ after+ memo_reset()+ end.++memo_reuses_the_stripped_term_for_an_identical_members_term_test() ->+ memo_reset(),+ try+ Members = memo_members([1, 2]),+ First = memoised_strip_members(Members),+ Second = memoised_strip_members(Members),+ ?assertEqual(reference_strip_members(Members), First),+ ?assert(erts_debug:same(First, Second)),+ ?assertEqual({Members, First}, erlang:get(?STRIPPED_MEMBERS_MEMO))+ after+ memo_reset()+ end.++memo_recomputes_for_an_equal_but_distinct_members_term_test() ->+ memo_reset(),+ try+ Members = memo_members([1, 2]),+ Copy = binary_to_term(term_to_binary(Members)),+ Stripped = memoised_strip_members(Members),+ ?assertNot(erts_debug:same(Members, Copy)),+ ?assertEqual(Stripped, memoised_strip_members(Copy)),+ {Source, _Stripped} = erlang:get(?STRIPPED_MEMBERS_MEMO),+ ?assert(erts_debug:same(Copy, Source))+ after+ memo_reset()+ end.++memo_changed_member_maps(Members) ->+ [+ Members#{2 => memo_member(2, [<<"7">>])},+ Members#{3 => memo_member(3, [<<"42">>])},+ maps:remove(1, Members),+ maps:map(fun(Id, _Member) -> memo_member(Id, []) end, Members)+ ].++memo_follows_every_change_to_the_member_map_test() ->+ memo_reset(),+ try+ Members = memo_members([1, 2]),+ _Stripped = memoised_strip_members(Members),+ lists:foreach(+ fun assert_memo_matches_reference/1,+ memo_changed_member_maps(Members)+ )+ after+ memo_reset()+ end.++memo_never_suppresses_an_invalid_role_id_test() ->+ memo_reset(),+ try+ Good = memo_members([1]),+ Stripped = memoised_strip_members(Good),+ Bad = #{2 => memo_member(2, [bad])},+ ?assertError({invalid_snowflake, bad}, memoised_strip_members(Bad)),+ ?assertEqual(undefined, erlang:get(?STRIPPED_MEMBERS_MEMO)),+ ?assertError({invalid_snowflake, bad}, memoised_strip_members(Bad)),+ ?assertEqual(Stripped, memoised_strip_members(Good)),+ ?assertEqual({Good, Stripped}, erlang:get(?STRIPPED_MEMBERS_MEMO))+ after+ memo_reset()+ end.++memo_ignores_an_unrecognised_entry_test() ->+ memo_reset(),+ try+ Members = memo_members([1]),+ Expected = reference_strip_members(Members),+ erlang:put(?STRIPPED_MEMBERS_MEMO, {Members, junk}),+ ?assertEqual(Expected, memoised_strip_members(Members)),+ erlang:put(?STRIPPED_MEMBERS_MEMO, stale_shape),+ ?assertEqual(Expected, memoised_strip_members(Members)),+ ?assertEqual({Members, Expected}, erlang:get(?STRIPPED_MEMBERS_MEMO))+ after+ memo_reset()+ end.++memo_disabled_matches_reference_and_releases_the_memo_test() ->+ memo_reset(),+ Members = memo_members([1, 2]),+ Expected = reference_strip_members(Members),+ ?assertEqual(Expected, memoised_strip_members(Members)),+ application:set_env(fluxer_gateway, permission_cache_strip_memo_enabled, false),+ try+ ?assertEqual(Expected, memoised_strip_members(Members)),+ ?assertEqual(undefined, erlang:get(?STRIPPED_MEMBERS_MEMO))+ after+ application:unset_env(fluxer_gateway, permission_cache_strip_memo_enabled),+ memo_reset()+ end.++%% strip_data/1 with the memo off is the computation this module ran before the memo.+unmemoised_strip_data(Data) ->+ application:set_env(fluxer_gateway, permission_cache_strip_memo_enabled, false),+ try+ strip_data(Data)+ after+ application:unset_env(fluxer_gateway, permission_cache_strip_memo_enabled)+ end.++memo_test_data(OwnerId) ->+ guild_data_index:normalize_data(#{+ <<"guild">> => #{<<"owner_id">> => OwnerId},+ <<"roles">> => [#{<<"id">> => <<"42">>, <<"permissions">> => <<"0">>}],+ <<"members">> => [memo_member(1, [<<"42">>]), memo_member(2, [])],+ <<"channels">> => [#{<<"id">> => <<"500">>, <<"permission_overwrites">> => []}]+ }).++strip_data_reuses_the_stripped_members_across_a_non_member_change_test() ->+ memo_reset(),+ Data = memo_test_data(<<"1">>),+ Updated = Data#{<<"guild">> => #{<<"owner_id">> => <<"9">>}},+ Expected = unmemoised_strip_data(Data),+ ExpectedUpdated = unmemoised_strip_data(Updated),+ memo_reset(),+ try+ ?assertEqual(Expected, strip_data(Data)),+ ?assertEqual(ExpectedUpdated, strip_data(Updated)),+ MembersA = maps:get(<<"members">>, strip_data(Data)),+ MembersB = maps:get(<<"members">>, strip_data(Updated)),+ ?assert(erts_debug:same(MembersA, MembersB))+ after+ memo_reset()+ end.++put_normalized_data_writes_the_same_snapshot_with_a_warm_memo_test() ->+ GuildId = 907,+ memo_reset(),+ Data = memo_test_data(<<"1">>),+ try+ ok = put_normalized_data(GuildId, Data),+ {ok, First} = get_snapshot(GuildId),+ ok = put_normalized_data(GuildId, Data),+ {ok, Second} = get_snapshot(GuildId),+ ?assertEqual(First, Second),+ ?assertEqual(#{id => GuildId, data => unmemoised_strip_data(Data)}, Second)+ after+ memo_reset(),+ ok = delete(GuildId)+ end.+-endif.diff --git a/fluxer_gateway/src/guild/guild_presence.erl b/fluxer_gateway/src/guild/guild_presence.erlindex 07719adab..dd06a85c0 100644--- a/fluxer_gateway/src/guild/guild_presence.erl+++ b/fluxer_gateway/src/guild/guild_presence.erl@@ -14,8 +14,13 @@-type member() :: map().-type user_id() :: integer().--define(HEAVY_MEMBER_DATA_KEYS, [<<"members">>, members_normalized, <<"member_role_index">>]).+%% members_sorted_ids trims with the member map it indexes: a snapshot that kept it would+%% answer sorted_member_ids/2 with ids for members the snapshot no longer carries.+-define(HEAVY_MEMBER_DATA_KEYS, [+ <<"members">>, members_normalized, <<"member_role_index">>, members_sorted_ids+]).-define(PRESENCE_SNAPSHOT_TRIM_MEMBER_THRESHOLD_DEFAULT, 5000).+-define(SESSIONS_PROJECTION_CACHE, presence_snapshot_sessions_cache).-ifdef(TEST).-include_lib("eunit/include/eunit.hrl").@@ -112,12 +117,17 @@ spawn_presence_broadcast(UserId, OldPresence, PresenceMap, OldState, NewState) -PresenceMap),{Pid, NewState2} = guild_broadcaster:ensure(NewState1),- NewSnap = build_broadcast_snapshot(NewState2),+ {NewSnap, NewState3} = build_broadcast_snapshot_cached(NewState2),guild_broadcaster:cast_presence(Pid, UserId, PresenceMap, #{}, NewSnap),- NewState2.+ NewState3.-spec build_broadcast_snapshot(guild_state()) -> map().build_broadcast_snapshot(State) ->+ {Snapshot, _State} = build_broadcast_snapshot_cached(State),+ Snapshot.++-spec build_broadcast_snapshot_cached(guild_state()) -> {map(), guild_state()}.+build_broadcast_snapshot_cached(State) ->maybe_trim_snapshot(build_base_snapshot(State), State).-spec build_base_snapshot(guild_state()) -> map().@@ -139,11 +149,11 @@ build_base_snapshot(State) ->Keys).--spec maybe_trim_snapshot(map(), guild_state()) -> map().+-spec maybe_trim_snapshot(map(), guild_state()) -> {map(), guild_state()}.maybe_trim_snapshot(Snapshot, State) ->case should_trim_snapshot(State) of- true -> trim_snapshot(Snapshot);- false -> Snapshot+ true -> trim_snapshot(Snapshot, State);+ false -> {Snapshot, State}end.-spec should_trim_snapshot(guild_state()) -> boolean().@@ -179,9 +189,9 @@ presence_snapshot_trim_member_threshold() ->_ -> ?PRESENCE_SNAPSHOT_TRIM_MEMBER_THRESHOLD_DEFAULTend.--spec trim_snapshot(map()) -> map().-trim_snapshot(Snapshot) ->- trim_snapshot_sessions(trim_snapshot_data(Snapshot)).+-spec trim_snapshot(map(), guild_state()) -> {map(), guild_state()}.+trim_snapshot(Snapshot, State) ->+ trim_snapshot_sessions(trim_snapshot_data(Snapshot), State).-spec trim_snapshot_data(map()) -> map().trim_snapshot_data(#{data := Data} = Snapshot) when is_map(Data) ->@@ -189,15 +199,33 @@ trim_snapshot_data(#{data := Data} = Snapshot) when is_map(Data) ->trim_snapshot_data(Snapshot) ->Snapshot.--spec trim_snapshot_sessions(map()) -> map().-trim_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) ->- Projected = maps:map(+-spec trim_snapshot_sessions(map(), guild_state()) -> {map(), guild_state()}.+trim_snapshot_sessions(#{sessions := Sessions} = Snapshot, State) when is_map(Sessions) ->+ {Projected, NewState} = cached_projected_sessions(Sessions, State),+ {Snapshot#{sessions => Projected}, NewState};+trim_snapshot_sessions(Snapshot, State) ->+ {Snapshot, State}.++%% Keyed by the sessions map itself, so every writer of that map invalidates the+%% projection without having to know this cache exists.+-spec cached_projected_sessions(map(), guild_state()) -> {map(), guild_state()}.+cached_projected_sessions(Sessions, State) ->+ case maps:get(?SESSIONS_PROJECTION_CACHE, State, undefined) of+ {Sessions, Projected} when is_map(Projected) -> {Projected, State};+ _ -> store_projected_sessions(Sessions, State)+ end.++-spec store_projected_sessions(map(), guild_state()) -> {map(), guild_state()}.+store_projected_sessions(Sessions, State) ->+ Projected = project_sessions(Sessions),+ {Projected, State#{?SESSIONS_PROJECTION_CACHE => {Sessions, Projected}}}.++-spec project_sessions(map()) -> map().+project_sessions(Sessions) ->+ maps:map(fun(_SessionId, SessionData) -> project_session(SessionData) end,Sessions- ),- Snapshot#{sessions => Projected};-trim_snapshot_sessions(Snapshot) ->- Snapshot.+ ).-spec project_session(term()) -> term().project_session(SessionData) when is_map(SessionData) ->@@ -440,6 +468,7 @@ snapshot_trim_test_state(Tab) -><<"members">> => #{1 => #{<<"user">> => #{<<"id">> => <<"1">>}}},members_normalized => #{1 => #{<<"user">> => #{<<"id">> => <<"1">>}}},<<"member_role_index">> => #{1 => []},+ members_sorted_ids => [1],<<"channels">> => [],<<"roles">> => []},@@ -463,6 +492,7 @@ build_broadcast_snapshot_trims_by_default_test() ->?assertNot(maps:is_key(<<"members">>, SnapData)),?assertNot(maps:is_key(members_normalized, SnapData)),?assertNot(maps:is_key(<<"member_role_index">>, SnapData)),+ ?assertNot(maps:is_key(members_sorted_ids, SnapData)),?assertEqual(Tab, maps:get(members_ets, SnapData)),?assert(maps:is_key(<<"channels">>, SnapData)),SnapSession = maps:get(<<"s1">>, maps:get(sessions, Snap)),@@ -499,4 +529,149 @@ build_broadcast_snapshot_no_trim_below_threshold_test() ->ets:delete(Tab)end.+reference_project_session(SessionData) when is_map(SessionData) ->+ maps:with([user_id, pid, viewable_channels], SessionData);+reference_project_session(SessionData) ->+ SessionData.++reference_project_sessions(Sessions) ->+ maps:map(+ fun(_SessionId, SessionData) -> reference_project_session(SessionData) end,+ Sessions+ ).++reference_trim_snapshot_sessions(#{sessions := Sessions} = Snapshot) when is_map(Sessions) ->+ Snapshot#{sessions => reference_project_sessions(Sessions)};+reference_trim_snapshot_sessions(Snapshot) ->+ Snapshot.++reference_maybe_trim_snapshot(true, Snapshot) ->+ reference_trim_snapshot_sessions(trim_snapshot_data(Snapshot));+reference_maybe_trim_snapshot(false, Snapshot) ->+ Snapshot.++reference_broadcast_snapshot(State) ->+ reference_maybe_trim_snapshot(should_trim_snapshot(State), build_base_snapshot(State)).++cache_test_session(UserId, ViewableChannels) ->+ #{+ user_id => UserId,+ pid => self(),+ viewable_channels => ViewableChannels,+ active_guilds => [42],+ pending_connect => false+ }.++cache_test_state(Sessions) ->+ #{+ id => 42,+ member_count => 10,+ data => #{<<"channels">> => [], <<"members">> => #{}},+ sessions => Sessions+ }.++assert_cached_snapshot_matches_reference(Sessions, State) ->+ NextState = State#{sessions => Sessions},+ {Snapshot, CachedState} = build_broadcast_snapshot_cached(NextState),+ ?assertEqual(reference_broadcast_snapshot(NextState), Snapshot),+ ?assertNot(maps:is_key(?SESSIONS_PROJECTION_CACHE, Snapshot)),+ CachedState.++session_projection_mutation_sequence() ->+ A = cache_test_session(1, #{100 => true}),+ B = cache_test_session(2, #{100 => true, 200 => true}),+ S1 = #{<<"a">> => A},+ S2 = S1#{<<"b">> => B},+ S3 = S2#{<<"a">> => cache_test_session(1, #{100 => true, 300 => true})},+ S4 = maps:remove(<<"b">>, S3),+ S5 = S4#{<<"a">> => (maps:get(<<"a">>, S4))#{pending_connect => true}},+ S6 = S5#{<<"c">> => not_a_map},+ [#{}, S1, S1, S2, S3, S4, S5, S6, S1, #{}].++cached_projection_matches_reference_across_mutations_test() ->+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),+ try+ lists:foldl(+ fun assert_cached_snapshot_matches_reference/2,+ cache_test_state(#{}),+ session_projection_mutation_sequence()+ )+ after+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)+ end.++cached_projection_is_reused_when_sessions_unchanged_test() ->+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),+ try+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},+ {Snap1, State1} = build_broadcast_snapshot_cached(cache_test_state(Sessions)),+ {Snap2, State2} = build_broadcast_snapshot_cached(State1),+ Projected1 = maps:get(sessions, Snap1),+ Projected2 = maps:get(sessions, Snap2),+ ?assertEqual(reference_project_sessions(Sessions), Projected1),+ ?assertEqual(Projected1, Projected2),+ ?assert(erts_debug:same(Projected1, Projected2)),+ ?assertEqual({Sessions, Projected1}, maps:get(?SESSIONS_PROJECTION_CACHE, State2))+ after+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)+ end.++cached_projection_ignores_foreign_cache_entry_test() ->+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),+ try+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},+ Stale = #{<<"z">> => cache_test_session(9, #{999 => true})},+ Foreign = {Stale, reference_project_sessions(Stale)},+ State = (cache_test_state(Sessions))#{?SESSIONS_PROJECTION_CACHE => Foreign},+ {Snapshot, _NewState} = build_broadcast_snapshot_cached(State),+ ?assertEqual(reference_broadcast_snapshot(State), Snapshot),+ ?assertEqual(reference_project_sessions(Sessions), maps:get(sessions, Snapshot))+ after+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)+ end.++cached_projection_ignores_corrupt_cache_entry_test() ->+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),+ try+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},+ State = (cache_test_state(Sessions))#{?SESSIONS_PROJECTION_CACHE => {Sessions, junk}},+ {Snapshot, _NewState} = build_broadcast_snapshot_cached(State),+ ?assertEqual(reference_project_sessions(Sessions), maps:get(sessions, Snapshot))+ after+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)+ end.++cached_projection_not_stored_when_trim_disabled_test() ->+ application:set_env(fluxer_gateway, presence_snapshot_trim_enabled, false),+ try+ Sessions = #{<<"a">> => cache_test_session(1, #{100 => true})},+ State = cache_test_state(Sessions),+ {Snapshot, NewState} = build_broadcast_snapshot_cached(State),+ ?assertEqual(reference_broadcast_snapshot(State), Snapshot),+ ?assertEqual(Sessions, maps:get(sessions, Snapshot)),+ ?assertNot(maps:is_key(?SESSIONS_PROJECTION_CACHE, NewState))+ after+ application:unset_env(fluxer_gateway, presence_snapshot_trim_enabled)+ end.++handle_bus_presence_threads_projection_cache_test() ->+ application:set_env(fluxer_gateway, presence_snapshot_trim_member_threshold, 2),+ Sessions = #{<<"s1">> => cache_test_session(1, #{100 => true})},+ State = (presence_test_state())#{member_count => 10, sessions => Sessions},+ Payload = #{+ <<"status">> => <<"online">>,+ <<"mobile">> => true,+ <<"afk">> => false,+ <<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"Alpha">>}+ },+ try+ {noreply, NewState} = handle_bus_presence(1, Payload, State),+ ?assertEqual(+ {Sessions, reference_project_sessions(Sessions)},+ maps:get(?SESSIONS_PROJECTION_CACHE, NewState)+ )+ after+ application:unset_env(fluxer_gateway, presence_snapshot_trim_member_threshold)+ end.+-endif.diff --git a/fluxer_gateway/src/guild/guild_public_online.erl b/fluxer_gateway/src/guild/guild_public_online.erlindex c118903d3..894975d6c 100644--- a/fluxer_gateway/src/guild/guild_public_online.erl+++ b/fluxer_gateway/src/guild/guild_public_online.erl@@ -15,6 +15,8 @@-type channel() :: map().-type user_id() :: integer().+-define(ALL_PERMS_VIEWER_KEY, public_online_all_perms_viewer).+-spec compute_count(guild_state()) -> non_neg_integer().compute_count(State) ->case resolve_data(State) of@@ -57,7 +59,15 @@ classify_everyone_viewable(Channels, State) ->case {OpenAcc, RestrictedAcc} of{[], []} -> empty;{[_ | _], _} -> {open, OpenAcc};- {[], _} -> {restricted, sets:from_list(RestrictedAcc)}+ {[], _} -> classify_restricted(RestrictedAcc)+ end.++-spec classify_restricted([integer()]) ->+ {open, [integer()]} | {restricted, sets:set(integer())}.+classify_restricted(RestrictedIds) ->+ case application:get_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, false) of+ true -> {open, RestrictedIds};+ _ -> {restricted, sets:from_list(RestrictedIds)}end.-spec classify_channel_visibility(@@ -200,9 +210,66 @@ build_state(Channels, Members, Roles, Presences) -><<"channels">> => Channels},member_presence => Tab,+ member_list_engine => build_engine(Presences),sessions => #{}}.+build_engine(Presences) ->+ Online = length([S || {_UserId, S} <- Presences, S =/= <<"offline">>, S =/= <<"invisible">>]),+ Ref = ets:new(test_member_list_engine, [set, public]),+ ets:insert(Ref, [{total_count, length(Presences)}, {online_count, Online}]),+ Ref.++deny_view_overwrite(Id, Type) ->+ #{+ <<"id">> => integer_to_binary(Id),+ <<"type">> => Type,+ <<"allow">> => <<"0">>,+ <<"deny">> => integer_to_binary(view_perm())+ }.++channel(ChannelId, Overwrites) ->+ #{+ <<"id">> => integer_to_binary(ChannelId),+ <<"type">> => 0,+ <<"permission_overwrites">> => Overwrites+ }.++role(RoleId) ->+ #{<<"id">> => integer_to_binary(RoleId), <<"permissions">> => <<"0">>}.++with_all_perms_viewer(Fun) ->+ Previous = application:get_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY),+ application:set_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, true),+ try+ Fun()+ after+ restore_all_perms_viewer(Previous)+ end.++restore_all_perms_viewer(undefined) ->+ application:unset_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY);+restore_all_perms_viewer({ok, Value}) ->+ application:set_env(fluxer_gateway, ?ALL_PERMS_VIEWER_KEY, Value).++verify_gate_state() ->+ VerifiedRoleId = 4242,+ Channels = [channel(400, [deny_view_overwrite(VerifiedRoleId, 0)])],+ Members = [{1, [VerifiedRoleId]}, {2, [VerifiedRoleId]}, {3, [VerifiedRoleId]}, {4, []}],+ Presences = [+ {1, <<"online">>},+ {2, <<"online">>},+ {3, <<"idle">>},+ {4, <<"online">>}+ ],+ build_state(Channels, Members, [role(VerifiedRoleId)], Presences).++user_deny_state() ->+ Channels = [channel(700, [deny_view_overwrite(42, 1)])],+ Members = [{41, []}, {42, []}, {43, []}],+ Presences = [{41, <<"online">>}, {42, <<"online">>}, {43, <<"online">>}],+ build_state(Channels, Members, [], Presences).+returns_zero_when_no_channels_test() ->State = build_state([], [], [], []),?assertEqual(0, compute_count(State)).@@ -303,4 +370,36 @@ channel_has_view_restricting_overrides_no_overwrites_test() ->Channel = #{<<"permission_overwrites">> => []},?assertNot(channel_has_view_restricting_overrides(Channel, view_perm())).+verify_gate_collapses_count_without_flag_test() ->+ ?assertEqual(1, compute_count(verify_gate_state())).++verify_gate_reports_full_count_with_flag_test() ->+ State = verify_gate_state(),+ ?assertEqual(4, with_all_perms_viewer(fun() -> compute_count(State) end)).++everyone_deny_channel_stays_excluded_with_flag_test() ->+ GuildId = guild_id(),+ Channels = [channel(500, [deny_view_overwrite(GuildId, 0)])],+ State = build_state(Channels, [{1, []}, {2, []}], [], [{1, <<"online">>}, {2, <<"idle">>}]),+ ?assertEqual(0, compute_count(State)),+ ?assertEqual(0, with_all_perms_viewer(fun() -> compute_count(State) end)).++no_channels_stays_zero_with_flag_test() ->+ State = build_state([], [{1, []}], [], [{1, <<"online">>}]),+ ?assertEqual(0, with_all_perms_viewer(fun() -> compute_count(State) end)).++plain_open_channel_unchanged_by_flag_test() ->+ Members = [{1, []}, {2, []}, {3, []}],+ Presences = [{1, <<"online">>}, {2, <<"idle">>}, {3, <<"offline">>}],+ State = build_state([channel(600, [])], Members, [], Presences),+ ?assertEqual(2, compute_count(State)),+ ?assertEqual(2, with_all_perms_viewer(fun() -> compute_count(State) end)).++user_deny_collapses_count_without_flag_test() ->+ ?assertEqual(2, compute_count(user_deny_state())).++user_deny_reports_full_count_with_flag_test() ->+ State = user_deny_state(),+ ?assertEqual(3, with_all_perms_viewer(fun() -> compute_count(State) end)).+-endif.diff --git a/fluxer_gateway/src/guild/guild_sessions.erl b/fluxer_gateway/src/guild/guild_sessions.erlindex 95a5035e2..63e3ea108 100644--- a/fluxer_gateway/src/guild/guild_sessions.erl+++ b/fluxer_gateway/src/guild/guild_sessions.erl@@ -28,6 +28,8 @@build_viewable_channel_map/1]).+-define(MAX_PERM_MEMO_ENTRIES, 8192).+-type guild_state() :: map().-type session_id() :: binary().-type user_id() :: integer().@@ -36,6 +38,8 @@-type session_data() :: map().-type sessions_map() :: #{session_id() => session_data()}.-type session_pair() :: {session_id(), session_data()}.+-type perm_memo() :: #{user_id() => non_neg_integer()}.+-type message_ctx() :: {channel_id(), binary(), session_id() | undefined, guild_state()}.-export_type([guild_state/0,session_id/0,@@ -59,7 +63,116 @@ handle_session_connect(Request, Pid, State) ->-spec handle_session_down(reference(), guild_state()) ->{noreply, guild_state()} | {stop, normal, guild_state()}.handle_session_down(Ref, State) ->- guild_sessions_connect:handle_session_down(Ref, State).+ case pending_session_by_ref(Ref, State) of+ {ok, SessionId, Session, Sessions} ->+ handle_pending_ref_down(SessionId, Session, Ref, Sessions, State);+ not_found ->+ guild_sessions_connect:handle_session_down(Ref, State)+ end.++-spec handle_pending_ref_down(+ session_id(), session_data(), reference(), sessions_map(), guild_state()+) -> {noreply, guild_state()} | {stop, normal, guild_state()}.+handle_pending_ref_down(SessionId, Session, Ref, Sessions, State) ->+ case pending_session_owns_connected_tracking(Session, Sessions, State) of+ true -> guild_sessions_connect:handle_session_down(Ref, State);+ false -> handle_pending_session_down(SessionId, Ref, Sessions, State)+ end.++-spec pending_session_by_ref(reference(), guild_state()) ->+ {ok, session_id(), session_data(), sessions_map()} | not_found.+pending_session_by_ref(Ref, State) ->+ Sessions = maps:get(sessions, State, #{}),+ case session_id_by_ref(Ref, Sessions, State) of+ SessionId when is_binary(SessionId) -> pending_session_entry(SessionId, Sessions);+ _ -> not_found+ end.++-spec pending_session_entry(session_id(), sessions_map()) ->+ {ok, session_id(), session_data(), sessions_map()} | not_found.+pending_session_entry(SessionId, Sessions) ->+ case maps:get(SessionId, Sessions, undefined) of+ #{pending_connect := true} = Session -> {ok, SessionId, Session, Sessions};+ _ -> not_found+ end.++-spec session_id_by_ref(reference(), sessions_map(), guild_state()) -> session_id() | undefined.+session_id_by_ref(Ref, Sessions, State) ->+ Refs = maps:get(guild_session_refs, State, #{}),+ case maps:get(Ref, Refs, undefined) of+ SessionId when is_binary(SessionId) -> SessionId;+ _ -> session_id_by_ref_scan(Ref, Sessions)+ end.++-spec session_id_by_ref_scan(reference(), sessions_map()) -> session_id() | undefined.+session_id_by_ref_scan(Ref, Sessions) ->+ maps:fold(+ fun(SessionId, Session, Found) ->+ match_session_ref(Ref, SessionId, Session, Found)+ end,+ undefined,+ Sessions+ ).++-spec match_session_ref(reference(), session_id(), session_data(), session_id() | undefined) ->+ session_id() | undefined.+match_session_ref(Ref, SessionId, #{mref := Ref}, _Found) -> SessionId;+match_session_ref(_Ref, _SessionId, _Session, Found) -> Found.++-spec pending_session_owns_connected_tracking(session_data(), sessions_map(), guild_state()) ->+ boolean().+pending_session_owns_connected_tracking(Session, Sessions, State) ->+ UserId = maps:get(user_id, Session, undefined),+ Counts = maps:get(user_session_counts, State, #{}),+ TrackedCount = non_negative_count(maps:get(UserId, Counts, 0)),+ TrackedCount > active_session_count(UserId, Sessions).++-spec non_negative_count(term()) -> non_neg_integer().+non_negative_count(Count) when is_integer(Count), Count >= 0 -> Count;+non_negative_count(_) -> 0.++-spec active_session_count(user_id() | undefined, sessions_map()) -> non_neg_integer().+active_session_count(UserId, Sessions) ->+ maps:fold(+ fun(_SessionId, Session, Count) ->+ count_active_session(UserId, Session, Count)+ end,+ 0,+ Sessions+ ).++-spec count_active_session(user_id() | undefined, session_data(), non_neg_integer()) ->+ non_neg_integer().+count_active_session(UserId, #{user_id := UserId} = Session, Count) ->+ case maps:get(pending_connect, Session, false) of+ true -> Count;+ false -> Count + 1+ end;+count_active_session(_UserId, _Session, Count) ->+ Count.++-spec handle_pending_session_down(session_id(), reference(), sessions_map(), guild_state()) ->+ {noreply, guild_state()}.+handle_pending_session_down(SessionId, Ref, Sessions, State) ->+ NewSessions = maps:remove(SessionId, Sessions),+ SessionRefs = maps:get(guild_session_refs, State, #{}),+ State1 = State#{+ sessions => NewSessions,+ guild_session_refs => maps:remove(Ref, SessionRefs)+ },+ State2 = guild_sessions_connect_cleanup:cleanup_connect_admission_for_session(+ SessionId, State1+ ),+ finish_pending_session_down(NewSessions, State2).++-spec finish_pending_session_down(sessions_map(), guild_state()) -> {noreply, guild_state()}.+finish_pending_session_down(NewSessions, State) ->+ case map_size(NewSessions) of+ 0 ->+ {noreply, guild_sessions_connect_cleanup:maybe_mark_auto_stop_pending(State)};+ _ ->+ {noreply, guild_sessions_connect_cleanup:clear_auto_stop_pending(State)}+ end.-spec remove_session(session_id(), guild_state()) -> guild_state().remove_session(SessionId, State) ->@@ -121,21 +234,68 @@ filter_sessions_for_channel(Sessions, ChannelId, SessionIdOpt, State) ->sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state()) -> [session_pair()].filter_sessions_for_message(Sessions, ChannelId, MessageId, SessionIdOpt, State) ->- filter_active_sessions(Sessions, SessionIdOpt, fun(S, _Sid) ->- session_can_view_channel(S, ChannelId, State) andalso- session_can_access_message(S, ChannelId, MessageId, State)- end).+ filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State).--spec session_can_access_message(map(), channel_id(), binary(), guild_state()) -> boolean().-session_can_access_message(SessionData, ChannelId, MessageId, State) ->- case maps:get(user_id, SessionData, undefined) of+-spec filter_message_memo(+ sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state()+) -> [session_pair()].+filter_message_memo(Sessions, ChannelId, MessageId, SessionIdOpt, State) ->+ Ctx = {ChannelId, MessageId, SessionIdOpt, State},+ {Acc, _Memo} = maps:fold(+ fun(Sid, S, In) -> collect_message_session(Sid, S, Ctx, In) end,+ {[], #{}},+ Sessions+ ),+ Acc.++-spec collect_message_session(+ session_id(), session_data(), message_ctx(), {[session_pair()], perm_memo()}+) -> {[session_pair()], perm_memo()}.+collect_message_session(Sid, S, Ctx, {Acc, Memo}) ->+ {ChannelId, _MessageId, SessionIdOpt, State} = Ctx,+ Visible =+ not is_pending_or_excluded(Sid, S, SessionIdOpt) andalso+ session_can_view_channel(S, ChannelId, State),+ case Visible of+ true -> memo_message_session(Sid, S, Ctx, Acc, Memo);+ false -> {Acc, Memo}+ end.++-spec memo_message_session(+ session_id(), session_data(), message_ctx(), [session_pair()], perm_memo()+) -> {[session_pair()], perm_memo()}.+memo_message_session(Sid, S, {ChannelId, MessageId, _SessionIdOpt, State}, Acc, Memo) ->+ case maps:get(user_id, S, undefined) ofUserId when is_integer(UserId) ->- Perms = guild_permissions:get_member_permissions(UserId, ChannelId, State),- guild_permissions:can_access_message_by_permissions(Perms, MessageId, State);+ {Perms, Memo1} = memo_member_permissions(UserId, ChannelId, State, Memo),+ Ok = guild_permissions:can_access_message_by_permissions(Perms, MessageId, State),+ {prepend_session(Ok, Sid, S, Acc), Memo1};_ ->- false+ {Acc, Memo}end.+-spec memo_member_permissions(user_id(), channel_id(), guild_state(), perm_memo()) ->+ {non_neg_integer(), perm_memo()}.+memo_member_permissions(UserId, ChannelId, State, Memo) ->+ case maps:get(UserId, Memo, undefined) of+ Perms when is_integer(Perms) ->+ {Perms, Memo};+ _ ->+ Computed = guild_permissions:get_member_permissions(UserId, ChannelId, State),+ {Computed, store_perm_memo(UserId, Computed, Memo)}+ end.++-spec store_perm_memo(user_id(), non_neg_integer(), perm_memo()) -> perm_memo().+store_perm_memo(UserId, Perms, Memo) when map_size(Memo) < ?MAX_PERM_MEMO_ENTRIES ->+ Memo#{UserId => Perms};+store_perm_memo(_UserId, _Perms, Memo) ->+ Memo.++-spec prepend_session(boolean(), session_id(), session_data(), [session_pair()]) ->+ [session_pair()].+prepend_session(true, Sid, S, Acc) -> [{Sid, S} | Acc];+prepend_session(false, _Sid, _S, Acc) -> Acc.+-spec filter_sessions_for_manage_channels(sessions_map(), channel_id(), session_id() | undefined, guild_state()) -> [session_pair()].@@ -341,4 +501,182 @@ filter_active_sessions_with_predicate_test() ->?assertEqual(1, length(Result)),[{<<"b">>, _}] = Result.+store_perm_memo_test() ->+ ?assertEqual(#{7 => 42}, store_perm_memo(7, 42, #{})),+ ?assertEqual(#{7 => 42}, store_perm_memo(7, 42, #{7 => 42})).++store_perm_memo_bound_test() ->+ Full = maps:from_list([{I, 0} || I <- lists:seq(1, ?MAX_PERM_MEMO_ENTRIES)]),+ ?assertEqual(Full, store_perm_memo(0, 1, Full)),+ ?assertEqual(?MAX_PERM_MEMO_ENTRIES, map_size(store_perm_memo(0, 1, Full))).++memo_member_permissions_hit_test() ->+ Memo = #{9 => 123},+ ?assertEqual({123, Memo}, memo_member_permissions(9, 5, #{}, Memo)).++memo_member_permissions_miss_test() ->+ ?assertEqual({0, #{9 => 0}}, memo_member_permissions(9, 5, #{}, #{})).++-spec reference_session_can_access_message(map(), channel_id(), binary(), guild_state()) ->+ boolean().+reference_session_can_access_message(SessionData, ChannelId, MessageId, State) ->+ case maps:get(user_id, SessionData, undefined) of+ UserId when is_integer(UserId) ->+ Perms = guild_permissions:get_member_permissions(UserId, ChannelId, State),+ guild_permissions:can_access_message_by_permissions(Perms, MessageId, State);+ _ ->+ false+ end.++-spec reference_filter_message_direct(+ sessions_map(), channel_id(), binary(), session_id() | undefined, guild_state()+) -> [session_pair()].+reference_filter_message_direct(Sessions, ChannelId, MessageId, SessionIdOpt, State) ->+ filter_active_sessions(Sessions, SessionIdOpt, fun(S, _Sid) ->+ session_can_view_channel(S, ChannelId, State) andalso+ reference_session_can_access_message(S, ChannelId, MessageId, State)+ end).++filter_message_memo_matches_direct_test() ->+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},+ Owner1 = #{user_id => 1, viewable_channels => #{5 => true}},+ Owner2 = #{user_id => 1, viewable_channels => #{5 => true}},+ Other = #{user_id => 2, viewable_channels => #{5 => true}},+ Sessions = #{<<"a">> => Owner1, <<"b">> => Owner2, <<"c">> => Other},+ Direct = reference_filter_message_direct(Sessions, 5, <<"1">>, undefined, State),+ Memoized = filter_message_memo(Sessions, 5, <<"1">>, undefined, State),+ ?assertEqual(Direct, Memoized),+ ?assertEqual([<<"a">>, <<"b">>], lists:sort([Sid || {Sid, _} <- Memoized])).++filter_message_memo_collects_visible_sessions_test() ->+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},+ Owner1 = #{user_id => 1, viewable_channels => #{5 => true}},+ Owner2 = #{user_id => 1, viewable_channels => #{5 => true}},+ Other = #{user_id => 2, viewable_channels => #{5 => true}},+ Sessions = #{<<"a">> => Owner1, <<"b">> => Owner2, <<"c">> => Other},+ Memoized = filter_message_memo(Sessions, 5, <<"1">>, undefined, State),+ ?assertEqual([<<"a">>, <<"b">>], lists:sort([Sid || {Sid, _} <- Memoized])).++filter_message_memo_skips_excluded_test() ->+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},+ S1 = #{user_id => 1, viewable_channels => #{5 => true}},+ S2 = #{user_id => 1, viewable_channels => #{5 => true}, pending_connect => true},+ Sessions = #{<<"a">> => S1, <<"b">> => S2},+ ?assertEqual([], filter_message_memo(Sessions, 5, <<"1">>, <<"a">>, State)).++filter_sessions_for_message_test() ->+ State = #{data => #{<<"guild">> => #{<<"owner_id">> => <<"1">>}}},+ Sessions = #{<<"a">> => #{user_id => 1, viewable_channels => #{5 => true}}},+ ?assertEqual(+ [{<<"a">>, maps:get(<<"a">>, Sessions)}],+ filter_sessions_for_message(Sessions, 5, <<"1">>, undefined, State)+ ).++non_negative_count_test() ->+ ?assertEqual(3, non_negative_count(3)),+ ?assertEqual(0, non_negative_count(0)),+ ?assertEqual(0, non_negative_count(-1)),+ ?assertEqual(0, non_negative_count(undefined)).++active_session_count_test() ->+ Sessions = #{+ <<"a">> => #{user_id => 1, pending_connect => true},+ <<"b">> => #{user_id => 1, pending_connect => false},+ <<"c">> => #{user_id => 1},+ <<"d">> => #{user_id => 2}+ },+ ?assertEqual(2, active_session_count(1, Sessions)),+ ?assertEqual(1, active_session_count(2, Sessions)),+ ?assertEqual(0, active_session_count(3, Sessions)).++session_id_by_ref_uses_index_test() ->+ Ref = make_ref(),+ Sessions = #{<<"a">> => #{mref => make_ref()}},+ State = #{guild_session_refs => #{Ref => <<"a">>}},+ ?assertEqual(<<"a">>, session_id_by_ref(Ref, Sessions, State)).++session_id_by_ref_scan_fallback_test() ->+ Ref = make_ref(),+ Sessions = #{<<"a">> => #{mref => make_ref()}, <<"b">> => #{mref => Ref}},+ ?assertEqual(<<"b">>, session_id_by_ref(Ref, Sessions, #{})),+ ?assertEqual(undefined, session_id_by_ref(make_ref(), Sessions, #{})).++pending_session_by_ref_found_test() ->+ Ref = make_ref(),+ Session = #{user_id => 1, mref => Ref, pending_connect => true},+ Sessions = #{<<"a">> => Session},+ State = #{sessions => Sessions, guild_session_refs => #{Ref => <<"a">>}},+ ?assertEqual({ok, <<"a">>, Session, Sessions}, pending_session_by_ref(Ref, State)),+ ?assertEqual(+ {ok, <<"a">>, Session, Sessions},+ pending_session_by_ref(Ref, #{sessions => Sessions})+ ).++pending_session_by_ref_not_pending_test() ->+ Ref = make_ref(),+ Session = #{user_id => 1, mref => Ref, pending_connect => false},+ State = #{sessions => #{<<"a">> => Session}, guild_session_refs => #{Ref => <<"a">>}},+ ?assertEqual(not_found, pending_session_by_ref(Ref, State)).++pending_session_by_ref_unknown_ref_test() ->+ Sessions = #{<<"a">> => #{user_id => 1, mref => make_ref(), pending_connect => true}},+ ?assertEqual(not_found, pending_session_by_ref(make_ref(), #{sessions => Sessions})).++pending_session_by_ref_stale_index_test() ->+ Ref = make_ref(),+ State = #{sessions => #{}, guild_session_refs => #{Ref => <<"gone">>}},+ ?assertEqual(not_found, pending_session_by_ref(Ref, State)).++pending_session_owns_connected_tracking_untracked_test() ->+ Session = #{user_id => 1, pending_connect => true},+ Sessions = #{<<"a">> => Session},+ ?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, #{})).++pending_session_owns_connected_tracking_other_session_owns_test() ->+ Session = #{user_id => 1, pending_connect => true},+ Active = #{user_id => 1, pending_connect => false},+ Sessions = #{<<"a">> => Session, <<"b">> => Active},+ State = #{user_session_counts => #{1 => 1}},+ ?assertEqual(false, pending_session_owns_connected_tracking(Session, Sessions, State)).++pending_session_owns_connected_tracking_true_test() ->+ Session = #{user_id => 1, pending_connect => true},+ Sessions = #{<<"a">> => Session},+ State = #{user_session_counts => #{1 => 1}},+ ?assertEqual(true, pending_session_owns_connected_tracking(Session, Sessions, State)).++handle_pending_session_down_keeps_tracking_test() ->+ Ref = make_ref(),+ Pending = #{session_id => <<"a">>, user_id => 1, mref => Ref, pending_connect => true},+ Active = #{session_id => <<"b">>, user_id => 1, mref => make_ref()},+ Sessions = #{<<"a">> => Pending, <<"b">> => Active},+ State = #{+ sessions => Sessions,+ guild_session_refs => #{Ref => <<"a">>},+ session_connect_pending => #{<<"a">> => #{}},+ user_session_counts => #{1 => 1},+ connected_user_ids => sets:from_list([1])+ },+ {noreply, NewState} = handle_pending_session_down(<<"a">>, Ref, Sessions, State),+ ?assertEqual(#{<<"b">> => Active}, maps:get(sessions, NewState)),+ ?assertEqual(#{}, maps:get(guild_session_refs, NewState)),+ ?assertEqual(#{}, maps:get(session_connect_pending, NewState)),+ ?assertEqual(#{1 => 1}, maps:get(user_session_counts, NewState)),+ ?assertEqual([1], sets:to_list(maps:get(connected_user_ids, NewState))),+ ?assertEqual(false, maps:is_key(auto_stop_pending, NewState)).++handle_pending_session_down_last_session_test() ->+ Ref = make_ref(),+ Pending = #{session_id => <<"a">>, user_id => 1, mref => Ref, pending_connect => true},+ Sessions = #{<<"a">> => Pending},+ State = #{+ sessions => Sessions,+ guild_session_refs => #{Ref => <<"a">>},+ disable_auto_stop_on_empty => true+ },+ {noreply, NewState} = handle_pending_session_down(<<"a">>, Ref, Sessions, State),+ ?assertEqual(#{}, maps:get(sessions, NewState)),+ ?assertEqual(#{}, maps:get(guild_session_refs, NewState)),+ ?assertEqual(false, maps:is_key(auto_stop_pending, NewState)).+-endif.diff --git a/fluxer_gateway/src/guild/guild_subscription_handler.erl b/fluxer_gateway/src/guild/guild_subscription_handler.erlindex 3e0274135..2062a719a 100644--- a/fluxer_gateway/src/guild/guild_subscription_handler.erl+++ b/fluxer_gateway/src/guild/guild_subscription_handler.erl@@ -22,6 +22,8 @@-define(LAZY_SUBSCRIBE_COALESCE_MS, 100).-define(MAX_BUFFERED_LAZY_SUBSCRIBE_RANGES, 10).+-define(ENGINES_KEY, channel_member_list_engines).+-define(DISPATCHED_KEY, lazy_subscribe_dispatched).-spec handle_call(term(), gen_server:from(), guild_state()) ->{reply, term(), guild_state()}.@@ -47,6 +49,73 @@ handle_info(flush_lazy_subscribe_buffer, State) ->-spec buffer_lazy_subscribe(map(), guild_state()) -> guild_state().buffer_lazy_subscribe(Request, State) ->+ case dispatch_immediately(Request, State) of+ true -> dispatch_lazy_subscribe_now(Request, State);+ false -> enqueue_lazy_subscribe(Request, State)+ end.++-spec dispatch_immediately(map(), guild_state()) -> boolean().+dispatch_immediately(Request, State) ->+ maps:get(lazy_subscribe_timer, State, undefined) =:= undefined andalso+ map_size(maps:get(lazy_subscribe_buffer, State, #{})) =:= 0 andalso+ engine_already_built(Request, State).++-spec engine_already_built(map(), guild_state()) -> boolean().+engine_already_built(#{channel_id := ChannelId}, State) ->+ case guild_member_list:calculate_list_id(ChannelId, State) of+ ListId when is_binary(ListId) ->+ maps:is_key(ListId, maps:get(?ENGINES_KEY, State, #{}));+ _ ->+ false+ end;+engine_already_built(_Request, _State) ->+ false.++%% A leading-edge dispatch still buffers its request, marked as dispatched, so a second+%% subscribe for the same key inside the window merges into it exactly as it did before+%% the leading edge existed. The mark only survives while nothing merges in, because every+%% merge_lazy_subscribe_request/2 clause returns a map derived from the incoming request.+-spec dispatch_lazy_subscribe_now(map(), guild_state()) -> guild_state().+dispatch_lazy_subscribe_now(Request, State) ->+ try process_lazy_subscribe(Request, State) of+ NewState -> enqueue_lazy_subscribe(mark_lazy_subscribe_dispatched(Request), NewState)+ catch+ Class:Reason:Stack ->+ log_lazy_subscribe_dispatch_error(Request, Class, Reason, Stack),+ State+ end.++-spec mark_lazy_subscribe_dispatched(map()) -> map().+mark_lazy_subscribe_dispatched(Request) ->+ Request#{?DISPATCHED_KEY => true}.++-spec log_lazy_subscribe_dispatch_error(map(), atom(), term(), list()) -> ok.+log_lazy_subscribe_dispatch_error(Request, Class, Reason, Stack) ->+ logger:warning(+ "guild_lazy_subscribe_dispatch_failed: session_id=~p channel_id=~p error=~p:~p ~p",+ [+ maps:get(session_id, Request, undefined),+ maps:get(channel_id, Request, undefined),+ Class,+ Reason,+ Stack+ ]+ ).++-spec arm_lazy_subscribe_timer(guild_state()) -> guild_state().+arm_lazy_subscribe_timer(State) ->+ case maps:get(lazy_subscribe_timer, State, undefined) of+ undefined ->+ Ref = erlang:send_after(+ ?LAZY_SUBSCRIBE_COALESCE_MS, self(), flush_lazy_subscribe_buffer+ ),+ State#{lazy_subscribe_timer => Ref};+ _ ->+ State+ end.++-spec enqueue_lazy_subscribe(map(), guild_state()) -> guild_state().+enqueue_lazy_subscribe(Request, State) ->#{session_id := SessionId, channel_id := ChannelId} = Request,BufferKey = {SessionId, ChannelId},Buffer = maps:get(lazy_subscribe_buffer, State, #{}),@@ -59,13 +128,8 @@ buffer_lazy_subscribe(Request, State) ->TimerRef = maps:get(lazy_subscribe_timer, State, undefined),NewState = State#{lazy_subscribe_buffer => NewBuffer, lazy_subscribe_order => NewOrder},case TimerRef of- undefined ->- Ref = erlang:send_after(- ?LAZY_SUBSCRIBE_COALESCE_MS, self(), flush_lazy_subscribe_buffer- ),- NewState#{lazy_subscribe_timer => Ref};- _ ->- NewState+ undefined -> arm_lazy_subscribe_timer(NewState);+ _ -> NewStateend.-spec merge_lazy_subscribe_request(map() | undefined, map()) -> map().@@ -139,6 +203,8 @@ ordered_lazy_subscribe_keys(State, Buffer) ->guild_state().process_buffered_lazy_subscribe(BufferKey, Buffer, State) ->case maps:find(BufferKey, Buffer) of+ {ok, #{?DISPATCHED_KEY := true}} ->+ State;{ok, Request} ->process_lazy_subscribe(Request, State);error ->@@ -294,13 +360,9 @@ handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) -FilteredMemberIds = filter_member_ids_for_subscription(GuildId, SessionUserId, MemberIds, State),- OldSubscriptions = guild_subscriptions:get_user_ids_for_session(SessionId, MemberSubs),- NewMemberSubs = guild_subscriptions:update_subscriptions(+ {NewMemberSubs, Added, Removed} = guild_subscriptions:update_subscriptions_with_delta(SessionId, FilteredMemberIds, MemberSubs),- NewSubscriptions = guild_subscriptions:get_user_ids_for_session(SessionId, NewMemberSubs),- Added = sets:to_list(sets:subtract(NewSubscriptions, OldSubscriptions)),- Removed = sets:to_list(sets:subtract(OldSubscriptions, NewSubscriptions)),State1 = State#{member_subscriptions => NewMemberSubs},State2 = handle_added_subscriptions(Added, SessionId, State1),handle_removed_subscriptions(Removed, State2).@@ -312,7 +374,7 @@ handle_update_member_subscriptions_local(GuildId, SessionId, MemberIds, State) -filter_member_ids_for_subscription(_GuildId, undefined, _MemberIds, _State) ->[];filter_member_ids_for_subscription(_GuildId, SessionUserId, MemberIds, State) ->- filter_member_ids_with_mutual_channels(SessionUserId, MemberIds, State).+ guild_subscription_mutual_channels:filter_member_ids(SessionUserId, MemberIds, State).-spec handle_added_subscriptions([user_id()], session_id(), guild_state()) -> guild_state().handle_added_subscriptions(Added, SessionId, State) ->@@ -333,58 +395,6 @@ handle_removed_subscriptions(Removed, State) ->Removed).--spec filter_member_ids_with_mutual_channels(- user_id(), [user_id()], guild_state()-) -> [user_id()].-filter_member_ids_with_mutual_channels(SessionUserId, MemberIds, State) ->- SessionMap = get_session_channel_map(SessionUserId, State),- lists:filtermap(- fun(MemberId) ->- has_mutual_channel(- MemberId, SessionUserId, SessionMap, State- )- end,- MemberIds- ).---spec get_session_channel_map(user_id(), guild_state()) -> map().-get_session_channel_map(SessionUserId, State) ->- case- guild_visibility_channels:get_cached_viewable_channel_map(- SessionUserId, State- )- of- undefined ->- guild_sessions:build_viewable_channel_map(- guild_visibility:get_user_viewable_channels(- SessionUserId, State- )- );- M ->- M- end.---spec has_mutual_channel(- term(), user_id(), map(), guild_state()-) -> {true, user_id()} | false.-has_mutual_channel(MemberId, SessionUserId, _SessionMap, _State) when- MemberId =:= SessionUserId; not is_integer(MemberId)-->- false;-has_mutual_channel(MemberId, _SessionUserId, SessionMap, State) ->- MemberChannels = guild_visibility:get_user_viewable_channels(- MemberId, State- ),- HasMutual = has_shared_channel(MemberChannels, SessionMap),- case HasMutual of- true -> {true, MemberId};- false -> false- end.---spec has_shared_channel([integer()], map()) -> boolean().-has_shared_channel(MemberChannels, SessionMap) ->- lists:any(fun(Ch) -> maps:is_key(Ch, SessionMap) end, MemberChannels).--ifdef(TEST).-spec disabled_operations_state(integer() | binary()) -> guild_state().@@ -497,4 +507,138 @@ flush_lazy_subscribe_buffer_clears_state_test() ->?assertEqual(error, maps:find(lazy_subscribe_order, NewState)),?assertEqual(error, maps:find(lazy_subscribe_timer, NewState)).+warm_list_state(ChannelId) ->+ ListId = integer_to_binary(ChannelId),+ #{+ data => #{+ <<"channel_index">> => #{+ ChannelId => #{<<"id">> => ListId}+ }+ },+ ?ENGINES_KEY => #{ListId => warm_list_state_engine_ref}+ }.++engine_already_built_detects_built_engine_test() ->+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ ?assertEqual(true, engine_already_built(Request, warm_list_state(500))).++engine_already_built_false_without_engine_test() ->+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ State = maps:put(?ENGINES_KEY, #{}, warm_list_state(500)),+ ?assertEqual(false, engine_already_built(Request, State)).++engine_already_built_false_for_unknown_channel_test() ->+ Request = #{session_id => <<"s1">>, channel_id => 999, ranges => [{0, 99}]},+ ?assertEqual(false, engine_already_built(Request, warm_list_state(500))).++engine_already_built_false_on_bare_state_test() ->+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ ?assertEqual(false, engine_already_built(Request, #{})).++dispatch_immediately_requires_idle_window_test() ->+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ Warm = warm_list_state(500),+ ?assertEqual(true, dispatch_immediately(Request, Warm)),+ ?assertEqual(+ false, dispatch_immediately(Request, Warm#{lazy_subscribe_timer => make_ref()})+ ),+ ?assertEqual(+ false,+ dispatch_immediately(Request, Warm#{+ lazy_subscribe_buffer => #{{<<"s2">>, 501} => Request}+ })+ ).++warm_subscribe_records_the_dispatched_request_test() ->+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ NewState = buffer_lazy_subscribe(Request, warm_list_state(500)),+ Buffer = maps:get(lazy_subscribe_buffer, NewState),+ ?assertEqual(Request#{?DISPATCHED_KEY => true}, maps:get({<<"s1">>, 500}, Buffer)),+ ?assertEqual([{<<"s1">>, 500}], maps:get(lazy_subscribe_order, NewState)),+ ?assertNotEqual(undefined, maps:get(lazy_subscribe_timer, NewState, undefined)).++subscribe_behind_a_warm_dispatch_is_coalesced_test() ->+ Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)),+ Request2 = #{session_id => <<"s2">>, channel_id => 500, ranges => [{0, 99}]},+ State2 = buffer_lazy_subscribe(Request2, State1),+ Buffer = maps:get(lazy_subscribe_buffer, State2),+ ?assertEqual(Request2, maps:get({<<"s2">>, 500}, Buffer)),+ ?assertEqual(Request1#{?DISPATCHED_KEY => true}, maps:get({<<"s1">>, 500}, Buffer)),+ ?assertEqual(2, map_size(Buffer)).++%% Reference oracle: what enqueue_lazy_subscribe/2 buffered for a repeated+%% {session, channel} before the leading-edge dispatch existed.+reference_coalesced_request(Request1, Request2) ->+ merge_lazy_subscribe_request(Request1, Request2).++repeated_warm_subscribe_merges_like_the_pre_leading_edge_path_test() ->+ Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ Request2 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{100, 199}]},+ State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)),+ State2 = buffer_lazy_subscribe(Request2, State1),+ Buffer = maps:get(lazy_subscribe_buffer, State2),+ Merged = maps:get({<<"s1">>, 500}, Buffer),+ ?assertEqual(1, map_size(Buffer)),+ ?assertEqual(reference_coalesced_request(Request1, Request2), Merged),+ ?assertNot(maps:is_key(?DISPATCHED_KEY, Merged)),+ ?assertEqual([{<<"s1">>, 500}], maps:get(lazy_subscribe_order, State2)).++repeated_warm_subscribe_with_empty_ranges_replaces_test() ->+ Request1 = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ Request2 = #{session_id => <<"s1">>, channel_id => 500, ranges => []},+ State1 = buffer_lazy_subscribe(Request1, warm_list_state(500)),+ State2 = buffer_lazy_subscribe(Request2, State1),+ Merged = maps:get({<<"s1">>, 500}, maps:get(lazy_subscribe_buffer, State2)),+ ?assertEqual(reference_coalesced_request(Request1, Request2), Merged),+ ?assertNot(maps:is_key(?DISPATCHED_KEY, Merged)).++dispatched_entry_state() ->+ Key = {<<"s1">>, 500},+ Dispatched = #{session_id => <<"s1">>, channel_id => 500, ?DISPATCHED_KEY => true},+ State = #{+ lazy_subscribe_buffer => #{Key => Dispatched},+ lazy_subscribe_order => [Key],+ lazy_subscribe_timer => make_ref()+ },+ {Key, Dispatched, State}.++flush_skips_an_already_dispatched_entry_test() ->+ {_Key, _Dispatched, State} = dispatched_entry_state(),+ NewState = flush_lazy_subscribe_buffer(State),+ ?assertEqual(error, maps:find(lazy_subscribe_buffer, NewState)),+ ?assertEqual(error, maps:find(lazy_subscribe_order, NewState)),+ ?assertEqual(error, maps:find(lazy_subscribe_timer, NewState)).++flush_processes_the_same_entry_once_unmarked_test() ->+ {Key, Dispatched, State} = dispatched_entry_state(),+ Unmarked = maps:remove(?DISPATCHED_KEY, Dispatched),+ ?assertError(+ {badmatch, _},+ flush_lazy_subscribe_buffer(State#{lazy_subscribe_buffer => #{Key => Unmarked}})+ ).++failed_warm_dispatch_is_not_re_enqueued_test() ->+ Warm = warm_list_state(500),+ Broken = #{session_id => <<"s1">>, channel_id => 500},+ ?assertEqual(Warm, buffer_lazy_subscribe(Broken, Warm)).++failed_warm_dispatch_does_not_escape_handle_call_test() ->+ Warm = warm_list_state(500),+ Broken = #{session_id => <<"s1">>, channel_id => 500},+ From = {self(), make_ref()},+ ?assertEqual({reply, ok, Warm}, handle_call({lazy_subscribe, Broken}, From, Warm)).++cold_list_still_uses_the_coalesce_buffer_test() ->+ Request = #{session_id => <<"s1">>, channel_id => 500, ranges => [{0, 99}]},+ State = maps:put(?ENGINES_KEY, #{}, warm_list_state(500)),+ NewState = buffer_lazy_subscribe(Request, State),+ Buffer = maps:get(lazy_subscribe_buffer, NewState),+ ?assertEqual(Request, maps:get({<<"s1">>, 500}, Buffer)).++arm_lazy_subscribe_timer_is_idempotent_test() ->+ Ref = make_ref(),+ State = #{lazy_subscribe_timer => Ref},+ ?assertEqual(Ref, maps:get(lazy_subscribe_timer, arm_lazy_subscribe_timer(State))).+-endif.diff --git a/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erlnew file mode 100644index 000000000..404bdf93f--- /dev/null+++ b/fluxer_gateway/src/guild/guild_subscription_mutual_channels.erl@@ -0,0 +1,287 @@+%% SPDX-License-Identifier: AGPL-3.0-or-later++-module(guild_subscription_mutual_channels).+-typing([eqwalizer]).++-export([filter_member_ids/3]).++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").+-endif.++-type guild_state() :: map().+-type user_id() :: integer().+-type memo() :: #{exceptions := sets:set(user_id()), cache := #{term() => boolean()}}.+-export_type([guild_state/0, user_id/0]).++-spec filter_member_ids(user_id(), [user_id()], guild_state()) -> [user_id()].+filter_member_ids(_SessionUserId, [], _State) ->+ [];+filter_member_ids(SessionUserId, MemberIds, State) ->+ SessionMap = session_channel_map(SessionUserId, State),+ {Kept, _Memo} = lists:foldl(+ fun(MemberId, Acc) ->+ keep_member_with_mutual_channel(MemberId, SessionUserId, SessionMap, State, Acc)+ end,+ {[], new_memo(State)},+ MemberIds+ ),+ lists:reverse(Kept).++-spec session_channel_map(user_id(), guild_state()) -> map().+session_channel_map(SessionUserId, State) ->+ case guild_visibility_channels:get_cached_viewable_channel_map(SessionUserId, State) of+ undefined ->+ guild_sessions:build_viewable_channel_map(+ guild_visibility:get_user_viewable_channels(SessionUserId, State)+ );+ Map ->+ Map+ end.++-spec keep_member_with_mutual_channel(+ term(), user_id(), map(), guild_state(), {[user_id()], memo()}+) -> {[user_id()], memo()}.+keep_member_with_mutual_channel(MemberId, SessionUserId, _SessionMap, _State, Acc) when+ MemberId =:= SessionUserId; not is_integer(MemberId)+->+ Acc;+keep_member_with_mutual_channel(MemberId, _SessionUserId, SessionMap, State, {Kept, Memo}) ->+ case memoised_has_mutual_channel(MemberId, SessionMap, State, Memo) of+ {true, Memo1} -> {[MemberId | Kept], Memo1};+ {false, Memo1} -> {Kept, Memo1}+ end.++%% get_user_viewable_channels/2 reads nothing from a member except its raw roles+%% term, so two members holding the same term see the same channels and give the same+%% answer against a fixed session map. The user-specific inputs are the guild owner,+%% virtual channel access and user-type permission overwrites, and every user named+%% by one of those bypasses the memo.+-spec new_memo(guild_state()) -> memo().+new_memo(State) ->+ #{exceptions => exceptions(State), cache => #{}}.++-spec memoised_has_mutual_channel(user_id(), map(), guild_state(), memo()) ->+ {boolean(), memo()}.+memoised_has_mutual_channel(MemberId, SessionMap, State, Memo) ->+ #{exceptions := Exceptions, cache := Cache} = Memo,+ case memo_key(MemberId, Exceptions, State) of+ bypass ->+ {has_mutual_channel(MemberId, SessionMap, State), Memo};+ {ok, RawRoles} ->+ cached_has_mutual_channel(MemberId, RawRoles, SessionMap, State, Cache, Memo)+ end.++-spec cached_has_mutual_channel(+ user_id(), term(), map(), guild_state(), #{term() => boolean()}, memo()+) -> {boolean(), memo()}.+cached_has_mutual_channel(MemberId, RawRoles, SessionMap, State, Cache, Memo) ->+ case maps:find(RawRoles, Cache) of+ {ok, HasMutual} ->+ {HasMutual, Memo};+ error ->+ HasMutual = has_mutual_channel(MemberId, SessionMap, State),+ {HasMutual, Memo#{cache := Cache#{RawRoles => HasMutual}}}+ end.++-spec memo_key(user_id(), sets:set(user_id()), guild_state()) -> {ok, term()} | bypass.+memo_key(MemberId, Exceptions, State) ->+ case sets:is_element(MemberId, Exceptions) of+ true -> bypass;+ false -> member_roles_key(guild_permissions:find_member_by_user_id(MemberId, State))+ end.++-spec member_roles_key(map() | undefined) -> {ok, term()} | bypass.+member_roles_key(Member) when is_map(Member) ->+ {ok, maps:get(<<"roles">>, Member, [])};+member_roles_key(_Member) ->+ bypass.++-spec exceptions(guild_state()) -> sets:set(user_id()).+exceptions(State) ->+ Data = map_utils:ensure_map(map_utils:get_safe(State, data, #{})),+ Guild = map_utils:ensure_map(maps:get(<<"guild">>, Data, #{})),+ Owner = owner_ids(maps:get(<<"owner_id">>, Guild, undefined)),+ Virtual = maps:keys(map_utils:ensure_map(maps:get(virtual_channel_access, State, #{}))),+ sets:from_list(Owner ++ Virtual ++ overwrite_target_ids(Data)).++%% parse_maybe/1, not parse_optional/1: an id this rejects can never match a user in+%% a permission check either, so leaving it out of the exception set keeps the memo+%% exact while keeping the set itself total.+-spec owner_ids(term()) -> [user_id()].+owner_ids(OwnerIdRaw) ->+ case snowflake_id:parse_maybe(OwnerIdRaw) of+ OwnerId when is_integer(OwnerId) -> [OwnerId];+ _ -> []+ end.++%% Every overwrite target on every channel, read from both places a permission check+%% can take them from, and role targets alongside user targets. Role ids can never+%% collide with user ids, so the superset only costs a few members their memo entry.+-spec overwrite_target_ids(map()) -> [user_id()].+overwrite_target_ids(Data) ->+ Cached = maps:fold(+ fun(_ChannelId, Entries, Acc) -> cached_target_ids(Entries, Acc) end,+ [],+ map_utils:ensure_map(maps:get(overwrite_perms_cache, Data, #{}))+ ),+ maps:fold(+ fun(_ChannelId, Channel, Acc) -> channel_target_ids(Channel, Acc) end,+ Cached,+ guild_data_index:channel_index(Data)+ ).++-spec cached_target_ids(term(), [user_id()]) -> [user_id()].+cached_target_ids(Entries, Acc) when is_list(Entries) ->+ [Id || {Id, _Type, _Allow, _Deny} <- Entries, is_integer(Id)] ++ Acc;+cached_target_ids(_Entries, Acc) ->+ Acc.++-spec channel_target_ids(term(), [user_id()]) -> [user_id()].+channel_target_ids(Channel, Acc) when is_map(Channel) ->+ lists:foldl(+ fun overwrite_target_id/2,+ Acc,+ map_utils:ensure_list(maps:get(<<"permission_overwrites">>, Channel, []))+ );+channel_target_ids(_Channel, Acc) ->+ Acc.++-spec overwrite_target_id(term(), [user_id()]) -> [user_id()].+overwrite_target_id(Overwrite, Acc) when is_map(Overwrite) ->+ case snowflake_id:parse_maybe(maps:get(<<"id">>, Overwrite, undefined)) of+ Id when is_integer(Id) -> [Id | Acc];+ _ -> Acc+ end;+overwrite_target_id(_Overwrite, Acc) ->+ Acc.++-spec has_mutual_channel(user_id(), map(), guild_state()) -> boolean().+has_mutual_channel(MemberId, SessionMap, State) ->+ MemberChannels = guild_visibility:get_user_viewable_channels(MemberId, State),+ has_shared_channel(MemberChannels, SessionMap).++-spec has_shared_channel([integer()], map()) -> boolean().+has_shared_channel(MemberChannels, SessionMap) ->+ lists:any(fun(Ch) -> maps:is_key(Ch, SessionMap) end, MemberChannels).++-ifdef(TEST).++-define(GUILD_ID, 42).+-define(VIEWER_ROLE, 1000).+-define(OTHER_ROLE, 2000).++test_role(RoleId) ->+ #{<<"id">> => integer_to_binary(RoleId), <<"permissions">> => <<"0">>}.++test_member(UserId, RoleIds) ->+ #{+ <<"user">> => #{<<"id">> => integer_to_binary(UserId)},+ <<"roles">> => [integer_to_binary(RoleId) || RoleId <- RoleIds]+ }.++test_overwrite(TargetId, Type) ->+ #{+ <<"id">> => integer_to_binary(TargetId),+ <<"type">> => Type,+ <<"allow">> => integer_to_binary(constants:view_channel_permission()),+ <<"deny">> => <<"0">>+ }.++test_channel(ChannelId, Overwrites) ->+ #{+ <<"id">> => integer_to_binary(ChannelId),+ <<"type">> => 0,+ <<"permission_overwrites">> => Overwrites+ }.++%% Session user 10 and members 20 and 21 reach channel 500 through the viewer role.+%% Members 30, 31, 99 and 4242 all carry the same roles term and reach only channel+%% 600 by role, but 99 holds virtual access to 500 and 4242 is a user-overwrite+%% target on 500, so both must still come out true. Member 7 owns the guild.+test_state() ->+ Channels = [+ test_channel(500, [test_overwrite(?VIEWER_ROLE, 0), test_overwrite(4242, 1)]),+ test_channel(600, [test_overwrite(?OTHER_ROLE, 0)])+ ],+ Roles = [test_role(?GUILD_ID), test_role(?VIEWER_ROLE), test_role(?OTHER_ROLE)],+ Members = maps:from_list(+ [{7, test_member(7, [])}] +++ [{Id, test_member(Id, [?VIEWER_ROLE])} || Id <- [10, 20, 21]] +++ [{Id, test_member(Id, [?OTHER_ROLE])} || Id <- [30, 31, 99, 4242]]+ ),+ #{+ id => ?GUILD_ID,+ virtual_channel_access => #{99 => sets:from_list([500])},+ data => #{+ <<"guild">> => #{<<"owner_id">> => <<"7">>},+ <<"roles">> => Roles,+ <<"members">> => Members,+ <<"channels">> => Channels,+ <<"channel_index">> => guild_data_index:build_id_index(Channels)+ }+ }.++%% filter_member_ids/3 as it read before the memo: every candidate materialises its+%% own complete viewable channel list.+reference_filter_member_ids(SessionUserId, MemberIds, State) ->+ SessionMap = session_channel_map(SessionUserId, State),+ lists:filtermap(+ fun(MemberId) -> reference_keep(MemberId, SessionUserId, SessionMap, State) end,+ MemberIds+ ).++reference_keep(MemberId, SessionUserId, _SessionMap, _State) when+ MemberId =:= SessionUserId; not is_integer(MemberId)+->+ false;+reference_keep(MemberId, _SessionUserId, SessionMap, State) ->+ case has_mutual_channel(MemberId, SessionMap, State) of+ true -> {true, MemberId};+ false -> false+ end.++candidate_ids() ->+ [20, 30, 21, 31, 10, 7, 99, 4242, 777, 20].++exceptions_cover_owner_virtual_and_overwrite_targets_test() ->+ Exceptions = exceptions(test_state()),+ ?assertEqual([true, true, true], [+ sets:is_element(Id, Exceptions)+ || Id <- [7, 99, 4242]+ ]),+ ?assertEqual([false, false], [sets:is_element(Id, Exceptions) || Id <- [20, 30]]).++filter_member_ids_matches_unmemoised_test() ->+ State = test_state(),+ Ids = candidate_ids(),+ ?assertEqual([20, 21, 7, 99, 4242, 20], filter_member_ids(10, Ids, State)),+ ?assertEqual(+ [reference_filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]],+ [filter_member_ids(Viewer, Ids, State) || Viewer <- [10, 30, 7, 99]]+ ).++memo_holds_one_entry_per_roles_term_test() ->+ State = test_state(),+ SessionMap = session_channel_map(10, State),+ {First, Memo1} = memoised_has_mutual_channel(20, SessionMap, State, new_memo(State)),+ {Second, Memo2} = memoised_has_mutual_channel(21, SessionMap, State, Memo1),+ ?assert(First),+ ?assertEqual(First, Second),+ ?assertEqual([1, 1], [map_size(maps:get(cache, M)) || M <- [Memo1, Memo2]]).++exception_user_is_not_answered_from_its_role_set_test() ->+ State = test_state(),+ SessionMap = session_channel_map(10, State),+ {false, Memo} = memoised_has_mutual_channel(30, SessionMap, State, new_memo(State)),+ ?assertEqual(+ [{true, Memo}, {true, Memo}, {true, Memo}],+ [memoised_has_mutual_channel(Id, SessionMap, State, Memo) || Id <- [7, 99, 4242]]+ ).++non_member_candidate_is_dropped_test() ->+ State = test_state(),+ ?assertEqual([], filter_member_ids(10, [777], State)),+ ?assertEqual([], reference_filter_member_ids(10, [777], State)).++-endif.diff --git a/fluxer_gateway/src/guild/guild_subscriptions.erl b/fluxer_gateway/src/guild/guild_subscriptions.erlindex 50c5b8850..2ca02b260 100644--- a/fluxer_gateway/src/guild/guild_subscriptions.erl+++ b/fluxer_gateway/src/guild/guild_subscriptions.erl@@ -9,6 +9,7 @@unsubscribe/3,unsubscribe_session/2,update_subscriptions/3,+ update_subscriptions_with_delta/3,get_subscribed_sessions/2,is_subscribed/3,get_user_ids_for_session/2@@ -52,30 +53,44 @@ remove_session_subscriber(UserId, SessionId, Subscribers, State) ->_ -> State#{UserId => NewSubscribers}end.+%% Every subscriber set in the map is non-empty: this module, guild_maintenance and+%% guild_presence_sync all drop a user key once its last subscriber goes. Entries the+%% session is absent from are therefore already correct and are left as they are+%% instead of being rebuilt into a fresh map.-spec unsubscribe_session(session_id(), subscription_state()) -> subscription_state().unsubscribe_session(SessionId, State) ->maps:fold(fun(UserId, Subscribers, Acc) ->- remove_session_from_subs(UserId, SessionId, Subscribers, Acc)+ drop_session_subscriber(UserId, SessionId, Subscribers, Acc)end,- #{},+ State,State).--spec remove_session_from_subs(+-spec drop_session_subscriber(user_id(), session_id(), sets:set(session_id()), subscription_state()) ->subscription_state().-remove_session_from_subs(UserId, SessionId, Subscribers, Acc) ->- NewSubscribers = sets:del_element(SessionId, Subscribers),- case sets:size(NewSubscribers) of- 0 -> Acc;- _ -> Acc#{UserId => NewSubscribers}+drop_session_subscriber(UserId, SessionId, Subscribers, Acc) ->+ case sets:is_element(SessionId, Subscribers) of+ true -> remove_session_subscriber(UserId, SessionId, Subscribers, Acc);+ false -> Accend.-spec update_subscriptions(session_id(), [user_id()], subscription_state()) ->subscription_state().update_subscriptions(SessionId, NewMemberIds, State) ->+ {NewState, _Added, _Removed} = update_subscriptions_with_delta(+ SessionId, NewMemberIds, State+ ),+ NewState.++%% After both folds the session subscribes to exactly NewMemberIds, so the added and+%% removed sets are the ToAdd and ToRemove already computed here. Deriving them from+%% the resulting map instead would cost two more full folds of the subscription map.+-spec update_subscriptions_with_delta(session_id(), [user_id()], subscription_state()) ->+ {subscription_state(), [user_id()], [user_id()]}.+update_subscriptions_with_delta(SessionId, NewMemberIds, State) ->CurrentlySubscribed = get_user_ids_for_session(SessionId, State),NewMemberIdSet = sets:from_list(NewMemberIds),ToRemove = sets:subtract(CurrentlySubscribed, NewMemberIdSet),@@ -87,13 +102,14 @@ update_subscriptions(SessionId, NewMemberIds, State) ->State,ToRemove),- sets:fold(+ State2 = sets:fold(fun(UserId, AccState) ->subscribe(SessionId, UserId, AccState)end,State1,ToAdd- ).+ ),+ {State2, sets:to_list(ToAdd), sets:to_list(ToRemove)}.-spec get_subscribed_sessions(user_id(), subscription_state()) -> [session_id()].get_subscribed_sessions(UserId, State) ->@@ -193,4 +209,83 @@ get_user_ids_for_session_test() ->UserIds = get_user_ids_for_session(<<"session1">>, State2),?assertEqual([100, 200], lists:sort(sets:to_list(UserIds))).+%% The subscription map as unsubscribe_session/2 built it before it stopped+%% rebuilding untouched entries.+reference_unsubscribe_session(SessionId, State) ->+ maps:fold(+ fun(UserId, Subscribers, Acc) ->+ reference_drop_subscriber(UserId, sets:del_element(SessionId, Subscribers), Acc)+ end,+ #{},+ State+ ).++reference_drop_subscriber(UserId, NewSubscribers, Acc) ->+ case sets:size(NewSubscribers) of+ 0 -> Acc;+ _ -> Acc#{UserId => NewSubscribers}+ end.++%% The state and delta handle_update_member_subscriptions_local/4 derived before it+%% stopped re-folding the map to rediscover what changed.+reference_update_with_delta(SessionId, NewMemberIds, State) ->+ Old = get_user_ids_for_session(SessionId, State),+ NewState = update_subscriptions(SessionId, NewMemberIds, State),+ New = get_user_ids_for_session(SessionId, NewState),+ {+ NewState,+ sets:to_list(sets:subtract(New, Old)),+ sets:to_list(sets:subtract(Old, New))+ }.++populate(Pairs) ->+ lists:foldl(+ fun({SessionId, UserId}, Acc) -> subscribe(SessionId, UserId, Acc) end,+ init_state(),+ Pairs+ ).++delta_fixture_states() ->+ [+ init_state(),+ populate([{<<"s1">>, 100}]),+ populate([{<<"s1">>, 100}, {<<"s1">>, 200}, {<<"s2">>, 200}, {<<"s2">>, 300}]),+ populate([{<<"s2">>, N} || N <- lists:seq(1, 64)]),+ populate(+ [{<<"s1">>, N} || N <- lists:seq(1, 200)] +++ [{<<"s2">>, N} || N <- lists:seq(150, 400)]+ )+ ].++delta_fixture_cases() ->+ Inputs = [[], [100], [200, 300], [300, 200], [100, 100, 200], lists:seq(100, 500)],+ [{State, Ids} || State <- delta_fixture_states(), Ids <- Inputs].++unsubscribe_session_matches_rebuilt_map_test() ->+ Cases = [+ {State, SessionId}+ || State <- delta_fixture_states(),+ SessionId <- [<<"s1">>, <<"s2">>, <<"never_subscribed">>]+ ],+ ?assertEqual(+ [reference_unsubscribe_session(S, M) || {M, S} <- Cases],+ [unsubscribe_session(S, M) || {M, S} <- Cases]+ ).++update_subscriptions_with_delta_matches_refold_test() ->+ Cases = delta_fixture_cases(),+ ?assertEqual(+ [reference_update_with_delta(<<"s1">>, Ids, M) || {M, Ids} <- Cases],+ [update_subscriptions_with_delta(<<"s1">>, Ids, M) || {M, Ids} <- Cases]+ ).++update_subscriptions_with_delta_leaves_other_sessions_alone_test() ->+ State0 = subscribe(<<"s2">>, 100, subscribe(<<"s1">>, 100, init_state())),+ {NewState, Added, Removed} = update_subscriptions_with_delta(<<"s1">>, [200], State0),+ ?assertEqual([200], Added),+ ?assertEqual([100], Removed),+ ?assert(is_subscribed(<<"s2">>, 100, NewState)),+ ?assertNot(is_subscribed(<<"s1">>, 100, NewState)),+ ?assert(is_subscribed(<<"s1">>, 200, NewState)).+-endif.diff --git a/fluxer_gateway/src/guild/voice/guild_voice.erl b/fluxer_gateway/src/guild/voice/guild_voice.erlindex 75992f6c5..e080649a8 100644--- a/fluxer_gateway/src/guild/voice/guild_voice.erl+++ b/fluxer_gateway/src/guild/voice/guild_voice.erl@@ -112,7 +112,7 @@ switch_voice_region(GuildId, ChannelId, GuildPid) ->-spec handle_virtual_channel_access_for_move(integer(), integer(), map(), pid()) -> ok.handle_virtual_channel_access_for_move(UserId, ChannelId, _ConnectionsToMove, GuildPid) ->- case gen_server:call(GuildPid, {get_sessions}, 10000) of+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) ofState when is_map(State) ->maybe_grant_move_access(UserId, ChannelId, State, GuildPid);_ ->@@ -159,9 +159,9 @@ cleanup_virtual_access_on_disconnect(UserId, GuildPid) ->-spec resolve_guild_id_from_pid(pid()) -> integer() | undefined.resolve_guild_id_from_pid(GuildPid) ->- try gen_server:call(GuildPid, {get_sessions}, 5000) of- State when is_map(State) ->- maps:get(id, State, undefined);+ try guild_voice_server_state:guild_id_call(GuildPid, 5000) of+ GuildId when is_integer(GuildId) ->+ GuildId;_ ->undefinedcatchdiff --git a/fluxer_gateway/src/guild/voice/guild_voice_move.erl b/fluxer_gateway/src/guild/voice/guild_voice_move.erlindex a4d620cbf..79b602e46 100644--- a/fluxer_gateway/src/guild/voice/guild_voice_move.erl+++ b/fluxer_gateway/src/guild/voice/guild_voice_move.erl@@ -176,7 +176,7 @@ send_voice_server_update_for_move(integer(), integer(), integer(), binary(), binary() | null, pid()) -> ok.do_send_voice_server_update(GuildId, ChannelId, UserId, SessionId, OldConnectionId, GuildPid) ->- try gen_server:call(GuildPid, {get_sessions}, 10000) of+ try guild_voice_server_state:guild_state_call(GuildPid, 10000) ofState when is_map(State) ->request_and_broadcast_token(GuildId, ChannelId, UserId, SessionId, OldConnectionId, Statediff --git a/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl b/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erlindex 8789c59d4..a64145252 100644--- a/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl+++ b/fluxer_gateway/src/guild/voice/guild_voice_move_execute.erl@@ -256,7 +256,7 @@ member_user_id(Member) ->-spec fetch_state_and_send(integer(), integer(), map(), pid(), integer(), map()) -> ok.fetch_state_and_send(GuildId, ChannelId, SessionInfo, GuildPid, UserId, Member) ->- try gen_server:call(GuildPid, {get_sessions}, 10000) of+ try guild_voice_server_state:guild_state_call(GuildPid, 10000) ofStateData when is_map(StateData) ->request_and_broadcast(GuildId, ChannelId, SessionInfo, GuildPid, UserId, Member, StateDatadiff --git a/fluxer_gateway/src/guild/voice/guild_voice_region.erl b/fluxer_gateway/src/guild/voice/guild_voice_region.erlindex fc9b2ef9d..5f6deb9ad 100644--- a/fluxer_gateway/src/guild/voice/guild_voice_region.erl+++ b/fluxer_gateway/src/guild/voice/guild_voice_region.erl@@ -40,7 +40,7 @@ switch_voice_region_channel_reply(Channel, State) ->-spec switch_voice_region(integer(), integer(), pid()) -> ok.switch_voice_region(GuildId, ChannelId, GuildPid) ->- case gen_server:call(GuildPid, {get_sessions}, 10000) of+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) ofState when is_map(State) ->do_switch_voice_region(GuildId, ChannelId, GuildPid, State);_ ->@@ -53,7 +53,7 @@ do_switch_voice_region(GuildId, ChannelId, GuildPid, State) ->UsersInChannel = collect_users_in_channel(VoiceStates, ChannelId),lists:foreach(fun(UserInfo) ->- maybe_send_region_switch_update(GuildId, ChannelId, GuildPid, UserInfo)+ maybe_send_region_switch_update(GuildId, ChannelId, GuildPid, State, UserInfo)end,UsersInChannel).@@ -70,17 +70,28 @@ collect_users_in_channel(VoiceStates, ChannelId) ->).-spec maybe_send_region_switch_update(- integer(), integer(), pid(), {integer(), binary() | undefined, binary(), voice_state()}+ integer(),+ integer(),+ pid(),+ map(),+ {integer(), binary() | undefined, binary(), voice_state()}) -> ok.maybe_send_region_switch_update(- _GuildId, _ChannelId, _GuildPid, {_UserId, undefined, _ConnId, _VS}+ _GuildId, _ChannelId, _GuildPid, _State, {_UserId, undefined, _ConnId, _VS}) ->ok;maybe_send_region_switch_update(- GuildId, ChannelId, GuildPid, {UserId, SessionId, ExistingConnectionId, VoiceState}+ GuildId, ChannelId, GuildPid, State, {UserId, SessionId, ExistingConnectionId, VoiceState}) ->- send_voice_server_update_for_region_switch(- GuildId, ChannelId, UserId, SessionId, ExistingConnectionId, VoiceState, GuildPid+ request_and_broadcast_region_switch(+ GuildId,+ ChannelId,+ UserId,+ SessionId,+ ExistingConnectionId,+ VoiceState,+ GuildPid,+ State).-spec collect_user_in_channel(binary(), voice_state(), integer(), list()) -> list().@@ -100,28 +111,6 @@ collect_user_voice_state(ConnectionId, VoiceState, Acc) ->[{UserId, SessionId, ConnectionId, VoiceState} | Acc]end.--spec send_voice_server_update_for_region_switch(- integer(), integer(), integer(), binary(), binary(), voice_state(), pid()-) -> ok.-send_voice_server_update_for_region_switch(- GuildId, ChannelId, UserId, SessionId, ExistingConnectionId, ExistingVoiceState, GuildPid-) ->- case gen_server:call(GuildPid, {get_sessions}, 10000) of- State when is_map(State) ->- request_and_broadcast_region_switch(- GuildId,- ChannelId,- UserId,- SessionId,- ExistingConnectionId,- ExistingVoiceState,- GuildPid,- State- );- _ ->- ok- end.--spec request_and_broadcast_region_switch(integer(),integer(),@@ -297,6 +286,201 @@ collect_users_in_channel_test() ->?assertEqual(<<"sess1">>, SessionId),?assertEqual(<<"conn1">>, ConnectionId).+switch_voice_region_reads_guild_state_once_test() ->+ with_region_stubs(fun() ->+ {Calls, _Dispatches, _Pending} = run_region_switch(fun switch_voice_region/3),+ ?assertEqual(1, Calls)+ end).++switch_voice_region_matches_per_user_refetch_reference_test() ->+ with_region_stubs(fun region_reference_equivalence_scenario/0).++region_reference_equivalence_scenario() ->+ {RefCalls, RefDispatches, RefPending} = run_region_switch(+ fun reference_switch_voice_region/3+ ),+ {NewCalls, NewDispatches, NewPending} = run_region_switch(fun switch_voice_region/3),+ ?assertEqual(3, RefCalls),+ ?assertEqual(1, NewCalls),+ ?assertEqual(2, length(RefDispatches)),+ ?assertEqual(RefDispatches, NewDispatches),+ ?assertEqual(RefPending, NewPending).++switch_voice_region_skips_users_without_a_session_id_test() ->+ with_region_stubs(fun region_missing_session_id_scenario/0).++region_missing_session_id_scenario() ->+ Anonymous = maps:remove(+ <<"session_id">>, region_voice_state(<<"conn-c">>, 10, <<"sess1">>)+ ),+ State = region_state(),+ Stateless = State#{voice_states => #{<<"conn-c">> => Anonymous}},+ {Calls, Dispatches, Pending} = run_region_switch_with(+ fun switch_voice_region/3, Stateless+ ),+ ?assertEqual(1, Calls),+ ?assertEqual([], Dispatches),+ ?assertEqual([], Pending).++%% The pre-change implementation, kept as a live oracle: it refetched the guild state once per+%% user in the channel instead of reusing the one the enumeration already fetched.+reference_switch_voice_region(GuildId, ChannelId, GuildPid) ->+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) of+ State when is_map(State) ->+ reference_do_switch_voice_region(GuildId, ChannelId, GuildPid, State);+ _ ->+ ok+ end.++reference_do_switch_voice_region(GuildId, ChannelId, GuildPid, State) ->+ VoiceStates = voice_state_utils:voice_states(State),+ UsersInChannel = collect_users_in_channel(VoiceStates, ChannelId),+ lists:foreach(+ fun(UserInfo) -> reference_send_update(GuildId, ChannelId, GuildPid, UserInfo) end,+ UsersInChannel+ ).++reference_send_update(_GuildId, _ChannelId, _GuildPid, {_UserId, undefined, _ConnId, _VS}) ->+ ok;+reference_send_update(GuildId, ChannelId, GuildPid, {UserId, SessionId, ConnId, VoiceState}) ->+ case guild_voice_server_state:guild_state_call(GuildPid, 10000) of+ State when is_map(State) ->+ request_and_broadcast_region_switch(+ GuildId, ChannelId, UserId, SessionId, ConnId, VoiceState, GuildPid, State+ );+ _ ->+ ok+ end.++run_region_switch(Fun) ->+ run_region_switch_with(Fun, region_state()).++run_region_switch_with(Fun, State) ->+ drain_region_dispatches(),+ drain_region_pending(),+ erlang:put(region_token_index, 0),+ GuildPid = spawn_region_guild(State),+ try+ Fun(999, 200, GuildPid),+ {region_guild_calls(GuildPid), drain_region_dispatches(), drain_region_pending()}+ after+ GuildPid ! stop+ end.++with_region_stubs(Fun) ->+ meck:new(rpc_client, [passthrough, no_link, non_strict]),+ meck:expect(rpc_client, call, fun(_Request) -> {ok, next_region_token()} end),+ meck:new(guild_voice_server, [passthrough, no_link]),+ meck:expect(guild_voice_server, resolve, fun(_GuildId, GuildPid) -> GuildPid end),+ try+ Fun()+ after+ meck:unload(guild_voice_server),+ meck:unload(rpc_client)+ end.++next_region_token() ->+ Index = erlang:get(region_token_index) + 1,+ erlang:put(region_token_index, Index),+ #{+ <<"token">> => <<"tok-", (integer_to_binary(Index))/binary>>,+ <<"endpoint">> => <<"wss://voice.example">>,+ <<"connectionId">> => <<"new-conn-", (integer_to_binary(Index))/binary>>,+ <<"regionId">> => <<"us-east">>,+ <<"serverId">> => <<"voice-1">>+ }.++spawn_region_guild(State) ->+ Owner = self(),+ spawn(fun() -> region_guild_loop(State, Owner, 0) end).++region_guild_loop(State, Owner, Calls) ->+ receive+ {'$gen_call', From, {get_voice_guild_state}} ->+ gen_server:reply(From, State),+ region_guild_loop(State, Owner, Calls + 1);+ {'$gen_call', From, {store_pending_connection, ConnId, Meta}} ->+ Owner ! {region_pending, ConnId, Meta},+ gen_server:reply(From, ok),+ region_guild_loop(State, Owner, Calls);+ {'$gen_call', From, _Other} ->+ gen_server:reply(From, ok),+ region_guild_loop(State, Owner, Calls);+ {region_calls, Caller} ->+ Caller ! {region_calls, Calls},+ region_guild_loop(State, Owner, Calls);+ stop ->+ ok+ after 30000 ->+ ok+ end.++region_guild_calls(GuildPid) ->+ GuildPid ! {region_calls, self()},+ receive+ {region_calls, Calls} -> Calls+ after 1000 ->+ error(region_guild_unresponsive)+ end.++drain_region_dispatches() ->+ receive+ {'$gen_cast', {dispatch, voice_server_update, Payload}} ->+ [Payload | drain_region_dispatches()]+ after 100 ->+ []+ end.++%% token_nonce and the timestamps are freshly generated per call by design, so they are+%% excluded before the reference and the threaded run are compared.+drain_region_pending() ->+ receive+ {region_pending, ConnId, Meta} ->+ Stable = maps:without([token_nonce, created_at, expires_at], Meta),+ [{ConnId, Stable} | drain_region_pending()]+ after 100 ->+ []+ end.++region_state() ->+ #{+ id => 999,+ data => #{+ <<"id">> => <<"999">>,+ <<"guild">> => #{<<"owner_id">> => <<"10">>},+ <<"channels">> => [#{<<"id">> => <<"200">>, <<"type">> => 2}],+ <<"members">> => [region_member(10), region_member(11)]+ },+ sessions => #{+ <<"sess1">> => #{user_id => 10, pid => self(), pending_connect => false},+ <<"sess2">> => #{user_id => 11, pid => self(), pending_connect => false}+ },+ voice_states => #{+ <<"conn-a">> => region_voice_state(<<"conn-a">>, 10, <<"sess1">>),+ <<"conn-b">> => region_voice_state(<<"conn-b">>, 11, <<"sess2">>)+ },+ pending_voice_connections => #{}+ }.++region_member(UserId) ->+ #{+ <<"user">> => #{<<"id">> => integer_to_binary(UserId)},+ <<"mute">> => false,+ <<"deaf">> => false+ }.++region_voice_state(ConnId, UserId, SessionId) ->+ #{+ <<"user_id">> => integer_to_binary(UserId),+ <<"guild_id">> => <<"999">>,+ <<"channel_id">> => <<"200">>,+ <<"connection_id">> => ConnId,+ <<"session_id">> => SessionId,+ <<"self_mute">> => false,+ <<"self_deaf">> => true,+ <<"member">> => region_member(UserId)+ }.+build_pending_metadata_preserves_voice_routing_test() ->Metadata = build_pending_metadata(10,diff --git a/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl b/fluxer_gateway/src/guild/voice/guild_voice_server_state.erlindex d74c665db..4f77dc31f 100644--- a/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl+++ b/fluxer_gateway/src/guild/voice/guild_voice_server_state.erl@@ -13,7 +13,9 @@repair_voice_state_from_guild_cache/2,voice_state_rpc_entries/1,pending_join_rpc_entries/1,- fetch_guild_data/1+ fetch_guild_data/1,+ guild_state_call/2,+ guild_id_call/2]).-export_type([@@ -335,9 +337,36 @@ pending_join_channel_id(Metadata) when is_map(Metadata) ->pending_join_channel_id(_) ->undefined.+-spec guild_state_call(pid(), timeout()) -> term().+guild_state_call(GuildPid, Timeout) ->+ case gen_server:call(GuildPid, {get_voice_guild_state}, Timeout) of+ GuildState when is_map(GuildState) -> GuildState;+ _ -> full_guild_state_call(GuildPid, Timeout)+ end.++-spec full_guild_state_call(pid(), timeout()) -> term().+full_guild_state_call(GuildPid, Timeout) ->+ gen_server:call(GuildPid, {get_sessions}, Timeout).++-spec guild_id_call(pid(), timeout()) -> integer() | undefined.+guild_id_call(GuildPid, Timeout) ->+ case gen_server:call(GuildPid, {get_guild_id}, Timeout) of+ GuildId when is_integer(GuildId) -> GuildId;+ _ -> guild_state_id(full_guild_state_call(GuildPid, Timeout))+ end.++-spec guild_state_id(term()) -> integer() | undefined.+guild_state_id(GuildState) when is_map(GuildState) ->+ case maps:get(id, GuildState, undefined) of+ GuildId when is_integer(GuildId) -> GuildId;+ _ -> undefined+ end;+guild_state_id(_) ->+ undefined.+-spec fetch_guild_data(pid()) -> map().fetch_guild_data(GuildPid) ->- try gen_server:call(GuildPid, {get_sessions}, ?GUILD_CALL_TIMEOUT) of+ try guild_state_call(GuildPid, ?GUILD_CALL_TIMEOUT) ofGuildState when is_map(GuildState) -> GuildState;_ -> #{}catch@@ -407,3 +436,89 @@ normalize_rpc_millisecond(Value) when is_binary(Value), byte_size(Value) > 0 ->end;normalize_rpc_millisecond(_) ->0.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++guild_state_call_uses_projection_test() ->+ with_fake_guild(fun new_guild_handler/1, fun(Pid) ->+ ?assertEqual(projected_guild_state(), guild_state_call(Pid, 1000)),+ ?assertEqual([{get_voice_guild_state}], drain_requests())+ end).++guild_state_call_retries_get_sessions_against_old_peer_test() ->+ with_fake_guild(fun old_guild_handler/1, fun(Pid) ->+ ?assertEqual(full_guild_state(), guild_state_call(Pid, 1000)),+ ?assertEqual([{get_voice_guild_state}, {get_sessions}], drain_requests())+ end).++fetch_guild_data_retries_get_sessions_against_old_peer_test() ->+ with_fake_guild(fun old_guild_handler/1, fun(Pid) ->+ ?assertEqual(full_guild_state(), fetch_guild_data(Pid)),+ ?assertEqual([{get_voice_guild_state}, {get_sessions}], drain_requests())+ end).++guild_id_call_uses_get_guild_id_test() ->+ with_fake_guild(fun new_guild_handler/1, fun(Pid) ->+ ?assertEqual(42, guild_id_call(Pid, 1000)),+ ?assertEqual([{get_guild_id}], drain_requests())+ end).++guild_id_call_returns_real_id_against_old_peer_test() ->+ with_fake_guild(fun old_guild_handler/1, fun(Pid) ->+ ?assertEqual(42, guild_id_call(Pid, 1000)),+ ?assertEqual([{get_guild_id}, {get_sessions}], drain_requests())+ end).++guild_id_call_returns_undefined_when_peer_has_no_id_test() ->+ with_fake_guild(fun(_) -> ok end, fun(Pid) ->+ ?assertEqual(undefined, guild_id_call(Pid, 1000)),+ ?assertEqual([{get_guild_id}, {get_sessions}], drain_requests())+ end).++new_guild_handler({get_voice_guild_state}) -> projected_guild_state();+new_guild_handler({get_guild_id}) -> 42;+new_guild_handler({get_sessions}) -> full_guild_state();+new_guild_handler(_) -> ok.++old_guild_handler({get_sessions}) -> full_guild_state();+old_guild_handler(_) -> ok.++full_guild_state() ->+ #{+ id => 42,+ sessions => #{},+ voice_states => #{},+ data => #{<<"id">> => <<"42">>, <<"members">> => #{}}+ }.++projected_guild_state() ->+ #{id => 42, sessions => #{}, voice_states => #{}, data => #{<<"id">> => <<"42">>}}.++with_fake_guild(Handler, Fun) ->+ _ = drain_requests(),+ Owner = self(),+ Pid = spawn(fun() -> fake_guild_loop(Owner, Handler) end),+ try+ Fun(Pid)+ after+ Pid ! stop+ end.++fake_guild_loop(Owner, Handler) ->+ receive+ {'$gen_call', From, Request} ->+ Owner ! {fake_guild_request, Request},+ gen_server:reply(From, Handler(Request)),+ fake_guild_loop(Owner, Handler);+ stop ->+ ok+ end.++drain_requests() ->+ receive+ {fake_guild_request, Request} -> [Request | drain_requests()]+ after 0 -> []+ end.++-endif.diff --git a/fluxer_gateway/src/presence/presence_broadcast.erl b/fluxer_gateway/src/presence/presence_broadcast.erlindex a5d1bf20f..b7988af38 100644--- a/fluxer_gateway/src/presence/presence_broadcast.erl+++ b/fluxer_gateway/src/presence/presence_broadcast.erl@@ -30,6 +30,8 @@-type state() :: map().-type sessions() :: #{binary() => map()}.+-define(CUSTOM_STATUS_EXPIRY_TIMER, custom_status_expiry_timer).+-spec publish_global_if_needed({reply, term(), state()} | {noreply, state()}) ->{reply, term(), state()} | {noreply, state()}.publish_global_if_needed({reply, Reply, NewState}) ->@@ -40,7 +42,8 @@ publish_global_if_needed({noreply, NewState}) ->{noreply, FinalState}.-spec publish_global_presence(sessions(), state()) -> state().-publish_global_presence(_Sessions, State) ->+publish_global_presence(_Sessions, State0) ->+ State = refresh_custom_status_expiry_timer(State0),{Payload, CurrentExternal, ExternalStatus} = build_presence_external(State),LastPublished = maps:get(last_published_presence, State, undefined),case presence_changed(LastPublished, CurrentExternal) of@@ -55,11 +58,33 @@ publish_global_presence(_Sessions, State) ->end.-spec force_publish_global_presence(state()) -> state().-force_publish_global_presence(State) ->+force_publish_global_presence(State0) ->+ State = refresh_custom_status_expiry_timer(State0),{Payload, CurrentExternal, ExternalStatus} = build_presence_external(State),NewState = publish_presence_payload(State, Payload, CurrentExternal, ExternalStatus),presence_update:maybe_update_push_eligibility(NewState).+-spec refresh_custom_status_expiry_timer(state()) -> state().+refresh_custom_status_expiry_timer(State) ->+ ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State, undefined)),+ arm_expiry_timer(+ custom_status_expiry:next_wakeup_ms(maps:get(custom_status, State, null)), State+ ).++-spec arm_expiry_timer({ok, pos_integer()} | none, state()) -> state().+arm_expiry_timer(none, State) ->+ maps:remove(?CUSTOM_STATUS_EXPIRY_TIMER, State);+arm_expiry_timer({ok, DelayMs}, State) ->+ Ref = erlang:send_after(DelayMs, self(), custom_status_expiry:reconcile_message()),+ State#{?CUSTOM_STATUS_EXPIRY_TIMER => Ref}.++-spec cancel_expiry_timer(term()) -> ok.+cancel_expiry_timer(Ref) when is_reference(Ref) ->+ _ = erlang:cancel_timer(Ref, [{async, true}, {info, false}]),+ ok;+cancel_expiry_timer(_Ref) ->+ ok.+-spec dispatch_global_presence(user_id(), map(), state()) -> {noreply, state()}.dispatch_global_presence(TargetId, Payload, State) ->UserId = maps:get(user_id, State),@@ -240,3 +265,97 @@ dispatch_global_user_update(TargetId, Payload, State) ->dispatch_to_sessions(Payload, State) ->SessionPids = presence_connect:collect_session_pids(State),gateway_dispatch_relay:dispatch_many(SessionPids, presence_update, Payload).++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++expiry_timer_not_armed_while_disabled_test() ->+ application:unset_env(fluxer_gateway, custom_status_expiry_enabled),+ State = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),+ ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, State)).++expiry_timer_not_armed_without_expires_at_test() ->+ with_expiry_enabled(fun() ->+ ?assertNot(+ maps:is_key(+ ?CUSTOM_STATUS_EXPIRY_TIMER,+ refresh_custom_status_expiry_timer(#{custom_status => null})+ )+ ),+ ?assertNot(+ maps:is_key(+ ?CUSTOM_STATUS_EXPIRY_TIMER,+ refresh_custom_status_expiry_timer(#{+ custom_status => #{<<"text">> => <<"hi">>}+ })+ )+ )+ end).++expiry_timer_armed_for_a_future_expiry_test() ->+ with_expiry_enabled(fun() ->+ State = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),+ ?assert(is_reference(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))),+ ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))+ end).++expiry_timer_not_armed_for_an_already_expired_status_test() ->+ with_expiry_enabled(fun() ->+ Expired = #{<<"expires_at">> => <<"2020-01-01T00:00:00.000Z">>},+ State = refresh_custom_status_expiry_timer(#{custom_status => Expired}),+ ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, State))+ end).++expiry_timer_replaces_the_previous_one_test() ->+ with_expiry_enabled(fun() ->+ First = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),+ Second = refresh_custom_status_expiry_timer(First),+ ?assertNotEqual(+ maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, First),+ maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, Second)+ ),+ ok = cancel_expiry_timer(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, Second))+ end).++expiry_timer_is_dropped_when_the_status_is_cleared_test() ->+ with_expiry_enabled(fun() ->+ Armed = refresh_custom_status_expiry_timer(#{custom_status => future_custom_status()}),+ Cleared = refresh_custom_status_expiry_timer(Armed#{custom_status => null}),+ ?assertNot(maps:is_key(?CUSTOM_STATUS_EXPIRY_TIMER, Cleared))+ end).++expiry_timer_delivers_a_reconcile_cast_test() ->+ State = arm_expiry_timer({ok, 1}, #{}),+ ?assert(is_reference(maps:get(?CUSTOM_STATUS_EXPIRY_TIMER, State))),+ Expected = custom_status_expiry:reconcile_message(),+ receive+ Expected -> ok+ after 1000 -> ?assert(false)+ end.++cancel_expiry_timer_ignores_non_references_test() ->+ ?assertEqual(ok, cancel_expiry_timer(undefined)),+ ?assertEqual(ok, cancel_expiry_timer(not_a_ref)).++future_custom_status() ->+ ExpiresAt = calendar:system_time_to_rfc3339(+ erlang:system_time(millisecond) + 3600000, [{unit, millisecond}, {offset, "Z"}]+ ),+ #{<<"text">> => <<"brb">>, <<"expires_at">> => list_to_binary(ExpiresAt)}.++with_expiry_enabled(Fun) ->+ Key = custom_status_expiry_enabled,+ Previous = application:get_env(fluxer_gateway, Key),+ application:set_env(fluxer_gateway, Key, true),+ try+ Fun()+ after+ restore_expiry_env(Key, Previous)+ end.++restore_expiry_env(Key, undefined) ->+ application:unset_env(fluxer_gateway, Key);+restore_expiry_env(Key, {ok, Value}) ->+ application:set_env(fluxer_gateway, Key, Value).++-endif.diff --git a/fluxer_gateway/src/presence/presence_payload.erl b/fluxer_gateway/src/presence/presence_payload.erlindex 44c2ed6c2..a1ea69bf3 100644--- a/fluxer_gateway/src/presence/presence_payload.erl+++ b/fluxer_gateway/src/presence/presence_payload.erl@@ -32,9 +32,12 @@ ensure_status_binary(Status) when is_binary(Status) -> Status;ensure_status_binary(_) -> <<"offline">>.-spec custom_status_for(binary(), custom_status()) -> custom_status().-custom_status_for(<<"offline">>, _CustomStatus) -> null;-custom_status_for(<<"invisible">>, _CustomStatus) -> null;-custom_status_for(_StatusBin, CustomStatus) -> normalize_custom_status(CustomStatus).+custom_status_for(<<"offline">>, _CustomStatus) ->+ null;+custom_status_for(<<"invisible">>, _CustomStatus) ->+ null;+custom_status_for(_StatusBin, CustomStatus) ->+ custom_status_expiry:clear_if_expired(normalize_custom_status(CustomStatus)).-spec normalize_custom_status(term()) -> custom_status().normalize_custom_status(null) -> null;@@ -81,16 +84,88 @@ normalize_custom_status_test() ->?assertEqual(null, normalize_custom_status(123)).build_invisible_atom_normalized_to_offline_test() ->- User = #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>},CustomStatus = #{<<"text">> => <<"hello">>},- Result = build(User, invisible, false, false, CustomStatus),+ Result = build(test_user(), invisible, false, false, CustomStatus),?assertEqual(<<"offline">>, maps:get(<<"status">>, Result)),?assertEqual(null, maps:get(<<"custom_status">>, Result)).build_invisible_binary_normalized_to_offline_test() ->- User = #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>},CustomStatus = #{<<"text">> => <<"hello">>},- Result = build(User, <<"invisible">>, false, false, CustomStatus),+ Result = build(test_user(), <<"invisible">>, false, false, CustomStatus),?assertEqual(<<"offline">>, maps:get(<<"status">>, Result)),?assertEqual(null, maps:get(<<"custom_status">>, Result)).++build_clears_expired_custom_status_test() ->+ with_expiry_enabled(fun() ->+ Result = build(test_user(), online, false, false, expired_custom_status()),+ ?assertEqual(<<"online">>, maps:get(<<"status">>, Result)),+ ?assertEqual(null, maps:get(<<"custom_status">>, Result))+ end).++build_coalesces_an_expired_status_with_no_status_test() ->+ with_expiry_enabled(fun() ->+ ?assertEqual(+ build(test_user(), online, false, false, null),+ build(test_user(), online, false, false, expired_custom_status())+ )+ end).++build_keeps_unexpired_custom_status_test() ->+ with_expiry_enabled(fun() ->+ Live = future_custom_status(),+ Result = build(test_user(), dnd, false, false, Live),+ ?assertEqual(Live, maps:get(<<"custom_status">>, Result))+ end).++build_keeps_custom_status_without_expires_at_test() ->+ with_expiry_enabled(fun() ->+ Live = #{<<"text">> => <<"forever">>},+ Result = build(test_user(), idle, false, false, Live),+ ?assertEqual(Live, maps:get(<<"custom_status">>, Result))+ end).++build_keeps_malformed_expires_at_test() ->+ with_expiry_enabled(fun() ->+ Live = #{<<"text">> => <<"hi">>, <<"expires_at">> => <<"not-a-date">>},+ Result = build(test_user(), online, false, false, Live),+ ?assertEqual(Live, maps:get(<<"custom_status">>, Result))+ end).++build_leaves_expired_status_alone_while_disabled_test() ->+ application:unset_env(fluxer_gateway, custom_status_expiry_enabled),+ Expired = expired_custom_status(),+ Result = build(test_user(), online, false, false, Expired),+ ?assertEqual(Expired, maps:get(<<"custom_status">>, Result)).++test_user() ->+ #{<<"id">> => <<"1">>, <<"username">> => <<"Test">>}.++expired_custom_status() ->+ #{+ <<"emoji_animated">> => false,+ <<"emoji_name">> => null,+ <<"expires_at">> => <<"2026-05-13T13:02:27.497Z">>,+ <<"text">> => <<"brb">>+ }.++future_custom_status() ->+ ExpiresAt = calendar:system_time_to_rfc3339(+ erlang:system_time(millisecond) + 3600000, [{unit, millisecond}, {offset, "Z"}]+ ),+ #{<<"text">> => <<"brb">>, <<"expires_at">> => list_to_binary(ExpiresAt)}.++with_expiry_enabled(Fun) ->+ Key = custom_status_expiry_enabled,+ Previous = application:get_env(fluxer_gateway, Key),+ application:set_env(fluxer_gateway, Key, true),+ try+ Fun()+ after+ restore_expiry_env(Key, Previous)+ end.++restore_expiry_env(Key, undefined) ->+ application:unset_env(fluxer_gateway, Key);+restore_expiry_env(Key, {ok, Value}) ->+ application:set_env(fluxer_gateway, Key, Value).-endif.diff --git a/fluxer_gateway/src/push/push.erl b/fluxer_gateway/src/push/push.erlindex f721926dc..d7508c7d1 100644--- a/fluxer_gateway/src/push/push.erl+++ b/fluxer_gateway/src/push/push.erl@@ -25,6 +25,36 @@-define(EVICT_INTERVAL_MS, 60000).-define(DEFAULT_MAX_ENTRIES, 500000).+-define(PUSH_COUNTER_TABLE, push_worker_counter).+-define(CNT_WORKER_POOL, push_loss_worker_pool).+-define(CNT_WARM_INFLIGHT, push_blocked_ids_warm_inflight).+-define(CNT_FETCH_ATTEMPTS, push_blocked_ids_fetch_attempts).+-define(CNT_FETCH_FAILURES, push_blocked_ids_fetch_failures).+-define(CNT_SUPPRESSED, push_blocked_ids_suppressed).+-define(CNT_BUDGET_EXHAUSTED, push_blocked_ids_budget_exhausted).+-define(CNT_WARM_DROPPED, push_blocked_ids_warm_dropped).+-define(CNT_DISPATCH_DROPPED, push_loss_dispatch_dropped).+-define(CNT_DISPATCH_DROPPED_USERS, push_loss_dispatch_dropped_users).+-define(CNT_CLEAR_DROPPED, push_loss_clear_dropped).+-define(CNT_QUEUE_FULL, push_loss_queue_full).+-define(CNT_INVALID_JOB, push_loss_invalid_job).+-define(CNT_ENQUEUE_TIMEOUT, push_loss_enqueue_timeout).+-define(CNT_ENQUEUE_FAILED, push_loss_enqueue_failed).+-define(CNT_JOB_CRASHED, push_loss_job_crashed).+-define(CNT_WORKER_DIED, push_loss_worker_died).+-define(CNT_DISPATCHER_RESTARTS, push_dispatcher_restarts).+-define(CNT_RESTART_DISCARDED, push_dispatcher_restart_discarded).+-define(CNT_QUEUE_ENQUEUED, push_dispatcher_queue_enqueued).+-define(CNT_QUEUE_DEQUEUED, push_dispatcher_queue_dequeued).++-define(MAX_WARM_INFLIGHT, 4).+-define(MAX_FETCH_RPCS, 8).+-define(DEFAULT_FETCH_USERS, 2000).+-define(MAX_FETCH_USERS, 5000).+-define(DEFAULT_FETCH_CHUNK, 500).+-define(MIN_FETCH_CHUNK, 50).+-define(MAX_FETCH_CHUNK, 1000).+-type state() :: #{badge_counts_ttl_seconds := non_neg_integer(),max_entries := non_neg_integer()@@ -61,8 +91,7 @@ init([]) ->-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}.handle_call(get_cache_stats, _From, State) ->- Stats = push_ets_cache:cache_stats(),- {reply, {ok, Stats}, State};+ {reply, {ok, cache_stats_with_counters()}, State};handle_call(_Request, _From, State) ->{reply, ok, State}.@@ -96,6 +125,7 @@ handle_cast(_Msg, State) ->-spec handle_info(term(), state()) -> {noreply, state()}.handle_info(evict_caches, State) ->+ unstick_warm_gate(),MaxEntries = maps:get(max_entries, State),push_ets_cache:evict_tables(#{user_guild_settings => MaxEntries,@@ -199,9 +229,7 @@ is_push_noop() ->-spec clear_channel_notifications(integer(), integer(), integer()) -> ok.clear_channel_notifications(UserId, ChannelId, MessageId) ->- case- is_push_active() andalso persistent_term:get(push_clear_notifications_enabled, false)- of+ case is_push_active() andalso clear_notifications_enabled() oftrue ->cast_to_push_owner(UserId, {clear_channel_notifications, UserId, ChannelId, MessageId}@@ -210,6 +238,13 @@ clear_channel_notifications(UserId, ChannelId, MessageId) ->okend.+-spec clear_notifications_enabled() -> boolean().+clear_notifications_enabled() ->+ case persistent_term:get(push_clear_notifications_enabled, undefined) of+ Value when is_boolean(Value) -> Value;+ _ -> env_boolean(push_clear_notifications_enabled, false)+ end.+-spec get_cache_stats() -> {ok, map()}.get_cache_stats() ->gen_server:call(?MODULE, get_cache_stats, 5000).@@ -299,13 +334,13 @@ do_handle_message_create_context(Context, State) ->MessageData,GuildDefaultNotifications,UserRolesMap,- ConnectedUsers+ ConnectedUsers,+ maps:get(large_guild_metadata, Context, undefined)),logger:debug("Push: eligibility result", #{message_id => MessageId,channel_id => ChannelId,- eligible_count => length(EligibleUsers),- eligible_user_ids => EligibleUsers+ eligible_count => length(EligibleUsers)}),dispatch_if_eligible(EligibleUsers,@@ -320,7 +355,15 @@ do_handle_message_create_context(Context, State) ->).-spec filter_eligible_users(- [integer()], integer(), integer(), integer(), map(), integer(), map(), map()+ [integer()],+ integer(),+ integer(),+ integer(),+ map(),+ integer(),+ map(),+ map(),+ map() | undefined) -> [integer()].filter_eligible_users(UserIds,@@ -330,11 +373,13 @@ filter_eligible_users(MessageData,GuildDefaultNotifications,UserRolesMap,- ConnectedUsers+ ConnectedUsers,+ SuppliedMetadata) ->- LargeGuildMetadata = large_guild_metadata(GuildId),- push_eligibility:prefetch_user_guild_settings(UserIds, AuthorId, GuildId),- lists:filter(+ LargeGuildMetadata = resolve_large_guild_metadata(GuildId, SuppliedMetadata),+ Candidates = drop_blocked_recipients(UserIds, AuthorId),+ push_eligibility:prefetch_user_guild_settings(Candidates, AuthorId, GuildId),+ EligibleUsers = lists:filter(fun(UserId) ->push_eligibility:is_eligible_for_push(UserId,@@ -348,15 +393,249 @@ filter_eligible_users(LargeGuildMetadata)end,- UserIds- ).+ Candidates+ ),+ warm_blocked_ids(EligibleUsers),+ EligibleUsers.--spec large_guild_metadata(integer()) -> map() | undefined.-large_guild_metadata(0) ->+-spec resolve_large_guild_metadata(integer(), map() | undefined) -> map() | undefined.+resolve_large_guild_metadata(0, _SuppliedMetadata) ->undefined;-large_guild_metadata(GuildId) ->+resolve_large_guild_metadata(GuildId, SuppliedMetadata) ->+ supplied_or_local_metadata(GuildId, SuppliedMetadata).++-spec supplied_or_local_metadata(integer(), map() | undefined) -> map() | undefined.+supplied_or_local_metadata(GuildId, undefined) ->+ large_guild_metadata_local(GuildId);+supplied_or_local_metadata(_GuildId, SuppliedMetadata) when is_map(SuppliedMetadata) ->+ SuppliedMetadata;+supplied_or_local_metadata(GuildId, _SuppliedMetadata) ->+ large_guild_metadata_local(GuildId).++-spec large_guild_metadata_local(integer()) -> map() | undefined.+large_guild_metadata_local(GuildId) ->push_eligibility_checks:get_guild_large_metadata(GuildId).+-spec drop_blocked_recipients([integer()], integer()) -> [integer()].+drop_blocked_recipients(UserIds, AuthorId) ->+ Kept = lists:filter(+ fun(UserId) -> not push_eligibility:is_user_blocked(UserId, AuthorId) end,+ UserIds+ ),+ count_suppressed(length(UserIds) - length(Kept)),+ Kept.++-spec count_suppressed(non_neg_integer()) -> ok.+count_suppressed(0) ->+ ok;+count_suppressed(Suppressed) ->+ bump_counter(?CNT_SUPPRESSED, Suppressed).++-spec warm_blocked_ids([integer()]) -> ok.+warm_blocked_ids([]) ->+ ok;+warm_blocked_ids(EligibleUsers) ->+ case missing_blocked_ids(EligibleUsers) of+ [] -> ok;+ Missing -> spawn_blocked_ids_warm(Missing)+ end.++-spec missing_blocked_ids([integer()]) -> [integer()].+missing_blocked_ids(EligibleUsers) ->+ lists:usort(lists:filter(fun is_blocked_ids_cache_miss/1, EligibleUsers)).++-spec is_blocked_ids_cache_miss(integer()) -> boolean().+is_blocked_ids_cache_miss(UserId) ->+ push_ets_cache:get_blocked_ids(UserId) =:= undefined.++-spec spawn_blocked_ids_warm([integer()]) -> ok.+spawn_blocked_ids_warm(Missing) ->+ case claim_warm_slot() of+ ok ->+ _ = spawn(fun() -> run_blocked_ids_warm(Missing) end),+ ok;+ full ->+ bump_counter(?CNT_WARM_DROPPED)+ end.++-spec run_blocked_ids_warm([integer()]) -> ok.+run_blocked_ids_warm(Missing) ->+ try+ warm_within_budget(Missing)+ catch+ throw:Reason -> warm_crashed(throw, Reason);+ error:Reason -> warm_crashed(error, Reason);+ exit:Reason -> warm_crashed(exit, Reason)+ after+ release_warm_slot()+ end.++-spec warm_crashed(throw | error | exit, term()) -> ok.+warm_crashed(Class, Reason) ->+ bump_counter(?CNT_FETCH_FAILURES),+ logger:debug("Push: blocked id warm crashed", #{class => Class, reason => Reason}),+ ok.++-spec warm_within_budget([integer()]) -> ok.+warm_within_budget(Missing) ->+ {Budget, ChunkSize} = blocked_ids_fetch_budget(),+ Budgeted = lists:sublist(Missing, Budget),+ count_budget_exhausted(length(Missing) - length(Budgeted)),+ fetch_blocked_ids_chunks(chunk_user_ids(Budgeted, ChunkSize, [])).++-spec blocked_ids_fetch_budget() -> {pos_integer(), pos_integer()}.+blocked_ids_fetch_budget() ->+ ChunkSize = blocked_ids_fetch_chunk(),+ MaxUsers = blocked_ids_fetch_max_users(),+ {min(MaxUsers, ChunkSize * ?MAX_FETCH_RPCS), ChunkSize}.++-spec count_budget_exhausted(non_neg_integer()) -> ok.+count_budget_exhausted(0) ->+ ok;+count_budget_exhausted(Skipped) ->+ bump_counter(?CNT_BUDGET_EXHAUSTED, Skipped).++-spec fetch_blocked_ids_chunks([[integer()]]) -> ok.+fetch_blocked_ids_chunks([]) ->+ ok;+fetch_blocked_ids_chunks([Chunk | Rest]) ->+ case fetch_blocked_ids_chunk(Chunk) of+ ok -> fetch_blocked_ids_chunks(Rest);+ error -> ok+ end.++-spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].+chunk_user_ids([], _ChunkSize, Acc) ->+ lists:reverse(Acc);+chunk_user_ids(UserIds, ChunkSize, Acc) ->+ Chunk = lists:sublist(UserIds, ChunkSize),+ chunk_user_ids(drop_prefix(UserIds, ChunkSize), ChunkSize, [Chunk | Acc]).++-spec drop_prefix([integer()], non_neg_integer()) -> [integer()].+drop_prefix(UserIds, 0) ->+ UserIds;+drop_prefix([], _N) ->+ [];+drop_prefix([_UserId | Rest], N) ->+ drop_prefix(Rest, N - 1).++-spec fetch_blocked_ids_chunk([integer()]) -> ok | error.+fetch_blocked_ids_chunk([]) ->+ ok;+fetch_blocked_ids_chunk(UserIds) ->+ bump_counter(?CNT_FETCH_ATTEMPTS),+ Request = #{+ <<"type">> => <<"get_user_blocked_ids">>,+ <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]+ },+ case rpc_client:call(Request) of+ {ok, Data} when is_map(Data) ->+ cache_blocked_ids_response(UserIds, Data);+ {ok, _Data} ->+ fetch_blocked_ids_failed(malformed_response, length(UserIds));+ {error, Reason} ->+ fetch_blocked_ids_failed(Reason, length(UserIds))+ end.++-spec fetch_blocked_ids_failed(term(), non_neg_integer()) -> error.+fetch_blocked_ids_failed(Reason, UserCount) ->+ bump_counter(?CNT_FETCH_FAILURES),+ logger:debug("Push: blocked id fetch failed", #{+ reason => Reason, user_count => UserCount+ }),+ error.++-spec cache_blocked_ids_response([integer()], map()) -> ok.+cache_blocked_ids_response(UserIds, Data) ->+ lists:foreach(+ fun(UserId) -> cache_blocked_ids_entry(UserId, Data) end,+ UserIds+ ),+ ok.++-spec cache_blocked_ids_entry(integer(), map()) -> ok.+cache_blocked_ids_entry(UserId, Data) ->+ Raw = maps:get(integer_to_binary(UserId), Data, []),+ push_ets_cache:put_blocked_ids_fetched(UserId, blocked_ids_from_response(Raw)).++-spec blocked_ids_from_response(term()) -> [integer()].+blocked_ids_from_response(Values) when is_list(Values) ->+ lists:filtermap(fun snowflake_id:filter/1, Values);+blocked_ids_from_response(_Values) ->+ [].++-spec blocked_ids_fetch_max_users() -> pos_integer().+blocked_ids_fetch_max_users() ->+ Value = app_pos_integer(push_blocked_ids_fetch_max_users, ?DEFAULT_FETCH_USERS),+ min(Value, ?MAX_FETCH_USERS).++-spec blocked_ids_fetch_chunk() -> pos_integer().+blocked_ids_fetch_chunk() ->+ Value = app_pos_integer(push_blocked_ids_fetch_chunk, ?DEFAULT_FETCH_CHUNK),+ min(max(Value, ?MIN_FETCH_CHUNK), ?MAX_FETCH_CHUNK).++-spec app_pos_integer(atom(), pos_integer()) -> pos_integer().+app_pos_integer(Key, Default) ->+ case application:get_env(fluxer_gateway, Key, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> Default+ end.++-spec blocked_ids_counters() -> map().+blocked_ids_counters() ->+ #{+ blocked_ids_fetch_attempts => read_counter(?CNT_FETCH_ATTEMPTS),+ blocked_ids_fetch_failures => read_counter(?CNT_FETCH_FAILURES),+ blocked_ids_suppressed => read_counter(?CNT_SUPPRESSED),+ blocked_ids_budget_exhausted => read_counter(?CNT_BUDGET_EXHAUSTED),+ blocked_ids_warm_dropped => read_counter(?CNT_WARM_DROPPED),+ blocked_ids_warm_inflight => read_counter(?CNT_WARM_INFLIGHT)+ }.++-spec claim_warm_slot() -> ok | full.+claim_warm_slot() ->+ try ets:update_counter(?PUSH_COUNTER_TABLE, ?CNT_WARM_INFLIGHT, {2, 1}) of+ Value when is_integer(Value), Value =< ?MAX_WARM_INFLIGHT ->+ ok;+ _Value ->+ release_warm_slot(),+ full+ catch+ error:badarg -> claim_first_warm_slot()+ end.++-spec claim_first_warm_slot() -> ok | full.+claim_first_warm_slot() ->+ try ets:insert_new(?PUSH_COUNTER_TABLE, {?CNT_WARM_INFLIGHT, 1}) of+ true -> ok;+ false -> full+ catch+ error:badarg -> full+ end.++-spec release_warm_slot() -> ok.+release_warm_slot() ->+ try ets:update_counter(?PUSH_COUNTER_TABLE, ?CNT_WARM_INFLIGHT, {2, -1, 0, 0}) of+ _Value -> ok+ catch+ error:badarg -> ok+ end.++-spec unstick_warm_gate() -> ok.+unstick_warm_gate() ->+ case read_counter(?CNT_WARM_INFLIGHT) of+ Value when is_integer(Value), Value >= ?MAX_WARM_INFLIGHT -> reset_warm_gate();+ _Value -> ok+ end.++-spec reset_warm_gate() -> ok.+reset_warm_gate() ->+ try ets:insert(?PUSH_COUNTER_TABLE, {?CNT_WARM_INFLIGHT, 0}) of+ _Value -> ok+ catch+ error:badarg -> ok+ end.+-spec dispatch_if_eligible([integer()],map(),@@ -408,15 +687,30 @@ dispatch_if_eligible(ok ->ok;dropped ->- logger:debug("Push: dispatcher saturated, dropping notification job", #{- message_id => MessageId,- channel_id => ChannelId,- guild_id => GuildId,- eligible_count => length(EligibleUsers)- }),+ EligibleCount = length(EligibleUsers),+ count_dispatch_dropped(EligibleCount),+ log_dispatch_drop(+ loss_logging_enabled(), MessageId, ChannelId, GuildId, EligibleCount+ ),okend.+-spec log_dispatch_drop(boolean(), integer(), integer(), integer(), non_neg_integer()) -> ok.+log_dispatch_drop(true, MessageId, ChannelId, GuildId, EligibleCount) ->+ logger:error("Push: dispatcher saturated, dropping notification job", #{+ message_id => MessageId,+ channel_id => ChannelId,+ guild_id => GuildId,+ eligible_count => EligibleCount+ });+log_dispatch_drop(false, MessageId, ChannelId, GuildId, EligibleCount) ->+ logger:debug("Push: dispatcher saturated, dropping notification job", #{+ message_id => MessageId,+ channel_id => ChannelId,+ guild_id => GuildId,+ eligible_count => EligibleCount+ }).+-spec handle_sync_user_blocked_ids(integer(), term(), state()) -> {noreply, state()}.handle_sync_user_blocked_ids(UserId, BlockedIds, State) ->case push_normalize:integer_list(BlockedIds) of@@ -449,22 +743,138 @@ handle_clear_channel_notifications(UserId, ChannelId, MessageId, State) ->ok ->ok;dropped ->- logger:debug("Push: dispatcher saturated, dropping clear notification job", #{- user_id => UserId, channel_id => ChannelId, message_id => MessageId- })+ count_clear_dropped(),+ log_clear_drop(loss_logging_enabled(), UserId, ChannelId, MessageId)end,{noreply, State}.+-spec log_clear_drop(boolean(), integer(), integer(), integer()) -> ok.+log_clear_drop(true, UserId, ChannelId, MessageId) ->+ logger:warning("Push: dispatcher saturated, dropping clear notification job", #{+ user_id => UserId, channel_id => ChannelId, message_id => MessageId+ });+log_clear_drop(false, UserId, ChannelId, MessageId) ->+ logger:debug("Push: dispatcher saturated, dropping clear notification job", #{+ user_id => UserId, channel_id => ChannelId, message_id => MessageId+ }).+-spec log_message_worker_drop(ok | dropped, map()) -> ok.log_message_worker_drop(ok, _Params) ->ok;log_message_worker_drop(dropped, Params) ->+ bump_counter(?CNT_WORKER_POOL),MessageData = maps:get(message_data, Params, #{}),+ log_worker_pool_drop(+ loss_logging_enabled(),+ maps:get(<<"id">>, MessageData, undefined),+ maps:get(<<"channel_id">>, MessageData, undefined)+ ).++-spec log_worker_pool_drop(boolean(), term(), term()) -> ok.+log_worker_pool_drop(true, MessageId, ChannelId) ->+ logger:error("Push: worker pool saturated, dropping message create", #{+ message_id => MessageId, channel_id => ChannelId+ });+log_worker_pool_drop(false, MessageId, ChannelId) ->logger:debug("Push: worker pool saturated, dropping message create", #{- message_id => maps:get(<<"id">>, MessageData, undefined),- channel_id => maps:get(<<"channel_id">>, MessageData, undefined)+ message_id => MessageId, channel_id => ChannelId}).+-spec cache_stats_with_counters() -> map().+cache_stats_with_counters() ->+ Base = maps:merge(push_ets_cache:cache_stats(), blocked_ids_counters()),+ maps:merge(Base, push_loss_counters()).++-spec push_loss_counters() -> map().+push_loss_counters() ->+ #{+ counters => counter_table_status(),+ worker_pool_dropped => read_counter(?CNT_WORKER_POOL),+ dispatch_dropped => read_counter(?CNT_DISPATCH_DROPPED),+ dispatch_dropped_users => read_counter(?CNT_DISPATCH_DROPPED_USERS),+ clear_dispatch_dropped => read_counter(?CNT_CLEAR_DROPPED),+ dispatcher_queue_full => read_counter(?CNT_QUEUE_FULL),+ dispatcher_invalid_job => read_counter(?CNT_INVALID_JOB),+ dispatcher_enqueue_timeout => read_counter(?CNT_ENQUEUE_TIMEOUT),+ dispatcher_enqueue_failed => read_counter(?CNT_ENQUEUE_FAILED),+ dispatcher_job_crashed => read_counter(?CNT_JOB_CRASHED),+ dispatcher_worker_died => read_counter(?CNT_WORKER_DIED),+ dispatcher_restarts => read_counter(?CNT_DISPATCHER_RESTARTS),+ dispatcher_restart_discarded => read_counter(?CNT_RESTART_DISCARDED),+ dispatcher_queue_backlog => dispatcher_queue_backlog()+ }.++-spec count_dispatch_dropped(non_neg_integer()) -> ok.+count_dispatch_dropped(EligibleCount) ->+ bump_counter(?CNT_DISPATCH_DROPPED),+ bump_counter(?CNT_DISPATCH_DROPPED_USERS, EligibleCount).++-spec count_clear_dropped() -> ok.+count_clear_dropped() ->+ bump_counter(?CNT_CLEAR_DROPPED).++-spec counter_table_status() -> live | unavailable.+counter_table_status() ->+ case ets:info(?PUSH_COUNTER_TABLE, size) of+ Size when is_integer(Size) -> live;+ _ -> unavailable+ end.++-spec dispatcher_queue_backlog() -> non_neg_integer() | unavailable.+dispatcher_queue_backlog() ->+ backlog(read_counter(?CNT_QUEUE_ENQUEUED), read_counter(?CNT_QUEUE_DEQUEUED)).++-spec backlog(non_neg_integer() | unavailable, non_neg_integer() | unavailable) ->+ non_neg_integer() | unavailable.+backlog(Enqueued, Dequeued) when is_integer(Enqueued), is_integer(Dequeued) ->+ max(0, Enqueued - Dequeued);+backlog(Enqueued, unavailable) when is_integer(Enqueued) ->+ Enqueued;+backlog(_Enqueued, _Dequeued) ->+ unavailable.++-spec loss_logging_enabled() -> boolean().+loss_logging_enabled() ->+ application:get_env(fluxer_gateway, push_loss_logging, false) =:= true.++-spec read_counter(atom()) -> non_neg_integer() | unavailable.+read_counter(Key) ->+ try ets:lookup(?PUSH_COUNTER_TABLE, Key) of+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;+ _ -> unavailable+ catch+ error:badarg -> unavailable+ end.++-spec bump_counter(atom()) -> ok.+bump_counter(Key) ->+ bump_counter(Key, 1).++-spec bump_counter(atom(), non_neg_integer()) -> ok.+bump_counter(Key, Increment) ->+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of+ _Value -> ok+ catch+ error:badarg -> insert_missing_counter(Key, Increment)+ end.++-spec insert_missing_counter(atom(), non_neg_integer()) -> ok.+insert_missing_counter(Key, Increment) ->+ try ets:insert_new(?PUSH_COUNTER_TABLE, {Key, Increment}) of+ true -> ok;+ false -> retry_bump_counter(Key, Increment)+ catch+ error:badarg -> ok+ end.++-spec retry_bump_counter(atom(), non_neg_integer()) -> ok.+retry_bump_counter(Key, Increment) ->+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of+ _Value -> ok+ catch+ error:badarg -> ok+ end.+-spec local_cache_mutation(fun(() -> ok)) -> ok.local_cache_mutation(Fun) ->case whereis(?MODULE) of@@ -521,6 +931,13 @@ env_boolean(Key) ->_ -> falseend.+-spec env_boolean(atom(), boolean()) -> boolean().+env_boolean(Key, Default) ->+ case fluxer_gateway_env:get(Key) of+ Value when is_boolean(Value) -> Value;+ _ -> Default+ end.+-spec env_non_neg_integer(atom(), non_neg_integer()) -> non_neg_integer().env_non_neg_integer(Key, Default) ->case fluxer_gateway_env:get(Key) of@@ -580,12 +997,13 @@ filter_eligible_users_fetches_large_metadata_once_test() ->fun(UserId) -> push_ets_cache:put_user_guild_settings(UserId, 42, #{}) end,[1, 2, 3]),+ lists:foreach(fun(UserId) -> push_ets_cache:put_blocked_ids(UserId, []) end, [1, 2, 3]),Self = self(),ok = meck:new(push_eligibility_checks, [passthrough, no_link]),tryok = meck:expect(push_eligibility_checks, get_guild_large_metadata, fun(42) ->Self ! metadata_lookup,- #{member_count => 300, features => []}+ #{member_count => 3000, features => []}end),MessageData = #{<<"channel_type">> => 0,@@ -593,7 +1011,9 @@ filter_eligible_users_fetches_large_metadata_once_test() ->},?assertEqual([1],- filter_eligible_users([1, 2, 3], 999, 42, 10, MessageData, 0, #{}, #{})+ filter_eligible_users(+ [1, 2, 3], 999, 42, 10, MessageData, 0, #{}, #{}, undefined+ )),?assertEqual(1, drain_metadata_lookup_count(0))after@@ -604,6 +1024,212 @@ filter_eligible_users_fetches_large_metadata_once_test() ->)end.+blocked_ids_fetch_defaults_are_bounded_test() ->+ ?assertEqual(?DEFAULT_FETCH_USERS, blocked_ids_fetch_max_users()),+ ?assertEqual(?DEFAULT_FETCH_CHUNK, blocked_ids_fetch_chunk()).++blocked_ids_from_response_skips_unparsable_entries_test() ->+ ?assertEqual([123, 456], blocked_ids_from_response([<<"123">>, <<"abc">>, 456, null])),+ ?assertEqual([], blocked_ids_from_response(<<"not_a_list">>)).++chunk_user_ids_splits_into_bounded_batches_test() ->+ ?assertEqual([], chunk_user_ids([], 2, [])),+ ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])),+ ?assertEqual([[1, 2, 3]], chunk_user_ids([1, 2, 3], 500, [])).++blocked_ids_fetch_budget_is_capped_test() ->+ assert_budget_capped(),+ with_fetch_env(1, 1000000, fun assert_budget_capped/0),+ with_fetch_env(1000000, 1000000, fun assert_budget_capped/0),+ assert_budget_capped().++assert_budget_capped() ->+ {Budget, ChunkSize} = blocked_ids_fetch_budget(),+ ?assert(ChunkSize >= ?MIN_FETCH_CHUNK),+ ?assert(ChunkSize =< ?MAX_FETCH_CHUNK),+ ?assert(Budget =< ?MAX_FETCH_USERS),+ ?assert(length(chunk_user_ids(lists:seq(1, Budget), ChunkSize, [])) =< ?MAX_FETCH_RPCS).++synced_blocked_recipients_are_dropped_test() ->+ push_ets_cache:init(),+ ok = push_ets_cache:put_blocked_ids(5020, [999]),+ ?assertEqual([], filter_dm_recipients([5020], 999)).++fetched_blocked_recipients_are_dropped_test() ->+ push_ets_cache:init(),+ ok = push_ets_cache:put_blocked_ids_fetched(5031, [999]),+ ?assertEqual([], filter_dm_recipients([5031], 999)).++block_suppression_drops_blocked_recipient_test() ->+ push_ets_cache:init(),+ ok = push_ets_cache:put_blocked_ids(5002, [999]),+ ok = push_ets_cache:put_blocked_ids(5003, []),+ ?assertEqual([5003], filter_dm_recipients([5002, 5003], 999)).++blocked_ids_fetch_stops_after_a_failing_chunk_test() ->+ ?assertEqual(ok, fetch_blocked_ids_chunks([])),+ ?assertEqual(ok, fetch_blocked_ids_chunks([[], []])).++cached_recipients_need_no_blocked_ids_warm_test() ->+ push_ets_cache:init(),+ ok = push_ets_cache:put_blocked_ids_fetched(5004, []),+ ?assertEqual([], missing_blocked_ids([5004])),+ ?assertEqual(ok, warm_blocked_ids([5004])).++blocked_ids_counters_are_unavailable_without_the_shared_table_test() ->+ delete_counter_table(),+ ?assertEqual(unavailable, read_counter(?CNT_SUPPRESSED)),+ ?assertEqual(full, claim_warm_slot()),+ ?assertEqual(ok, release_warm_slot()),+ ?assertEqual(ok, unstick_warm_gate()).++blocked_ids_warm_gate_bounds_concurrency_test() ->+ with_counter_table(fun() ->+ Claims = [claim_warm_slot() || _ <- lists:seq(1, ?MAX_WARM_INFLIGHT + 2)],+ ?assertEqual(?MAX_WARM_INFLIGHT, length([ok || ok <- Claims])),+ ?assertEqual(?MAX_WARM_INFLIGHT, read_counter(?CNT_WARM_INFLIGHT)),+ lists:foreach(+ fun(_) -> release_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT + 2)+ ),+ ?assertEqual(0, read_counter(?CNT_WARM_INFLIGHT))+ end).++blocked_ids_warm_gate_unsticks_in_the_eviction_sweep_test() ->+ with_counter_table(fun() ->+ lists:foreach(fun(_) -> claim_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT)),+ ?assertEqual(full, claim_warm_slot()),+ unstick_warm_gate(),+ ?assertEqual(0, read_counter(?CNT_WARM_INFLIGHT)),+ ?assertEqual(ok, claim_warm_slot())+ end).++blocked_ids_warm_is_dropped_when_the_gate_is_full_test() ->+ push_ets_cache:init(),+ with_counter_table(fun() ->+ lists:foreach(fun(_) -> claim_warm_slot() end, lists:seq(1, ?MAX_WARM_INFLIGHT)),+ ?assertEqual(ok, warm_blocked_ids([5099])),+ ?assertEqual(1, read_counter(?CNT_WARM_DROPPED))+ end).++blocked_ids_counters_are_exposed_in_cache_stats_test() ->+ push_ets_cache:init(),+ with_counter_table(fun() ->+ ok = push_ets_cache:put_blocked_ids(5010, [999]),+ ?assertEqual([], filter_dm_recipients([5010], 999)),+ ?assertEqual(1, read_counter(?CNT_SUPPRESSED)),+ assert_cache_stats_expose_blocked_ids()+ end).++assert_cache_stats_expose_blocked_ids() ->+ Stats = maps:merge(push_ets_cache:cache_stats(), blocked_ids_counters()),+ lists:foreach(+ fun(Key) -> ?assertEqual(true, maps:is_key(Key, Stats)) end,+ [+ blocked_ids_size,+ blocked_ids_fetch_attempts,+ blocked_ids_fetch_failures,+ blocked_ids_suppressed,+ blocked_ids_budget_exhausted,+ blocked_ids_warm_dropped,+ blocked_ids_warm_inflight+ ]+ ).++push_loss_counters_expose_every_counter_push_writes_test() ->+ with_counter_table(fun() ->+ ok = log_message_worker_drop(dropped, #{}),+ ok = count_dispatch_dropped(12),+ ok = count_clear_dropped(),+ Stats = push_loss_counters(),+ ?assertEqual(live, maps:get(counters, Stats)),+ ?assertEqual(1, maps:get(worker_pool_dropped, Stats)),+ ?assertEqual(1, maps:get(dispatch_dropped, Stats)),+ ?assertEqual(12, maps:get(dispatch_dropped_users, Stats)),+ ?assertEqual(1, maps:get(clear_dispatch_dropped, Stats))+ end).++push_loss_counters_are_unavailable_without_the_shared_table_test() ->+ delete_counter_table(),+ Stats = push_loss_counters(),+ ?assertEqual(unavailable, maps:get(counters, Stats)),+ ?assertEqual(unavailable, maps:get(worker_pool_dropped, Stats)),+ ?assertEqual(unavailable, maps:get(dispatch_dropped, Stats)),+ ?assertEqual(unavailable, maps:get(dispatcher_enqueue_timeout, Stats)),+ ?assertEqual(unavailable, maps:get(dispatcher_queue_backlog, Stats)).++push_loss_counters_keep_a_genuine_zero_distinct_from_absent_test() ->+ with_counter_table(fun() ->+ ok = count_dispatch_dropped(0),+ Stats = push_loss_counters(),+ ?assertEqual(1, maps:get(dispatch_dropped, Stats)),+ ?assertEqual(0, maps:get(dispatch_dropped_users, Stats)),+ ?assertEqual(unavailable, maps:get(clear_dispatch_dropped, Stats))+ end).++dispatcher_queue_backlog_survives_an_untrappable_dispatcher_kill_test() ->+ with_counter_table(fun() ->+ ?assertEqual(unavailable, dispatcher_queue_backlog()),+ ok = bump_counter(?CNT_QUEUE_ENQUEUED, 9),+ ?assertEqual(9, dispatcher_queue_backlog()),+ ok = bump_counter(?CNT_QUEUE_DEQUEUED, 4),+ ?assertEqual(5, dispatcher_queue_backlog())+ end).++cache_stats_with_counters_carries_the_loss_surface_test() ->+ push_ets_cache:init(),+ with_counter_table(fun() ->+ Stats = cache_stats_with_counters(),+ lists:foreach(+ fun(Key) -> ?assertEqual(true, maps:is_key(Key, Stats)) end,+ [+ blocked_ids_size,+ blocked_ids_suppressed,+ counters,+ worker_pool_dropped,+ dispatch_dropped,+ dispatch_dropped_users,+ clear_dispatch_dropped,+ dispatcher_queue_full,+ dispatcher_enqueue_timeout,+ dispatcher_job_crashed,+ dispatcher_worker_died,+ dispatcher_restarts,+ dispatcher_restart_discarded,+ dispatcher_queue_backlog+ ]+ )+ end).++with_counter_table(Fun) ->+ delete_counter_table(),+ _ = ets:new(?PUSH_COUNTER_TABLE, [named_table, public, set, {write_concurrency, true}]),+ try+ Fun()+ after+ delete_counter_table()+ end.++delete_counter_table() ->+ try ets:delete(?PUSH_COUNTER_TABLE) of+ _ -> ok+ catch+ error:badarg -> ok+ end.++filter_dm_recipients(UserIds, AuthorId) ->+ MessageData = #{<<"channel_type">> => 1},+ filter_eligible_users(UserIds, AuthorId, 0, 10, MessageData, 0, #{}, #{}, undefined).++with_fetch_env(ChunkSize, MaxUsers, Fun) ->+ application:set_env(fluxer_gateway, push_blocked_ids_fetch_chunk, ChunkSize),+ application:set_env(fluxer_gateway, push_blocked_ids_fetch_max_users, MaxUsers),+ try+ Fun()+ after+ application:unset_env(fluxer_gateway, push_blocked_ids_fetch_chunk),+ application:unset_env(fluxer_gateway, push_blocked_ids_fetch_max_users)+ end.+filter_eligible_users_batches_missing_settings_lookups_test() ->push_ets_cache:init(),Self = self(),@@ -615,7 +1241,7 @@ filter_eligible_users_batches_missing_settings_lookups_test() ->end),?assertEqual([1, 2, 3],- filter_eligible_users([1, 2, 3], 999, 43, 10, #{}, 0, #{}, #{})+ filter_eligible_users([1, 2, 3], 999, 43, 10, #{}, 0, #{}, #{}, undefined)),?assertEqual(1, drain_settings_request_count(0))afterdiff --git a/fluxer_gateway/src/push/push_dispatcher.erl b/fluxer_gateway/src/push/push_dispatcher.erlindex 73ca0158e..93e9755c2 100644--- a/fluxer_gateway/src/push/push_dispatcher.erl+++ b/fluxer_gateway/src/push/push_dispatcher.erl@@ -17,6 +17,19 @@-define(DEFAULT_MAX_QUEUE, 10000).-define(ENQUEUE_TIMEOUT_MS, 1000).+-define(PUSH_COUNTER_TABLE, push_worker_counter).+-define(CNT_QUEUE_FULL, push_loss_queue_full).+-define(CNT_INVALID_JOB, push_loss_invalid_job).+-define(CNT_ENQUEUE_TIMEOUT, push_loss_enqueue_timeout).+-define(CNT_ENQUEUE_FAILED, push_loss_enqueue_failed).+-define(CNT_JOB_CRASHED, push_loss_job_crashed).+-define(CNT_WORKER_DIED, push_loss_worker_died).+-define(CNT_WORKER_POOL, push_loss_worker_pool).+-define(CNT_RESTARTS, push_dispatcher_restarts).+-define(CNT_RESTART_DISCARDED, push_dispatcher_restart_discarded).+-define(CNT_QUEUE_ENQUEUED, push_dispatcher_queue_enqueued).+-define(CNT_QUEUE_DEQUEUED, push_dispatcher_queue_dequeued).+-type push_job() ::#{type := message_create,@@ -44,7 +57,27 @@inflight := non_neg_integer(),workers := #{reference() => true},max_inflight := pos_integer(),- max_queue := pos_integer()+ max_queue := pos_integer(),+ started_at => integer()+}.++-type counter_value() :: non_neg_integer() | unavailable.++-type stats() :: #{+ queued := non_neg_integer(),+ inflight := non_neg_integer(),+ counters := live | unavailable,+ dispatcher_uptime_seconds := non_neg_integer() | undefined,+ dispatcher_restarts := counter_value(),+ restart_discarded_jobs := counter_value(),+ queue_backlog_lost := counter_value(),+ queue_full_dropped := counter_value(),+ invalid_job_dropped := counter_value(),+ enqueue_timeout := counter_value(),+ enqueue_failed := counter_value(),+ job_crashed := counter_value(),+ worker_died := counter_value(),+ worker_pool_dropped := counter_value()}.-spec start_link() -> {ok, pid()} | {error, term()} | ignore.@@ -135,25 +168,57 @@ enqueue_clear_notifications(UserId, ChannelId, MessageId, BadgeCountsTtlSeconds)),safe_enqueue(Job).--spec stats() -> #{queued := non_neg_integer(), inflight := non_neg_integer()} | #{}.+-spec stats() -> stats() | #{}.stats() ->- try- gen_server:call(?MODULE, stats, 1000)+ Enqueued = read_counter(?CNT_QUEUE_ENQUEUED),+ try gen_server:call(?MODULE, stats, 1000) of+ #{queued := Queued, inflight := Inflight} = Reply ->+ StartedAt = maps:get(started_at, Reply, undefined),+ stats_with_loss(Queued, Inflight, StartedAt, Enqueued);+ _ ->+ #{}catchexit:_ -> #{};error:_ -> #{}end.+-spec stats_with_loss(non_neg_integer(), non_neg_integer(), term(), term()) -> stats().+stats_with_loss(Queued, Inflight, StartedAt, Enqueued) ->+ #{+ queued => Queued,+ inflight => Inflight,+ counters => counter_table_status(),+ dispatcher_uptime_seconds => uptime_seconds(StartedAt),+ dispatcher_restarts => read_counter(?CNT_RESTARTS),+ restart_discarded_jobs => read_counter(?CNT_RESTART_DISCARDED),+ queue_backlog_lost => queue_backlog_lost(Enqueued, Queued),+ queue_full_dropped => read_counter(?CNT_QUEUE_FULL),+ invalid_job_dropped => read_counter(?CNT_INVALID_JOB),+ enqueue_timeout => read_counter(?CNT_ENQUEUE_TIMEOUT),+ enqueue_failed => read_counter(?CNT_ENQUEUE_FAILED),+ job_crashed => read_counter(?CNT_JOB_CRASHED),+ worker_died => read_counter(?CNT_WORKER_DIED),+ worker_pool_dropped => read_counter(?CNT_WORKER_POOL)+ }.++-spec uptime_seconds(term()) -> non_neg_integer() | undefined.+uptime_seconds(StartedAt) when is_integer(StartedAt) ->+ max(0, erlang:monotonic_time(second) - StartedAt);+uptime_seconds(_StartedAt) ->+ undefined.+-spec init([]) -> {ok, state()}.init([]) ->erlang:process_flag(fullsweep_after, 10),+ bump_counter(?CNT_RESTARTS),{ok, #{queue => queue:new(),queued => 0,inflight => 0,workers => #{},max_inflight => budget_aware_max_inflight(),- max_queue => get_int_or_default(push_dispatcher_max_queue, ?DEFAULT_MAX_QUEUE)+ max_queue => get_int_or_default(push_dispatcher_max_queue, ?DEFAULT_MAX_QUEUE),+ started_at => erlang:monotonic_time(second)}}.-spec budget_aware_max_inflight() -> pos_integer().@@ -167,7 +232,12 @@ budget_aware_max_inflight() ->-spec handle_call(term(), gen_server:from(), state()) ->{reply, term(), state()}.handle_call(stats, _From, #{queued := Queued, inflight := Inflight} = State) ->- {reply, #{queued => Queued, inflight => Inflight}, State};+ Reply = #{+ queued => Queued,+ inflight => Inflight,+ started_at => maps:get(started_at, State, undefined)+ },+ {reply, Reply, State};handle_call({enqueue, Job}, _From, State) ->{Result, State1} = handle_enqueue(Job, State),{reply, Result, State1};@@ -183,11 +253,12 @@ handle_cast(_Msg, State) ->-spec handle_info(term(), state()) -> {noreply, state()}.handle_info(- {'DOWN', Ref, process, _Pid, _Reason},+ {'DOWN', Ref, process, _Pid, Reason},#{workers := Workers, inflight := Inflight} = State) ->case maps:is_key(Ref, Workers) oftrue ->+ count_worker_down(Reason),RemainingWorkers = maps:remove(Ref, Workers),DecrementedInflight = max(0, Inflight - 1),drain_queue(State#{@@ -201,8 +272,15 @@ handle_info(_Info, State) ->{noreply, State}.-spec terminate(term(), state()) -> ok.-terminate(_Reason, _State) ->- ok.+terminate(_Reason, State) ->+ bump_counter(?CNT_RESTART_DISCARDED, maps:get(queued, State, 0)).++-spec count_worker_down(term()) -> ok.+count_worker_down(normal) ->+ ok;+count_worker_down(_Reason) ->+ bump_counter(?CNT_JOB_CRASHED),+ bump_counter(?CNT_WORKER_DIED).-spec code_change(term(), state(), term()) -> {ok, state()}.code_change(_OldVsn, State, _Extra) ->@@ -239,22 +317,40 @@ maybe_enqueue_or_start(Job, #{inflight := Inflight, max_inflight := MaxInflight}maybe_enqueue(Job, #{queued := Queued, max_queue := MaxQueue, queue := Queue0} = State) ->case Queued < MaxQueue oftrue ->+ bump_counter(?CNT_QUEUE_ENQUEUED),Queue1 = queue:in(Job, Queue0),{ok, State#{queue := Queue1, queued := Queued + 1}};false ->+ bump_counter(?CNT_QUEUE_FULL),DropCount = bump_drop_count(),- logger:warning(- "Push: queue full, dropping job",- #{- message_id => maps:get(message_id, Job, undefined),- queued => Queued,- max_queue => MaxQueue,- total_dropped => DropCount- }- ),+ log_queue_full_drop(loss_logging_enabled(), Job, Queued, MaxQueue, DropCount),{dropped, State}end.+-spec log_queue_full_drop(+ boolean(), push_job(), non_neg_integer(), pos_integer(), non_neg_integer()+) -> ok.+log_queue_full_drop(true, Job, Queued, MaxQueue, _DropCount) ->+ logger:error(+ "Push: queue full, dropping job",+ #{+ message_id => maps:get(message_id, Job, undefined),+ queued => Queued,+ max_queue => MaxQueue,+ total_dropped => read_counter(?CNT_QUEUE_FULL)+ }+ );+log_queue_full_drop(false, Job, Queued, MaxQueue, DropCount) ->+ logger:warning(+ "Push: queue full, dropping job",+ #{+ message_id => maps:get(message_id, Job, undefined),+ queued => Queued,+ max_queue => MaxQueue,+ total_dropped => DropCount+ }+ ).+-spec bump_drop_count() -> non_neg_integer().bump_drop_count() ->Current =@@ -266,6 +362,68 @@ bump_drop_count() ->erlang:put(push_dispatcher_drop_count, Updated),Updated.+-spec loss_logging_enabled() -> boolean().+loss_logging_enabled() ->+ application:get_env(fluxer_gateway, push_loss_logging, false) =:= true.++-spec counter_table_status() -> live | unavailable.+counter_table_status() ->+ case ets:info(?PUSH_COUNTER_TABLE, size) of+ Size when is_integer(Size) -> live;+ _ -> unavailable+ end.++-spec queue_backlog_lost(term(), non_neg_integer()) -> counter_value().+queue_backlog_lost(Enqueued, Queued) when is_integer(Enqueued) ->+ max(0, Enqueued - dequeued_total() - Queued);+queue_backlog_lost(_Enqueued, _Queued) ->+ unavailable.++-spec dequeued_total() -> non_neg_integer().+dequeued_total() ->+ case read_counter(?CNT_QUEUE_DEQUEUED) of+ Dequeued when is_integer(Dequeued) -> Dequeued;+ unavailable -> 0+ end.++-spec read_counter(atom()) -> counter_value().+read_counter(Key) ->+ try ets:lookup(?PUSH_COUNTER_TABLE, Key) of+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;+ _ -> unavailable+ catch+ error:badarg -> unavailable+ end.++-spec bump_counter(atom()) -> ok.+bump_counter(Key) ->+ bump_counter(Key, 1).++-spec bump_counter(atom(), non_neg_integer()) -> ok.+bump_counter(Key, Increment) ->+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of+ _Value -> ok+ catch+ error:badarg -> insert_missing_counter(Key, Increment)+ end.++-spec insert_missing_counter(atom(), non_neg_integer()) -> ok.+insert_missing_counter(Key, Increment) ->+ try ets:insert_new(?PUSH_COUNTER_TABLE, {Key, Increment}) of+ true -> ok;+ false -> retry_bump_counter(Key, Increment)+ catch+ error:badarg -> ok+ end.++-spec retry_bump_counter(atom(), non_neg_integer()) -> ok.+retry_bump_counter(Key, Increment) ->+ try ets:update_counter(?PUSH_COUNTER_TABLE, Key, {2, Increment}) of+ _Value -> ok+ catch+ error:badarg -> ok+ end.+-spec start_job(push_job(), state()) -> state().start_job(Job, #{workers := Workers, inflight := Inflight} = State) ->{_Pid, Ref} =@@ -340,6 +498,7 @@ log_enqueue_send_notifications(UserIds, GuildId, ChannelId, MessageId) ->state()) -> {noreply, state()}.drain_available_queue({{value, Job}, Queue1}, Queued, State) ->+ bump_counter(?CNT_QUEUE_DEQUEUED),State1 = State#{queue := Queue1, queued := max(0, Queued - 1)},State2 = start_job(Job, State1),drain_queue(State2);@@ -353,14 +512,29 @@ run_job(#{message_id := MessageId} = Job) ->logger:debug("Push: worker completed", #{message_id => MessageId}),okcatch- Class:Reason ->- logger:debug(- "Push: worker crashed",- #{message_id => MessageId, class => Class, reason => Reason}- ),+ Class:Reason:Stacktrace ->+ bump_counter(?CNT_JOB_CRASHED),+ log_job_crash(loss_logging_enabled(), MessageId, Class, Reason, Stacktrace),okend.+-spec log_job_crash(boolean(), integer(), atom(), term(), list()) -> ok.+log_job_crash(true, MessageId, Class, Reason, Stacktrace) ->+ logger:error(+ "Push: worker crashed",+ #{+ message_id => MessageId,+ class => Class,+ reason => Reason,+ stacktrace => Stacktrace+ }+ );+log_job_crash(false, MessageId, Class, Reason, _Stacktrace) ->+ logger:debug(+ "Push: worker crashed",+ #{message_id => MessageId, class => Class, reason => Reason}+ ).+-spec run_typed_job(message_create | clear_channel, push_job()) -> ok.run_typed_job(message_create, #{user_ids := UserIds,@@ -412,20 +586,39 @@ get_int_or_default(Key, Default) ->-spec safe_enqueue(push_job()) -> ok | dropped.safe_enqueue(Job) ->try gen_server:call(?MODULE, {enqueue, Job}, ?ENQUEUE_TIMEOUT_MS) of- ok -> ok;- dropped -> dropped;- _ -> dropped+ ok ->+ ok;+ dropped ->+ dropped;+ _ ->+ bump_counter(?CNT_ENQUEUE_FAILED),+ droppedcatch- throw:_Reason -> dropped;- error:_Reason -> dropped;- exit:_Reason -> dropped+ throw:_Reason ->+ bump_counter(?CNT_ENQUEUE_FAILED),+ dropped;+ error:_Reason ->+ bump_counter(?CNT_ENQUEUE_FAILED),+ dropped;+ exit:Reason ->+ count_enqueue_exit(Reason),+ droppedend.+-spec count_enqueue_exit(term()) -> ok.+count_enqueue_exit({timeout, _Call}) ->+ bump_counter(?CNT_ENQUEUE_TIMEOUT);+count_enqueue_exit(_Reason) ->+ bump_counter(?CNT_ENQUEUE_FAILED).+-spec handle_enqueue(term(), state()) -> {ok | dropped, state()}.handle_enqueue(Job0, State) ->case push_job(Job0) of- {ok, Job} -> maybe_enqueue_or_start(Job, State);- error -> {dropped, State}+ {ok, Job} ->+ maybe_enqueue_or_start(Job, State);+ error ->+ bump_counter(?CNT_INVALID_JOB),+ {dropped, State}end.-spec push_job(term()) -> {ok, push_job()} | error.@@ -547,3 +740,135 @@ optional_binary(Value) when is_binary(Value) ->{ok, Value};optional_binary(_) ->error.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++absent_counter_table_reads_unavailable_test() ->+ delete_counter_table(),+ ?assertEqual(unavailable, counter_table_status()),+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)),+ ?assertEqual(ok, bump_counter(?CNT_QUEUE_FULL)),+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)).++absent_counter_key_is_distinct_from_zero_test() ->+ with_counter_table(fun() ->+ ?assertEqual(live, counter_table_status()),+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_FULL)),+ ok = bump_counter(?CNT_RESTART_DISCARDED, 0),+ ?assertEqual(0, read_counter(?CNT_RESTART_DISCARDED))+ end).++bump_counter_creates_then_increments_key_test() ->+ with_counter_table(fun() ->+ ok = bump_counter(?CNT_QUEUE_FULL),+ ?assertEqual(1, read_counter(?CNT_QUEUE_FULL)),+ ok = bump_counter(?CNT_QUEUE_FULL, 4),+ ?assertEqual(5, read_counter(?CNT_QUEUE_FULL))+ end).++enqueue_exit_separates_timeout_from_failure_test() ->+ with_counter_table(fun() ->+ ok = count_enqueue_exit({timeout, {gen_server, call, []}}),+ ok = count_enqueue_exit({noproc, {gen_server, call, []}}),+ ?assertEqual(1, read_counter(?CNT_ENQUEUE_TIMEOUT)),+ ?assertEqual(1, read_counter(?CNT_ENQUEUE_FAILED))+ end).++worker_down_counts_only_abnormal_exits_test() ->+ with_counter_table(fun() ->+ ok = count_worker_down(normal),+ ?assertEqual(unavailable, read_counter(?CNT_WORKER_DIED)),+ ok = count_worker_down(killed),+ ok = count_worker_down({shutdown, restarting}),+ ?assertEqual(2, read_counter(?CNT_WORKER_DIED))+ end).++terminate_counts_discarded_queue_depth_test() ->+ with_counter_table(fun() ->+ ?assertEqual(ok, terminate(shutdown, dispatcher_state(7, 10))),+ ?assertEqual(7, read_counter(?CNT_RESTART_DISCARDED))+ end).++invalid_job_enqueue_is_counted_test() ->+ with_counter_table(fun() ->+ State = dispatcher_state(0, 10),+ ?assertEqual({dropped, State}, handle_enqueue(#{type => invalid}, State)),+ ?assertEqual(1, read_counter(?CNT_INVALID_JOB))+ end).++uptime_seconds_reports_undefined_without_start_time_test() ->+ ?assertEqual(undefined, uptime_seconds(undefined)),+ ?assertEqual(0, uptime_seconds(erlang:monotonic_time(second) + 5)),+ ?assert(is_integer(uptime_seconds(erlang:monotonic_time(second) - 3))).++abnormal_worker_down_also_counts_a_crashed_job_test() ->+ with_counter_table(fun() ->+ ok = count_worker_down(normal),+ ?assertEqual(unavailable, read_counter(?CNT_JOB_CRASHED)),+ ok = count_worker_down(killed),+ ?assertEqual(1, read_counter(?CNT_JOB_CRASHED)),+ ?assertEqual(1, read_counter(?CNT_WORKER_DIED))+ end).++enqueue_records_queue_growth_outside_the_process_test() ->+ with_counter_table(fun() ->+ {ok, State1} = maybe_enqueue(clear_job(), dispatcher_state(0, 10)),+ ?assertEqual(1, maps:get(queued, State1)),+ ?assertEqual(1, read_counter(?CNT_QUEUE_ENQUEUED)),+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_DEQUEUED))+ end).++queue_full_drop_does_not_record_queue_growth_test() ->+ with_counter_table(fun() ->+ {dropped, _State1} = maybe_enqueue(clear_job(), dispatcher_state(3, 3)),+ ?assertEqual(1, read_counter(?CNT_QUEUE_FULL)),+ ?assertEqual(unavailable, read_counter(?CNT_QUEUE_ENQUEUED))+ end).++queue_backlog_lost_survives_an_untrappable_kill_test() ->+ with_counter_table(fun() ->+ ?assertEqual(unavailable, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 0)),+ ok = bump_counter(?CNT_QUEUE_ENQUEUED, 7),+ ok = bump_counter(?CNT_QUEUE_DEQUEUED, 2),+ ?assertEqual(0, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 5)),+ ?assertEqual(5, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 0)),+ ?assertEqual(0, queue_backlog_lost(read_counter(?CNT_QUEUE_ENQUEUED), 900))+ end).++dispatcher_state(Queued, MaxQueue) ->+ #{+ queue => queue:new(),+ queued => Queued,+ inflight => 0,+ workers => #{},+ max_inflight => 1,+ max_queue => MaxQueue+ }.++clear_job() ->+ #{+ type => clear_channel,+ user_id => 1,+ channel_id => 2,+ message_id => 3,+ badge_counts_ttl_seconds => 0+ }.++with_counter_table(Fun) ->+ delete_counter_table(),+ _ = ets:new(?PUSH_COUNTER_TABLE, [named_table, public, set, {write_concurrency, true}]),+ try+ Fun()+ after+ delete_counter_table()+ end.++delete_counter_table() ->+ try ets:delete(?PUSH_COUNTER_TABLE) of+ _ -> ok+ catch+ error:badarg -> ok+ end.++-endif.diff --git a/fluxer_gateway/src/push/push_eligibility_checks.erl b/fluxer_gateway/src/push/push_eligibility_checks.erlindex 2d4ec072f..c13d6c20b 100644--- a/fluxer_gateway/src/push/push_eligibility_checks.erl+++ b/fluxer_gateway/src/push/push_eligibility_checks.erl@@ -16,11 +16,12 @@-export([override_for_large_guild/2]).-export([enforce_only_mentions/1]).-export([is_large_guild/2]).+-export([large_guild_threshold/0]).-export([has_large_guild_override/1]).-export([get_guild_large_metadata/1]).-export([check_temp_muted/1]).--define(LARGE_GUILD_THRESHOLD, 250).+-define(LARGE_GUILD_THRESHOLD, 2500).-define(LARGE_GUILD_OVERRIDE_FEATURE, <<"LARGE_GUILD_OVERRIDE">>).-define(MESSAGE_NOTIFICATIONS_NULL, -1).-define(MESSAGE_NOTIFICATIONS_ALL, 0).@@ -215,10 +216,14 @@ enforce_only_mentions(CurrentLevel) -> CurrentLevel.-spec is_large_guild(integer() | term(), list()) -> boolean().is_large_guild(Count, Features) when is_integer(Count) ->- Count > ?LARGE_GUILD_THRESHOLD orelse has_large_guild_override(Features);+ Count > large_guild_threshold() orelse has_large_guild_override(Features);is_large_guild(_, Features) ->has_large_guild_override(Features).+-spec large_guild_threshold() -> pos_integer().+large_guild_threshold() ->+ ?LARGE_GUILD_THRESHOLD.+-spec has_large_guild_override(list() | term()) -> boolean().has_large_guild_override(Features) when is_list(Features) ->lists:member(?LARGE_GUILD_OVERRIDE_FEATURE, Features);@@ -380,11 +385,27 @@ enforce_only_mentions_test() ->?assertEqual(2, enforce_only_mentions(2)).is_large_guild_test() ->- ?assertEqual(true, is_large_guild(300, [])),+ ?assertEqual(true, is_large_guild(3000, [])),+ ?assertEqual(false, is_large_guild(300, [])),+ ?assertEqual(true, is_large_guild(?LARGE_GUILD_THRESHOLD + 1, [])),+ ?assertEqual(false, is_large_guild(?LARGE_GUILD_THRESHOLD, [])),?assertEqual(false, is_large_guild(100, [])),?assertEqual(true, is_large_guild(100, [<<"LARGE_GUILD_OVERRIDE">>])),+ ?assertEqual(true, is_large_guild(300, [<<"LARGE_GUILD_OVERRIDE">>])),?assertEqual(true, is_large_guild(undefined, [<<"LARGE_GUILD_OVERRIDE">>])).+large_guild_threshold_is_the_production_value_test() ->+ ?assertEqual(2500, ?LARGE_GUILD_THRESHOLD),+ ?assertEqual(?LARGE_GUILD_THRESHOLD, large_guild_threshold()).++mid_sized_guilds_keep_all_notifications_test() ->+ MessageData = #{<<"channel_type">> => 0},+ Metadata = #{member_count => 1000, features => []},+ ?assertEqual(+ true,+ check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, Metadata)+ ).+has_large_guild_override_test() ->?assertEqual(true, has_large_guild_override([<<"LARGE_GUILD_OVERRIDE">>])),?assertEqual(false, has_large_guild_override([<<"OTHER">>])),@@ -392,7 +413,7 @@ has_large_guild_override_test() ->cached_large_guild_metadata_overrides_all_notifications_test() ->MessageData = #{<<"channel_type">> => 0},- LargeMetadata = #{member_count => 300, features => []},+ LargeMetadata = #{member_count => 3000, features => []},?assertEqual(false,check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, LargeMetadata)@@ -403,7 +424,7 @@ cached_large_guild_metadata_keeps_mentions_allowed_test() -><<"channel_type">> => 0,<<"mentions">> => [#{<<"id">> => <<"100">>}]},- LargeMetadata = #{member_count => 300, features => []},+ LargeMetadata = #{member_count => 3000, features => []},?assertEqual(true,check_muted_and_notifications(100, 200, MessageData, 0, #{}, #{}, 1, #{}, LargeMetadata)diff --git a/fluxer_gateway/src/push/push_ets_cache.erl b/fluxer_gateway/src/push/push_ets_cache.erlindex e36272826..9a45b4f7c 100644--- a/fluxer_gateway/src/push/push_ets_cache.erl+++ b/fluxer_gateway/src/push/push_ets_cache.erl@@ -14,6 +14,7 @@delete_subscriptions/1,get_blocked_ids/1,put_blocked_ids/2,+ put_blocked_ids_fetched/2,get_badge_count/1,put_badge_count/3,delete_badge_count/1,@@ -31,6 +32,10 @@-define(MAX_TABLE_ENTRIES, 500000).-define(EVICT_BATCH, 4096).+-define(MAX_EVICT_RESEEKS, 8).+-define(DEFAULT_BLOCKED_IDS_TTL, 300).+-define(MIN_BLOCKED_IDS_TTL, 30).+-define(MAX_BLOCKED_IDS_TTL, 3600).-define(ETS_OPTS, [named_table, public, set, {read_concurrency, true}, {write_concurrency, true}@@ -125,17 +130,54 @@ get_blocked_ids(UserId) ->-spec get_blocked_ids_ets(integer()) -> [integer()] | undefined.get_blocked_ids_ets(UserId) ->try ets:lookup(?BLOCKED_IDS, UserId) of- [{UserId, BlockedIds}] -> BlockedIds;- _ -> undefined+ [{UserId, BlockedIds}] ->+ BlockedIds;+ [{UserId, BlockedIds, infinity}] ->+ BlockedIds;+ [{UserId, BlockedIds, ExpiresAt}] when is_integer(ExpiresAt) ->+ live_fetched_blocked_ids(BlockedIds, ExpiresAt);+ _ ->+ undefinedcatcherror:badarg -> undefinedend.+-spec live_fetched_blocked_ids([integer()], integer()) -> [integer()] | undefined.+live_fetched_blocked_ids(BlockedIds, ExpiresAt) ->+ case erlang:system_time(second) < ExpiresAt of+ true -> BlockedIds;+ false -> undefined+ end.+-spec put_blocked_ids(integer(), [integer()]) -> ok.put_blocked_ids(UserId, BlockedIds) ->+ insert_blocked_ids(UserId, BlockedIds, infinity).++-spec put_blocked_ids_fetched(integer(), [integer()]) -> ok.+put_blocked_ids_fetched(UserId, BlockedIds) ->+ ExpiresAt = erlang:system_time(second) + blocked_ids_ttl_seconds(),+ insert_blocked_ids(UserId, BlockedIds, ExpiresAt).++-spec insert_blocked_ids(integer(), [integer()], infinity | integer()) -> ok.+insert_blocked_ids(UserId, BlockedIds, ExpiresAt) ->guard_table_size(?BLOCKED_IDS),- ets:insert(?BLOCKED_IDS, {UserId, BlockedIds}),- ok.+ try ets:insert(?BLOCKED_IDS, {UserId, BlockedIds, ExpiresAt}) of+ _ -> ok+ catch+ error:badarg -> ok+ end.++-spec blocked_ids_ttl_seconds() -> pos_integer().+blocked_ids_ttl_seconds() ->+ Value = app_pos_integer(push_blocked_ids_cache_ttl_seconds, ?DEFAULT_BLOCKED_IDS_TTL),+ min(max(Value, ?MIN_BLOCKED_IDS_TTL), ?MAX_BLOCKED_IDS_TTL).++-spec app_pos_integer(atom(), pos_integer()) -> pos_integer().+app_pos_integer(Key, Default) ->+ case application:get_env(fluxer_gateway, Key, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> Default+ end.-spec get_badge_count(integer()) -> {non_neg_integer(), integer()} | undefined.get_badge_count(UserId) ->@@ -233,12 +275,24 @@ cache_stats() ->-spec evict_tables(map()) -> ok.evict_tables(MaxEntries) ->+ _ = expire_blocked_ids(),evict_table(?USER_GUILD_SETTINGS, maps:get(user_guild_settings, MaxEntries, undefined)),evict_table(?SUBSCRIPTIONS, maps:get(subscriptions, MaxEntries, undefined)),evict_table(?BLOCKED_IDS, maps:get(blocked_ids, MaxEntries, undefined)),evict_table(?BADGE_COUNTS, maps:get(badge_counts, MaxEntries, undefined)),ok.+-spec expire_blocked_ids() -> non_neg_integer().+expire_blocked_ids() ->+ Now = erlang:system_time(second),+ MatchSpec = [{{'_', '_', '$1'}, [{is_integer, '$1'}, {'=<', '$1', Now}], [true]}],+ try ets:select_delete(?BLOCKED_IDS, MatchSpec) of+ Deleted when is_integer(Deleted) -> Deleted;+ _ -> 0+ catch+ error:badarg -> 0+ end.+-spec guard_table_size(atom()) -> ok.guard_table_size(Table) ->case table_size(Table) >= ?MAX_TABLE_ENTRIES of@@ -279,20 +333,52 @@ evict_table(Table, MaxEntries) ->case Size > MaxEntries oftrue ->ToDelete = Size - MaxEntries,- evict_n(Table, ets:first(Table), ToDelete);+ evict_scan(Table, ToDelete, ?MAX_EVICT_RESEEKS);false ->okend.--spec evict_n(atom(), term(), non_neg_integer()) -> ok.-evict_n(_Table, '$end_of_table', _N) ->+-spec evict_scan(atom(), non_neg_integer(), non_neg_integer()) -> ok.+evict_scan(_Table, 0, _Reseeks) ->ok;-evict_n(_Table, _Key, 0) ->+evict_scan(_Table, _N, 0) ->ok;+evict_scan(Table, N, Reseeks) ->+ case evict_n(Table, first_key(Table), N) of+ {done, _Remaining} -> ok;+ {interrupted, Remaining} -> evict_scan(Table, Remaining, Reseeks - 1)+ end.++-spec evict_n(atom(), term(), non_neg_integer()) ->+ {done, non_neg_integer()} | {interrupted, non_neg_integer()}.+evict_n(_Table, '$end_of_table', N) ->+ {done, N};+evict_n(_Table, _Key, 0) ->+ {done, 0};evict_n(Table, Key, N) ->- Next = ets:next(Table, Key),- ets:delete(Table, Key),- evict_n(Table, Next, N - 1).+ case next_key(Table, Key) of+ {ok, Next} ->+ safe_delete(Table, Key),+ evict_n(Table, Next, N - 1);+ {error, badarg} ->+ {interrupted, N}+ end.++-spec first_key(atom()) -> term().+first_key(Table) ->+ try ets:first(Table) of+ Key -> Key+ catch+ error:badarg -> '$end_of_table'+ end.++-spec next_key(atom(), term()) -> {ok, term()} | {error, badarg}.+next_key(Table, Key) ->+ try ets:next(Table, Key) of+ Next -> {ok, Next}+ catch+ error:badarg -> {error, badarg}+ end.-spec rebalance_table(atom(), fun((term()) -> integer() | undefined)) -> non_neg_integer().rebalance_table(Table, UserIdFun) ->@@ -350,3 +436,67 @@ user_id_from_key(UserId) when is_integer(UserId) ->UserId;user_id_from_key(_) ->undefined.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++synced_blocked_ids_are_stored_without_an_expiry_test() ->+ init(),+ try+ ok = put_blocked_ids(4001, [4002]),+ ?assertEqual([{4001, [4002], infinity}], ets:lookup(?BLOCKED_IDS, 4001))+ after+ safe_delete(?BLOCKED_IDS, 4001)+ end.++synced_blocked_ids_never_expire_test() ->+ init(),+ try+ ok = put_blocked_ids(4002, [4003]),+ ?assertEqual([4003], get_blocked_ids(4002)),+ ok = evict_tables(#{}),+ ?assertEqual([4003], get_blocked_ids(4002))+ after+ safe_delete(?BLOCKED_IDS, 4002)+ end.++fetched_blocked_ids_expire_and_are_reclaimed_test() ->+ init(),+ try+ ok = put_blocked_ids_fetched(4004, [4005]),+ ?assertEqual([4005], get_blocked_ids(4004)),+ Stale = erlang:system_time(second) - 1,+ true = ets:insert(?BLOCKED_IDS, {4004, [4005], Stale}),+ ?assertEqual(undefined, get_blocked_ids(4004)),+ ok = evict_tables(#{}),+ ?assertEqual([], ets:lookup(?BLOCKED_IDS, 4004))+ after+ safe_delete(?BLOCKED_IDS, 4004)+ end.++legacy_untimed_blocked_ids_are_still_honoured_test() ->+ init(),+ try+ true = ets:insert(?BLOCKED_IDS, {4008, [4009]}),+ ?assertEqual([4009], get_blocked_ids(4008)),+ ok = evict_tables(#{}),+ ?assertEqual([4009], get_blocked_ids(4008))+ after+ safe_delete(?BLOCKED_IDS, 4008)+ end.++blocked_ids_ttl_is_clamped_test() ->+ ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, blocked_ids_ttl_seconds()),+ ?assertEqual(?MIN_BLOCKED_IDS_TTL, with_ttl_env(1, fun blocked_ids_ttl_seconds/0)),+ ?assertEqual(?MAX_BLOCKED_IDS_TTL, with_ttl_env(999999, fun blocked_ids_ttl_seconds/0)),+ ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, with_ttl_env(0, fun blocked_ids_ttl_seconds/0)).++with_ttl_env(Value, Fun) ->+ application:set_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds, Value),+ try+ Fun()+ after+ application:unset_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds)+ end.++-endif.diff --git a/fluxer_gateway/src/push/push_fcm_payload.erl b/fluxer_gateway/src/push/push_fcm_payload.erlindex 5296254f8..9796c03b0 100644--- a/fluxer_gateway/src/push/push_fcm_payload.erl+++ b/fluxer_gateway/src/push/push_fcm_payload.erl@@ -40,9 +40,9 @@ build_clear_message(DeviceToken, Payload) -><<"token">> => DeviceToken,<<"data">> => Data,<<"android">> => #{- <<"priority">> => <<"HIGH">>,- <<"ttl">> => <<"300s">>,- <<"collapse_key">> => Tag+ <<"priority">> => <<"NORMAL">>,+ <<"ttl">> => <<"3600s">>,+ <<"collapse_key">> => <<"clear:", Tag/binary>>},<<"fcm_options">> => #{<<"analytics_label">> => <<"notification_clear">>@@ -63,9 +63,8 @@ build_notification_message(DeviceToken, Payload) -><<"image">>, ImageUrl, #{<<"title">> => Title, <<"body">> => Body}),Data = build_notification_data(Payload, Title, Body, Tag, ImageUrl),- Group = resolve_notification_group(maps:get(<<"data">>, Payload, #{}), Tag),AndroidNotification = build_android_notification(Tag, ImageUrl),- wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification, Group).+ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification).-spec build_android_notification(binary(), binary() | undefined) -> map().build_android_notification(Tag, ImageUrl) ->@@ -75,8 +74,8 @@ build_android_notification(Tag, ImageUrl) -><<"click_action">> => <<"FLUXER_MESSAGE">>}).--spec wrap_notification_message(binary(), map(), map(), map(), binary()) -> map().-wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification, Group) ->+-spec wrap_notification_message(binary(), map(), map(), map()) -> map().+wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotification) ->#{<<"message">> => #{<<"token">> => DeviceToken,@@ -85,7 +84,6 @@ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotificati<<"android">> => #{<<"priority">> => <<"HIGH">>,<<"ttl">> => <<"86400s">>,- <<"collapse_key">> => Group,<<"notification">> => AndroidNotification},<<"fcm_options">> => #{@@ -94,41 +92,6 @@ wrap_notification_message(DeviceToken, NotificationBody, Data, AndroidNotificati}}.--spec resolve_notification_group(map(), binary()) -> binary().-resolve_notification_group(Data, MessageTag) ->- case- push_utils:normalize_binary(- maps:get(<<"notification_tag">>, Data, undefined), undefined- )- of- Group when is_binary(Group), byte_size(Group) > 0 ->- Group;- _ ->- resolve_channel_notification_group(Data, MessageTag)- end.---spec resolve_channel_notification_group(map(), binary()) -> binary().-resolve_channel_notification_group(Data, MessageTag) ->- case- push_utils:normalize_binary(- maps:get(<<"channel_id">>, Data, undefined), undefined- )- of- ChannelId when is_binary(ChannelId), byte_size(ChannelId) > 0 ->- <<"channel:", ChannelId/binary>>;- _ ->- derive_channel_tag_from_message_tag(MessageTag)- end.---spec derive_channel_tag_from_message_tag(binary()) -> binary().-derive_channel_tag_from_message_tag(MessageTag) ->- case binary:split(MessageTag, <<":">>, [global]) of- [<<"channel">>, ChannelId, _MessageId] when byte_size(ChannelId) > 0 ->- <<"channel:", ChannelId/binary>>;- _ ->- MessageTag- end.--spec build_notification_data(map(), binary(), binary(), binary(), binary() | undefined) ->map().build_notification_data(Payload, Title, Body, Tag, ImageUrl) ->@@ -349,7 +312,7 @@ build_message_includes_android_chat_notification_fields_test() ->?assertEqual(<<"fluxer_default_push">>, maps:get(<<"channel_id">>, AndroidNotification)),?assertEqual(<<"channel:123:456">>, maps:get(<<"tag">>, AndroidNotification)),Android = maps:get(<<"android">>, Message),- ?assertEqual(<<"channel:123">>, maps:get(<<"collapse_key">>, Android)),+ ?assertEqual(false, maps:is_key(<<"collapse_key">>, Android)),?assertEqual(<<"https://cdn.example/image.png">>, maps:get(<<"image">>, AndroidNotification)),@@ -370,9 +333,9 @@ build_clear_message_is_data_only_and_collapsible_test() ->#{<<"message">> := Message} = build_message(<<"device-token">>, Payload),?assertEqual(false, maps:is_key(<<"notification">>, Message)),Android = maps:get(<<"android">>, Message),- ?assertEqual(<<"HIGH">>, maps:get(<<"priority">>, Android)),- ?assertEqual(<<"300s">>, maps:get(<<"ttl">>, Android)),- ?assertEqual(<<"channel:123">>, maps:get(<<"collapse_key">>, Android)),+ ?assertEqual(<<"NORMAL">>, maps:get(<<"priority">>, Android)),+ ?assertEqual(<<"3600s">>, maps:get(<<"ttl">>, Android)),+ ?assertEqual(<<"clear:channel:123">>, maps:get(<<"collapse_key">>, Android)),Data = maps:get(<<"data">>, Message),?assertEqual(<<"notification_clear">>, maps:get(<<"type">>, Data)),?assertEqual(<<"clear_channel">>, maps:get(<<"action">>, Data)),diff --git a/fluxer_gateway/src/push/push_message_params.erl b/fluxer_gateway/src/push/push_message_params.erlindex a328dcad7..846b40ee3 100644--- a/fluxer_gateway/src/push/push_message_params.erl+++ b/fluxer_gateway/src/push/push_message_params.erl@@ -7,6 +7,8 @@-export_type([context/0]).+-define(MAX_GUILD_FEATURES, 64).+-type context() :: #{message_data := map(),user_ids := [pos_integer()],@@ -20,7 +22,8 @@role_names := map(),user_roles := map(),connected_users := map(),- markdown_context := map()+ markdown_context := map(),+ large_guild_metadata := map() | undefined}.-spec context(map()) -> {ok, context()} | {error, term()}.@@ -58,9 +61,34 @@ context_from_message_data(Params, MessageData) ->connected_users => maps:get(connected_users, Params, #{}),markdown_context => markdown_context(MessageData, GuildId, RoleNames, maps:get(markdown_context, Params, undefined)- )+ ),+ large_guild_metadata => large_guild_metadata(Params)}).+-spec large_guild_metadata(map()) -> map() | undefined.+large_guild_metadata(Params) ->+ MemberCount = maps:get(guild_member_count, Params, undefined),+ Features = maps:get(guild_features, Params, undefined),+ build_large_guild_metadata(MemberCount, Features).++-spec build_large_guild_metadata(term(), term()) -> map() | undefined.+build_large_guild_metadata(MemberCount, Features) when+ is_integer(MemberCount), MemberCount >= 0, is_list(Features)+->+ #{member_count => MemberCount, features => bounded_features(Features, [], 0)};+build_large_guild_metadata(_MemberCount, _Features) ->+ undefined.++-spec bounded_features(term(), [binary()], non_neg_integer()) -> [binary()].+bounded_features(_Features, Acc, ?MAX_GUILD_FEATURES) ->+ lists:reverse(Acc);+bounded_features([Feature | Rest], Acc, Count) when is_binary(Feature) ->+ bounded_features(Rest, [Feature | Acc], Count + 1);+bounded_features([_Feature | Rest], Acc, Count) ->+ bounded_features(Rest, Acc, Count);+bounded_features(_Features, Acc, _Count) ->+ lists:reverse(Acc).+-spec owner_key(map()) -> term().owner_key(Params) ->case user_ids(maps:get(user_ids, Params, [])) ofdiff --git a/fluxer_gateway/src/push/push_notification.erl b/fluxer_gateway/src/push/push_notification.erlindex 2bd103511..71fa239ec 100644--- a/fluxer_gateway/src/push/push_notification.erl+++ b/fluxer_gateway/src/push/push_notification.erl@@ -133,7 +133,7 @@ assemble_payload(ContentPreview,AuthorAvatarUrl) ->- Data = build_data(Ctx),+ Data = build_data(Ctx, AuthorAvatarUrl),Notification = build_notification_body(Ctx, Title, ContentPreview, AuthorAvatarUrl, Data),maps:merge(#{@@ -149,19 +149,23 @@ assemble_payload(ImageFields).--spec build_data(push_ctx()) -> map().-build_data(#{- channel_id := ChannelId,- message_id := MessageId,- guild_id := GuildId,- navigate_url := NavigateUrl,- badge_value := BadgeValue,- target_user_id := TargetUserId,- image_url := ImageUrl,- image_fields := ImageFields-}) ->+-spec build_data(push_ctx(), binary()) -> map().+build_data(+ #{+ channel_id := ChannelId,+ message_id := MessageId,+ guild_id := GuildId,+ navigate_url := NavigateUrl,+ badge_value := BadgeValue,+ target_user_id := TargetUserId,+ image_url := ImageUrl,+ image_fields := ImageFields+ },+ AuthorAvatarUrl+) ->BaseData = #{<<"channel_id">> => integer_to_binary(ChannelId),+ <<"author_avatar_url">> => AuthorAvatarUrl,<<"message_id">> => integer_to_binary(MessageId),<<"notification_tag">> => build_channel_tag(ChannelId),<<"guild_id">> =>diff --git a/fluxer_gateway/src/push/push_notification_format.erl b/fluxer_gateway/src/push/push_notification_format.erlindex 1fbcb8b3d..a9d2c6097 100644--- a/fluxer_gateway/src/push/push_notification_format.erl+++ b/fluxer_gateway/src/push/push_notification_format.erl@@ -326,9 +326,18 @@ first_nonempty_binary([Value | Rest]) ->-spec truncate_preview(binary()) -> binary().truncate_preview(Content) when byte_size(Content) > 100 ->- binary:part(Content, 0, 100);+ valid_utf8_prefix(binary:part(Content, 0, 100));truncate_preview(Content) ->- Content.+ valid_utf8_prefix(Content).++-spec valid_utf8_prefix(binary()) -> binary().+valid_utf8_prefix(Content) ->+ case unicode:characters_to_binary(Content, utf8, utf8) of+ Valid when is_binary(Valid) -> Valid;+ {incomplete, Valid, _Rest} when is_binary(Valid) -> Valid;+ {error, Valid, _Rest} when is_binary(Valid) -> Valid;+ _ -> <<>>+ end.-spec build_content_fallback_preview(map()) -> binary().build_content_fallback_preview(MessageData) ->@@ -568,6 +577,23 @@ lowercase_binary(Value) ->-ifdef(TEST).-include_lib("eunit/include/eunit.hrl").+truncate_preview_never_splits_a_utf8_character_test() ->+ Emoji = <<"\xF0\x9F\x98\x80">>,+ Content = <<(binary:copy(<<"a">>, 98))/binary, Emoji/binary>>,+ ?assert(byte_size(Content) > 100),+ Truncated = truncate_preview(Content),+ ?assert(byte_size(Truncated) =< 100),+ ?assertMatch(Bin when is_binary(Bin), unicode:characters_to_binary(Truncated, utf8, utf8)),+ ?assertEqual(binary:copy(<<"a">>, 98), Truncated).++truncate_preview_is_unchanged_for_valid_ascii_test() ->+ Content = binary:copy(<<"a">>, 150),+ ?assertEqual(binary:copy(<<"a">>, 100), truncate_preview(Content)).++truncate_preview_keeps_short_valid_content_identical_test() ->+ Content = <<"hello \xC3\xA9 world">>,+ ?assertEqual(Content, truncate_preview(Content)).+build_url_dm_test() ->?assertEqual(<<"/channels/@me/456/789">>, build_url(0, 456, 789)).diff --git a/fluxer_gateway/src/push/push_sender.erl b/fluxer_gateway/src/push/push_sender.erlindex 3726a801a..7d7c05bcd 100644--- a/fluxer_gateway/src/push/push_sender.erl+++ b/fluxer_gateway/src/push/push_sender.erl@@ -13,6 +13,11 @@-export_type([send_context/0]).+-define(DEFAULT_BADGE_FETCH_BATCH, 2000).+-define(BADGE_FETCH_MAX_CONSECUTIVE_FAILURES, 3).+-define(DEFAULT_BADGE_FETCH_BUDGET_MS, 120000).+-define(PUSH_COUNTERS, push_worker_counter).+-type send_context() :: #{message_data := map(),guild_id := integer(),@@ -257,16 +262,128 @@ check_badge_cache(UserId, TTL, Now, Acc, MissingAcc) ->-spec fetch_badge_counts([integer()], map(), integer()) -> map().fetch_badge_counts(UserIds, Counts, CachedAt) ->+ Batches = chunk_badge_user_ids(UserIds, badge_fetch_batch_size(), []),+ {Counted, FailedBatches, DefaultedUsers, _Consecutive} =+ fetch_badge_count_batches(Batches, {Counts, 0, 0, 0}, CachedAt),+ report_badge_fetch_failures(FailedBatches, DefaultedUsers),+ Counted.++-type badge_batch_acc() :: {map(), non_neg_integer(), non_neg_integer(), non_neg_integer()}.++-spec fetch_badge_count_batches([[integer()]], badge_batch_acc(), integer()) ->+ badge_batch_acc().+fetch_badge_count_batches(Batches, Acc, CachedAt) ->+ Deadline = erlang:monotonic_time(millisecond) + badge_fetch_budget_ms(),+ fetch_badge_count_batches(Batches, Acc, CachedAt, Deadline).++-spec fetch_badge_count_batches([[integer()]], badge_batch_acc(), integer(), integer()) ->+ badge_batch_acc().+fetch_badge_count_batches([], Acc, _CachedAt, _Deadline) ->+ Acc;+fetch_badge_count_batches(Remaining, Acc, CachedAt, Deadline) ->+ case badge_fetch_exhausted(Acc, Deadline) of+ true -> abandon_badge_batches(Remaining, Acc);+ false -> fetch_next_badge_batch(Remaining, Acc, CachedAt, Deadline)+ end.++-spec badge_fetch_exhausted(badge_batch_acc(), integer()) -> boolean().+badge_fetch_exhausted({_Counts, _FailedBatches, _DefaultedUsers, Consecutive}, Deadline) ->+ Consecutive >= ?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES orelse+ erlang:monotonic_time(millisecond) >= Deadline.++-spec abandon_badge_batches([[integer()]], badge_batch_acc()) -> badge_batch_acc().+abandon_badge_batches(Remaining, {Counts, FailedBatches, DefaultedUsers, Consecutive}) ->+ {Counts, FailedBatches + length(Remaining),+ DefaultedUsers + badge_batched_user_count(Remaining, 0), Consecutive}.++-spec badge_batched_user_count([[integer()]], non_neg_integer()) -> non_neg_integer().+badge_batched_user_count([], Acc) ->+ Acc;+badge_batched_user_count([Batch | Rest], Acc) ->+ badge_batched_user_count(Rest, Acc + length(Batch)).++-spec fetch_next_badge_batch([[integer()]], badge_batch_acc(), integer(), integer()) ->+ badge_batch_acc().+fetch_next_badge_batch(+ [Batch | Rest], {Counts, FailedBatches, DefaultedUsers, Consecutive}, CachedAt, Deadline+) ->Request = #{<<"type">> => <<"get_badge_counts">>,- <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]+ <<"user_ids">> => [integer_to_binary(UserId) || UserId <- Batch]},case rpc_client:call(Request) of{ok, Data} ->BadgeData = maps:get(<<"badge_counts">>, Data, #{}),- merge_badge_data(UserIds, BadgeData, Counts, CachedAt);+ Merged = merge_badge_data(Batch, BadgeData, Counts, CachedAt),+ fetch_badge_count_batches(+ Rest, {Merged, FailedBatches, DefaultedUsers, 0}, CachedAt, Deadline+ );{error, _Reason} ->- Counts+ fetch_badge_count_batches(+ Rest,+ {Counts, FailedBatches + 1, DefaultedUsers + length(Batch), Consecutive + 1},+ CachedAt,+ Deadline+ )+ end.++-spec report_badge_fetch_failures(non_neg_integer(), non_neg_integer()) -> ok.+report_badge_fetch_failures(0, _DefaultedUsers) ->+ ok;+report_badge_fetch_failures(FailedBatches, DefaultedUsers) ->+ bump_counter(badge_fetch_calls_failed, FailedBatches),+ bump_counter(badge_fetch_users_defaulted, DefaultedUsers),+ logger:warning("Push: badge count batches failed; users default to zero badge", #{+ failed_batches => FailedBatches, users_defaulted => DefaultedUsers+ }).++-spec bump_counter(atom(), integer()) -> ok.+bump_counter(Key, Delta) ->+ try+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),+ ok+ catch+ error:badarg -> seed_and_bump_counter(Key, Delta)+ end.++-spec seed_and_bump_counter(atom(), integer()) -> ok.+seed_and_bump_counter(Key, Delta) ->+ try+ _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}),+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),+ ok+ catch+ error:badarg -> ok+ end.++-spec chunk_badge_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].+chunk_badge_user_ids([], _BatchSize, Acc) ->+ lists:reverse(Acc);+chunk_badge_user_ids(UserIds, BatchSize, Acc) ->+ {Batch, Rest} = take_badge_user_id_batch(UserIds, BatchSize, []),+ chunk_badge_user_ids(Rest, BatchSize, [Batch | Acc]).++-spec take_badge_user_id_batch([integer()], non_neg_integer(), [integer()]) ->+ {[integer()], [integer()]}.+take_badge_user_id_batch(Rest, 0, Acc) ->+ {lists:reverse(Acc), Rest};+take_badge_user_id_batch([], _Remaining, Acc) ->+ {lists:reverse(Acc), []};+take_badge_user_id_batch([UserId | Rest], Remaining, Acc) ->+ take_badge_user_id_batch(Rest, Remaining - 1, [UserId | Acc]).++-spec badge_fetch_batch_size() -> pos_integer().+badge_fetch_batch_size() ->+ case application:get_env(fluxer_gateway, push_badge_fetch_batch_size, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> ?DEFAULT_BADGE_FETCH_BATCH+ end.++-spec badge_fetch_budget_ms() -> pos_integer().+badge_fetch_budget_ms() ->+ case application:get_env(fluxer_gateway, push_badge_fetch_budget_ms, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> ?DEFAULT_BADGE_FETCH_BUDGET_MSend.-spec merge_badge_data([integer()], map(), map(), integer()) -> map().@@ -285,3 +402,119 @@ merge_badge_data(UserIds, BadgeData, Counts, CachedAt) ->-spec normalize_badge_count(integer() | term()) -> non_neg_integer().normalize_badge_count(Value) when is_integer(Value), Value >= 0 -> Value;normalize_badge_count(_) -> 0.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++chunk_badge_user_ids_uses_bounded_batches_test() ->+ ?assertEqual([[1, 2], [3, 4], [5]], chunk_badge_user_ids([1, 2, 3, 4, 5], 2, [])),+ ?assertEqual([], chunk_badge_user_ids([], 2, [])).++fetch_badge_counts_in_batches_keeps_successful_batches_test() ->+ ok = meck:new(rpc_client, [passthrough, no_link]),+ ok = meck:new(push_ets_cache, [passthrough, no_link]),+ application:set_env(fluxer_gateway, push_badge_fetch_batch_size, 2),+ try+ ok = meck:expect(push_ets_cache, put_badge_count, fun(_UserId, _Count, _At) -> ok end),+ ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) ->+ case Ids of+ [<<"1">>, <<"2">>] ->+ {ok, #{<<"badge_counts">> => #{<<"1">> => 3, <<"2">> => 4}}};+ _ ->+ {error, unavailable}+ end+ end),+ ?assertEqual(#{1 => 3, 2 => 4}, fetch_badge_counts([1, 2, 3, 4], #{}, 0))+ after+ application:unset_env(fluxer_gateway, push_badge_fetch_batch_size),+ meck:unload(push_ets_cache),+ meck:unload(rpc_client)+ end.++fetch_badge_count_batches_stops_after_consecutive_failures_test() ->+ ok = meck:new(rpc_client, [passthrough, no_link]),+ try+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),+ Batches = [[N] || N <- lists:seq(1, 10)],+ ?assertEqual(+ {#{}, 10, 10, ?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES},+ fetch_badge_count_batches(Batches, {#{}, 0, 0, 0}, 0)+ ),+ ?assertEqual(?BADGE_FETCH_MAX_CONSECUTIVE_FAILURES, length(meck:history(rpc_client)))+ after+ meck:unload(rpc_client)+ end.++fetch_badge_count_batches_stops_at_the_wall_clock_budget_test() ->+ ok = meck:new(rpc_client, [passthrough, no_link]),+ application:set_env(fluxer_gateway, push_badge_fetch_budget_ms, 1),+ try+ ok = meck:expect(rpc_client, call, fun(_Req) ->+ {ok, #{<<"badge_counts">> => #{}}}+ end),+ Batches = [[N] || N <- lists:seq(1, 10)],+ Deadline = erlang:monotonic_time(millisecond) - 1,+ ?assertEqual(+ {#{}, 10, 10, 0},+ fetch_badge_count_batches(Batches, {#{}, 0, 0, 0}, 0, Deadline)+ ),+ ?assertEqual(0, length(meck:history(rpc_client)))+ after+ application:unset_env(fluxer_gateway, push_badge_fetch_budget_ms),+ meck:unload(rpc_client)+ end.++badge_fetch_defaults_missing_users_to_zero_test() ->+ ok = meck:new(rpc_client, [passthrough, no_link]),+ ok = meck:new(push_ets_cache, [passthrough, no_link]),+ try+ ok = meck:expect(push_ets_cache, put_badge_count, fun(_UserId, _Count, _At) -> ok end),+ ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) ->+ ?assertEqual([<<"1">>, <<"2">>, <<"3">>], Ids),+ {ok, #{<<"badge_counts">> => #{<<"1">> => 1}}}+ end),+ ?assertEqual(#{1 => 1, 2 => 0, 3 => 0}, fetch_badge_counts([1, 2, 3], #{}, 0))+ after+ meck:unload(push_ets_cache),+ meck:unload(rpc_client)+ end.++badge_fetch_failure_counts_defaulted_users_test() ->+ ok = meck:new(rpc_client, [passthrough, no_link]),+ ensure_test_counter_table(),+ ok = reset_test_counter(badge_fetch_calls_failed),+ ok = reset_test_counter(badge_fetch_users_defaulted),+ try+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),+ ?assertEqual(#{}, fetch_badge_counts([1, 2, 3], #{}, 0)),+ ?assertEqual(1, counter_value(badge_fetch_calls_failed)),+ ?assertEqual(3, counter_value(badge_fetch_users_defaulted))+ after+ meck:unload(rpc_client)+ end.++counter_value(Key) ->+ try ets:lookup(?PUSH_COUNTERS, Key) of+ [{Key, Value}] when is_integer(Value) -> Value;+ _ -> 0+ catch+ error:badarg -> 0+ end.++ensure_test_counter_table() ->+ case ets:info(?PUSH_COUNTERS, name) of+ undefined ->+ _ = ets:new(?PUSH_COUNTERS, [named_table, public, set, {write_concurrency, true}]),+ ok;+ _ ->+ ok+ end.++reset_test_counter(Key) ->+ try ets:insert(?PUSH_COUNTERS, {Key, 0}) of+ _ -> ok+ catch+ error:badarg -> ok+ end.++-endif.diff --git a/fluxer_gateway/src/push/push_sender_delivery.erl b/fluxer_gateway/src/push/push_sender_delivery.erlindex 73ab037f6..e0f7d52b3 100644--- a/fluxer_gateway/src/push/push_sender_delivery.erl+++ b/fluxer_gateway/src/push/push_sender_delivery.erl@@ -561,7 +561,11 @@ maybe_retry_transient_waits_before_retry_test() ->fun retry_request_meck/6),?assertEqual(false, maybe_retry_transient(Ctx, RecordSize, 0, {error, timeout})),- ?assertEqual({retry_wait, ?BASE_RETRY_DELAY_MS}, receive_retry_wait()),+ %% retry_delay/1 adds rand:uniform(Base div 4) of jitter, so the first+ %% retry is anywhere in [Base, Base + Base div 4 - 1].+ {retry_wait, DelayMs} = receive_retry_wait(),+ ?assert(DelayMs >= ?BASE_RETRY_DELAY_MS),+ ?assert(DelayMs =< ?BASE_RETRY_DELAY_MS + (?BASE_RETRY_DELAY_MS div 4) - 1),?assertEqual(ok, receive_retried_push_request(Endpoint)),?assert(meck:validate(gateway_retry_timer)),?assert(meck:validate(push_utils)),diff --git a/fluxer_gateway/src/push/push_subscriptions.erl b/fluxer_gateway/src/push/push_subscriptions.erlindex 5e3193651..c18e168d5 100644--- a/fluxer_gateway/src/push/push_subscriptions.erl+++ b/fluxer_gateway/src/push/push_subscriptions.erl@@ -11,10 +11,18 @@-export([fetch_and_cache_user_guild_settings/2]).-export([delete_failed_subscriptions/1]).-export([delivery_concurrency/0]).+-export([delivery_stats/0]).-define(DEFAULT_DELIVERY_CONCURRENCY, 8).--define(DELIVERY_WORKER_TIMEOUT_MS, 30000).--define(SUBSCRIPTION_FETCH_CHUNK_SIZE, 256).+-define(DEFAULT_DELIVERY_MAX_WORKERS, 64).+-define(DEFAULT_DELIVERY_TASKS_PER_WORKER, 128).+-define(DEFAULT_SUBSCRIPTION_FETCH_BATCH, 2000).+-define(SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES, 3).+-define(DEFAULT_SUBSCRIPTION_FETCH_BUDGET_MS, 120000).+-define(DELIVERY_IDLE_TIMEOUT_MS, 30000).+-define(PUSH_COUNTERS, push_worker_counter).+-define(CNT_LIVE_WORKERS, delivery_live_workers).+-define(CTX_BADGE_COUNTS, 9).-spec fetch_and_send_subscriptions([integer()],@@ -92,60 +100,118 @@ send_missing_if_any(MissingUserIds, Ctx) ->-spec fetch_and_send_missing_subscriptions([integer()], tuple()) -> ok.fetch_and_send_missing_subscriptions(UserIds, Ctx) ->- {FetchedUserIds, SubscriptionsData} = fetch_missing_subscriptions(UserIds),- send_fetched_subscriptions(FetchedUserIds, SubscriptionsData, Ctx).---spec fetch_missing_subscriptions([integer()]) -> {[integer()], map()}.-fetch_missing_subscriptions(UserIds) ->- Chunks = chunk_user_ids(UserIds, ?SUBSCRIPTION_FETCH_CHUNK_SIZE, []),- {FetchedChunks, SubscriptionsData} = lists:foldl(- fun fetch_subscriptions_chunk/2,- {[], #{}},- Chunks- ),- {lists:append(lists:reverse(FetchedChunks)), SubscriptionsData}.+ Batches = chunk_user_ids(UserIds, subscription_fetch_batch_size(), []),+ logger:debug("Push: fetching subscriptions via batched RPC", #{+ user_count => length(UserIds), batch_count => length(Batches)+ }),+ {Tasks, FailedBatches, FailedUsers, _Consecutive} =+ fetch_subscription_batches(Batches, {[], 0, 0, 0}),+ report_subscription_fetch_failures(FailedBatches, FailedUsers, Ctx),+ deliver_subscription_tasks(lists:reverse(Tasks), Ctx).--spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].-chunk_user_ids([], _ChunkSize, Acc) ->- lists:reverse(Acc);-chunk_user_ids(UserIds, ChunkSize, Acc) when length(UserIds) =< ChunkSize ->- lists:reverse([UserIds | Acc]);-chunk_user_ids(UserIds, ChunkSize, Acc) ->- {Chunk, Rest} = lists:split(ChunkSize, UserIds),- chunk_user_ids(Rest, ChunkSize, [Chunk | Acc]).---spec fetch_subscriptions_chunk([integer()], {[[integer()]], map()}) -> {[[integer()]], map()}.-fetch_subscriptions_chunk(UserIds, {FetchedAcc, DataAcc}) ->- SubscriptionsReq = #{+-type subscription_batch_acc() ::+ {[{integer(), list()}], non_neg_integer(), non_neg_integer(), non_neg_integer()}.++-spec fetch_subscription_batches([[integer()]], subscription_batch_acc()) ->+ subscription_batch_acc().+fetch_subscription_batches(Batches, Acc) ->+ Deadline = erlang:monotonic_time(millisecond) + subscription_fetch_budget_ms(),+ fetch_subscription_batches(Batches, Acc, Deadline).++-spec fetch_subscription_batches([[integer()]], subscription_batch_acc(), integer()) ->+ subscription_batch_acc().+fetch_subscription_batches([], Acc, _Deadline) ->+ Acc;+fetch_subscription_batches(Remaining, Acc, Deadline) ->+ case subscription_fetch_exhausted(Acc, Deadline) of+ true -> abandon_subscription_batches(Remaining, Acc);+ false -> fetch_next_subscription_batch(Remaining, Acc, Deadline)+ end.++-spec subscription_fetch_exhausted(subscription_batch_acc(), integer()) -> boolean().+subscription_fetch_exhausted({_Tasks, _FailedBatches, _FailedUsers, Consecutive}, Deadline) ->+ Consecutive >= ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES orelse+ erlang:monotonic_time(millisecond) >= Deadline.++-spec abandon_subscription_batches([[integer()]], subscription_batch_acc()) ->+ subscription_batch_acc().+abandon_subscription_batches(Remaining, {Tasks, FailedBatches, FailedUsers, Consecutive}) ->+ {Tasks, FailedBatches + length(Remaining), FailedUsers + batched_user_count(Remaining, 0),+ Consecutive}.++-spec fetch_next_subscription_batch([[integer()]], subscription_batch_acc(), integer()) ->+ subscription_batch_acc().+fetch_next_subscription_batch(+ [Batch | Rest], {Tasks, FailedBatches, FailedUsers, Consecutive}, Deadline+) ->+ Req = #{<<"type">> => <<"get_push_subscriptions">>,- <<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]+ <<"user_ids">> => [integer_to_binary(UserId) || UserId <- Batch]},- logger:debug("Push: fetching subscriptions via RPC", #{user_count => length(UserIds)}),- case rpc_client:call(SubscriptionsReq) of- {ok, SubscriptionsData} ->- logger:debug("Push: RPC returned subscriptions", #{- user_count => length(UserIds),- response_keys => maps:keys(SubscriptionsData)- }),- {[UserIds | FetchedAcc], maps:merge(DataAcc, SubscriptionsData)};+ case rpc_client:call(Req) of+ {ok, BatchData} ->+ BatchTasks = lists:foldl(+ fun(UserId, Acc) ->+ add_fetched_user_subscription_task(UserId, BatchData, Acc)+ end,+ Tasks,+ Batch+ ),+ fetch_subscription_batches(+ Rest, {BatchTasks, FailedBatches, FailedUsers, 0}, Deadline+ );{error, Reason} ->logger:debug("Push: RPC failed to fetch subscriptions",- #{user_count => length(UserIds), reason => Reason}+ #{user_count => length(Batch), reason => Reason}),- {FetchedAcc, DataAcc}+ fetch_subscription_batches(+ Rest,+ {Tasks, FailedBatches + 1, FailedUsers + length(Batch), Consecutive + 1},+ Deadline+ )end.--spec send_fetched_subscriptions([integer()], map(), tuple()) -> ok.-send_fetched_subscriptions(UserIds, SubscriptionsData, Ctx) ->- Tasks = lists:foldl(- fun(UserId, Acc) ->- add_fetched_user_subscription_task(UserId, SubscriptionsData, Acc)- end,- [],- UserIds- ),- send_subscription_tasks(lists:reverse(Tasks), Ctx).+-spec batched_user_count([[integer()]], non_neg_integer()) -> non_neg_integer().+batched_user_count([], Acc) ->+ Acc;+batched_user_count([Batch | Rest], Acc) ->+ batched_user_count(Rest, Acc + length(Batch)).++-spec report_subscription_fetch_failures(non_neg_integer(), non_neg_integer(), tuple()) -> ok.+report_subscription_fetch_failures(0, _FailedUsers, _Ctx) ->+ ok;+report_subscription_fetch_failures(FailedBatches, FailedUsers, Ctx) ->+ bump_counter(subscription_fetch_calls_failed, FailedBatches),+ bump_counter(subscription_fetch_users_dropped, FailedUsers),+ {GuildId, MessageId} = delivery_ids(Ctx),+ logger:warning("Push: subscription fetch batches failed; recipients not attempted", #{+ guild_id => GuildId,+ message_id => MessageId,+ failed_batches => FailedBatches,+ recipients_not_attempted => FailedUsers+ }).++-spec chunk_user_ids([integer()], pos_integer(), [[integer()]]) -> [[integer()]].+chunk_user_ids([], _BatchSize, Acc) ->+ lists:reverse(Acc);+chunk_user_ids(UserIds, BatchSize, Acc) ->+ {Batch, Rest} = take_user_id_batch(UserIds, BatchSize, []),+ chunk_user_ids(Rest, BatchSize, [Batch | Acc]).++-spec take_user_id_batch([integer()], non_neg_integer(), [integer()]) ->+ {[integer()], [integer()]}.+take_user_id_batch(Rest, 0, Acc) ->+ {lists:reverse(Acc), Rest};+take_user_id_batch([], _Remaining, Acc) ->+ {lists:reverse(Acc), []};+take_user_id_batch([UserId | Rest], Remaining, Acc) ->+ take_user_id_batch(Rest, Remaining - 1, [UserId | Acc]).++-spec count_subscription_fetch_loss(non_neg_integer()) -> ok.+count_subscription_fetch_loss(UserCount) ->+ bump_counter(subscription_fetch_calls_failed, 1),+ bump_counter(subscription_fetch_users_dropped, UserCount).-spec add_fetched_user_subscription_task(integer(), map(), [{integer(), list()}]) ->[{integer(), list()}].@@ -205,6 +271,7 @@ send_clear_rpc_result(UserId, ChannelId, MessageId, BadgeCount, {ok, SubscriptioUserId, ChannelId, MessageId, BadgeCount, maps:get(UserIdBin, SubscriptionsData, []));send_clear_rpc_result(UserId, _ChannelId, _MessageId, _BadgeCount, {error, Reason}) ->+ count_subscription_fetch_loss(1),logger:debug("Push: RPC failed to fetch subscriptions for notification clear",#{user_id => UserId, reason => Reason}@@ -238,7 +305,7 @@ send_cached_subscriptions(SubscriptionsByUser, Ctx) ->[],SubscriptionsByUser),- send_subscription_tasks(Tasks, Ctx).+ deliver_subscription_tasks(Tasks, Ctx).-spec add_cached_user_task(integer(), list(), [{integer(), list()}]) ->[{integer(), list()}].@@ -250,21 +317,335 @@ add_cached_user_task(UserId, Subscriptions, Acc) ->[{UserId, Subscriptions} | Acc]end.--spec send_subscription_tasks([{integer(), list()}], tuple()) -> ok.-send_subscription_tasks(Tasks, Ctx) ->- send_subscription_tasks(Tasks, Ctx, delivery_concurrency()).+-spec deliver_subscription_tasks([{integer(), list()}], tuple()) -> ok.+deliver_subscription_tasks(Tasks, Ctx) ->+ send_bounded_subscription_tasks(Tasks, Ctx).--spec send_subscription_tasks([{integer(), list()}], tuple(), pos_integer()) -> ok.-send_subscription_tasks([], _Ctx, _MaxConcurrency) ->+-spec send_bounded_subscription_tasks([{integer(), list()}], tuple()) -> ok.+send_bounded_subscription_tasks([], _Ctx) ->ok;-send_subscription_tasks(Tasks, Ctx, 1) ->- send_subscription_task_chunk(Tasks, Ctx);-send_subscription_tasks(Tasks, Ctx, MaxConcurrency) ->+send_bounded_subscription_tasks(Tasks, Ctx) ->TaskCount = length(Tasks),- WorkerCount = min(TaskCount, MaxConcurrency),+ Desired = bounded_worker_count(TaskCount, delivery_concurrency(), delivery_max_workers()),+ Granted = claim_delivery_slots(Desired, gateway_http_client:push_max_concurrency()),+ try+ run_claimed_delivery(Tasks, TaskCount, Ctx, Granted)+ after+ release_delivery_slots(Granted)+ end.++-spec run_claimed_delivery([{integer(), list()}], pos_integer(), tuple(), non_neg_integer()) ->+ ok.+run_claimed_delivery(Tasks, _TaskCount, Ctx, 0) ->+ bump_counter(delivery_budget_exhausted, 1),+ run_inline_delivery(Tasks, Ctx);+run_claimed_delivery(Tasks, TaskCount, Ctx, WorkerCount) ->ChunkSize = max(1, (TaskCount + WorkerCount - 1) div WorkerCount),Chunks = chunk_subscription_tasks(Tasks, ChunkSize, []),- wait_delivery_workers(start_delivery_workers(Chunks, Ctx, #{})).+ Workers = start_progress_workers(+ Chunks, shared_ctx_payload(Ctx), ctx_badge_counts(Ctx), self(), #{}+ ),+ await_progress_workers(Workers, 0, TaskCount, delivery_ids(Ctx)).++-spec run_inline_delivery([{integer(), list()}], tuple()) -> ok.+run_inline_delivery([], _Ctx) ->+ ok;+run_inline_delivery([Task | Rest], Ctx) ->+ run_subscription_task(Task, Ctx),+ run_inline_delivery(Rest, Ctx).++-spec claim_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer().+claim_delivery_slots(Desired, Budget) ->+ try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, Desired}) of+ Reserved -> settle_delivery_claim(Desired, Budget, Reserved)+ catch+ error:badarg -> claim_first_delivery_slots(Desired, Budget)+ end.++-spec claim_first_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer().+claim_first_delivery_slots(Desired, Budget) ->+ try ets:insert_new(?PUSH_COUNTERS, {?CNT_LIVE_WORKERS, 0}) of+ _Inserted -> retry_claim_delivery_slots(Desired, Budget)+ catch+ error:badarg -> 0+ end.++-spec retry_claim_delivery_slots(pos_integer(), pos_integer()) -> non_neg_integer().+retry_claim_delivery_slots(Desired, Budget) ->+ try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, Desired}) of+ Reserved -> settle_delivery_claim(Desired, Budget, Reserved)+ catch+ error:badarg -> 0+ end.++-spec settle_delivery_claim(pos_integer(), pos_integer(), integer()) -> non_neg_integer().+settle_delivery_claim(Desired, Budget, Reserved) ->+ Granted = min(Desired, max(0, Budget - (Reserved - Desired))),+ release_delivery_slots(Desired - Granted),+ Granted.++-spec release_delivery_slots(non_neg_integer()) -> ok.+release_delivery_slots(0) ->+ ok;+release_delivery_slots(Count) ->+ try ets:update_counter(?PUSH_COUNTERS, ?CNT_LIVE_WORKERS, {2, -Count, 0, 0}) of+ _Value -> ok+ catch+ error:badarg -> ok+ end.++-spec bounded_worker_count(pos_integer(), pos_integer(), pos_integer()) -> pos_integer().+bounded_worker_count(TaskCount, Base, Bound) ->+ TasksPerWorker = delivery_tasks_per_worker(),+ Sized = (TaskCount + TasksPerWorker - 1) div TasksPerWorker,+ min(TaskCount, min(Bound, max(Base, Sized))).++-type delivery_workers() :: #{pid() => {reference(), non_neg_integer()}}.++-spec start_progress_workers(+ [[{integer(), list()}]], binary(), map(), pid(), delivery_workers()+) -> delivery_workers().+start_progress_workers([], _Shared, _BadgeCounts, _Coordinator, Workers) ->+ Workers;+start_progress_workers([Chunk | Rest], Shared, BadgeCounts, Coordinator, Workers) ->+ ChunkSize = length(Chunk),+ ChunkBadges = chunk_badge_counts(Chunk, BadgeCounts),+ bump_counter(delivery_workers_spawned, 1),+ {Pid, Ref} = spawn_monitor(fun() ->+ run_progress_chunk(Chunk, restore_chunk_ctx(Shared, ChunkBadges), Coordinator)+ end),+ start_progress_workers(+ Rest, Shared, BadgeCounts, Coordinator, Workers#{Pid => {Ref, ChunkSize}}+ ).++-spec shared_ctx_payload(tuple()) -> binary().+shared_ctx_payload(Ctx) ->+ term_to_binary(setelement(?CTX_BADGE_COUNTS, Ctx, #{})).++-spec restore_chunk_ctx(binary(), map()) -> tuple().+restore_chunk_ctx(Shared, BadgeCounts) ->+ case binary_to_term(Shared) of+ Ctx when is_tuple(Ctx) -> setelement(?CTX_BADGE_COUNTS, Ctx, BadgeCounts)+ end.++-spec chunk_badge_counts([{integer(), list()}], map()) -> map().+chunk_badge_counts(Chunk, BadgeCounts) ->+ maps:with([UserId || {UserId, _Subscriptions} <- Chunk], BadgeCounts).++-spec ctx_badge_counts(tuple()) -> map().+ctx_badge_counts(Ctx) ->+ {+ _MsgData,+ _GuildId,+ _ChannelId,+ _MessageId,+ _GuildName,+ _ChannelName,+ _MarkdownContext,+ _ContentPreview,+ BadgeCounts+ } = Ctx,+ BadgeCounts.++-spec run_progress_chunk([{integer(), list()}], tuple(), pid()) -> ok.+run_progress_chunk([], _Ctx, _Coordinator) ->+ ok;+run_progress_chunk([Task | Rest], Ctx, Coordinator) ->+ run_subscription_task(Task, Ctx),+ Coordinator ! {push_delivery_progress, self()},+ run_progress_chunk(Rest, Ctx, Coordinator).++-spec run_subscription_task({integer(), list()}, tuple()) -> ok.+run_subscription_task({UserId, _Subscriptions} = Task, Ctx) ->+ try+ send_subscription_task(Task, Ctx)+ catch+ Class:Reason ->+ bump_counter(delivery_task_failures, 1),+ logger:warning("Push: delivery task raised", #{+ user_id => UserId, class => Class, reason => Reason+ })+ end,+ ok.++-spec await_progress_workers(+ delivery_workers(), non_neg_integer(), non_neg_integer(), {integer(), integer()}+) -> ok.+await_progress_workers(Workers, Unattempted, TaskCount, Ids) when map_size(Workers) =:= 0 ->+ report_unattempted_recipients(Unattempted, TaskCount, Ids);+await_progress_workers(Workers, Unattempted, TaskCount, Ids) ->+ receive+ {push_delivery_progress, Pid} ->+ await_progress_workers(record_task_done(Pid, Workers), Unattempted, TaskCount, Ids);+ {'DOWN', _Ref, process, Pid, normal} ->+ await_progress_workers(maps:remove(Pid, Workers), Unattempted, TaskCount, Ids);+ {'DOWN', _Ref, process, Pid, _Reason} ->+ {Lost, Rest} = discard_crashed_worker(Pid, Workers),+ await_progress_workers(Rest, Unattempted + Lost, TaskCount, Ids)+ after ?DELIVERY_IDLE_TIMEOUT_MS ->+ report_stalled_delivery(Workers, Unattempted, TaskCount, Ids)+ end.++-spec record_task_done(pid(), delivery_workers()) -> delivery_workers().+record_task_done(Pid, Workers) ->+ case Workers of+ #{Pid := {Ref, Left}} -> Workers#{Pid := {Ref, max(0, Left - 1)}};+ _ -> Workers+ end.++-spec discard_crashed_worker(pid(), delivery_workers()) ->+ {non_neg_integer(), delivery_workers()}.+discard_crashed_worker(Pid, Workers) ->+ case maps:take(Pid, Workers) of+ {{_Ref, Left}, Rest} ->+ bump_counter(delivery_worker_crashes, 1),+ bump_counter(delivery_recipients_not_attempted, Left),+ {Left, Rest};+ error ->+ {0, Workers}+ end.++-spec report_unattempted_recipients(+ non_neg_integer(), non_neg_integer(), {integer(), integer()}+) -> ok.+report_unattempted_recipients(0, _TaskCount, _Ids) ->+ ok;+report_unattempted_recipients(Unattempted, TaskCount, {GuildId, MessageId}) ->+ logger:warning("Push: delivery worker died with recipients unattempted", #{+ guild_id => GuildId,+ message_id => MessageId,+ recipients_total => TaskCount,+ recipients_not_attempted => Unattempted+ }).++-spec report_stalled_delivery(+ delivery_workers(), non_neg_integer(), non_neg_integer(), {integer(), integer()}+) -> ok.+report_stalled_delivery(Workers, Unattempted, TaskCount, {GuildId, MessageId}) ->+ Abandoned = map_size(Workers),+ Stranded = abandon_delivery_workers(maps:to_list(Workers), 0),+ bump_counter(delivery_workers_abandoned, Abandoned),+ bump_counter(delivery_recipients_unconfirmed, Stranded),+ logger:error("Push: subscription delivery stalled with recipients unattempted", #{+ guild_id => GuildId,+ message_id => MessageId,+ recipients_total => TaskCount,+ recipients_not_attempted => Unattempted,+ recipients_unconfirmed => Stranded,+ workers_still_running => Abandoned,+ idle_timeout_ms => ?DELIVERY_IDLE_TIMEOUT_MS+ }).++-spec abandon_delivery_workers(+ [{pid(), {reference(), non_neg_integer()}}], non_neg_integer()+) -> non_neg_integer().+abandon_delivery_workers([], Stranded) ->+ Stranded;+abandon_delivery_workers([{_Pid, {Ref, Left}} | Rest], Stranded) ->+ _ = erlang:demonitor(Ref, [flush]),+ abandon_delivery_workers(Rest, Stranded + Left).++-spec delivery_ids(tuple()) -> {integer(), integer()}.+delivery_ids(Ctx) ->+ {+ _MsgData,+ GuildId,+ _ChannelId,+ MessageId,+ _GuildName,+ _ChannelName,+ _MarkdownContext,+ _ContentPreview,+ _BadgeCounts+ } = Ctx,+ {GuildId, MessageId}.++-spec delivery_stats() -> #{atom() => term()}.+delivery_stats() ->+ Counters = maps:from_list([{Key, counter_value(Key)} || Key <- delivery_counter_keys()]),+ Counters#{delivery_last_crash => last_delivery_crash()}.++-spec delivery_counter_keys() -> [atom()].+delivery_counter_keys() ->+ [+ ?CNT_LIVE_WORKERS,+ delivery_workers_spawned,+ delivery_worker_crashes,+ delivery_workers_abandoned,+ delivery_recipients_not_attempted,+ delivery_recipients_unconfirmed,+ delivery_task_failures,+ delivery_budget_exhausted,+ subscription_fetch_calls_failed,+ subscription_fetch_users_dropped,+ badge_fetch_calls_failed,+ badge_fetch_users_defaulted+ ].++-spec counter_value(atom()) -> non_neg_integer().+counter_value(Key) ->+ try ets:lookup(?PUSH_COUNTERS, Key) of+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;+ _ -> 0+ catch+ error:badarg -> 0+ end.++-spec last_delivery_crash() -> term().+last_delivery_crash() ->+ try ets:lookup(?PUSH_COUNTERS, delivery_last_crash) of+ [{delivery_last_crash, Summary}] -> Summary;+ _ -> none+ catch+ error:badarg -> none+ end.++-spec bump_counter(atom(), integer()) -> ok.+bump_counter(Key, Delta) ->+ try+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),+ ok+ catch+ error:badarg -> seed_and_bump_counter(Key, Delta)+ end.++-spec seed_and_bump_counter(atom(), integer()) -> ok.+seed_and_bump_counter(Key, Delta) ->+ try+ _ = ets:insert_new(?PUSH_COUNTERS, {Key, 0}),+ _ = ets:update_counter(?PUSH_COUNTERS, Key, {2, Delta}),+ ok+ catch+ error:badarg -> ok+ end.++-spec delivery_max_workers() -> pos_integer().+delivery_max_workers() ->+ case application:get_env(fluxer_gateway, push_delivery_max_workers, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> ?DEFAULT_DELIVERY_MAX_WORKERS+ end.++-spec delivery_tasks_per_worker() -> pos_integer().+delivery_tasks_per_worker() ->+ case application:get_env(fluxer_gateway, push_delivery_tasks_per_worker, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> ?DEFAULT_DELIVERY_TASKS_PER_WORKER+ end.++-spec subscription_fetch_batch_size() -> pos_integer().+subscription_fetch_batch_size() ->+ case application:get_env(fluxer_gateway, push_subscription_fetch_batch_size, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> ?DEFAULT_SUBSCRIPTION_FETCH_BATCH+ end.++-spec subscription_fetch_budget_ms() -> pos_integer().+subscription_fetch_budget_ms() ->+ case application:get_env(fluxer_gateway, push_subscription_fetch_budget_ms, undefined) of+ Value when is_integer(Value), Value > 0 -> Value;+ _ -> ?DEFAULT_SUBSCRIPTION_FETCH_BUDGET_MS+ end.-spec delivery_concurrency() -> pos_integer().delivery_concurrency() ->@@ -290,35 +671,6 @@ take_subscription_chunk([], _Remaining, Acc) ->take_subscription_chunk([Task | Rest], Remaining, Acc) ->take_subscription_chunk(Rest, Remaining - 1, [Task | Acc]).--spec start_delivery_workers([[{integer(), list()}]], tuple(), #{reference() => true}) ->- #{reference() => true}.-start_delivery_workers([], _Ctx, Refs) ->- Refs;-start_delivery_workers([Chunk | Rest], Ctx, Refs) ->- {_Pid, Ref} = spawn_monitor(fun() ->- send_subscription_task_chunk(Chunk, Ctx)- end),- start_delivery_workers(Rest, Ctx, Refs#{Ref => true}).---spec wait_delivery_workers(#{reference() => true}) -> ok.-wait_delivery_workers(Refs) when map_size(Refs) =:= 0 ->- ok;-wait_delivery_workers(Refs) ->- receive- {'DOWN', Ref, process, _Pid, _Reason} ->- wait_delivery_workers(maps:remove(Ref, Refs))- after ?DELIVERY_WORKER_TIMEOUT_MS ->- logger:warning("Push: timed out waiting for subscription delivery workers", #{- remaining_workers => map_size(Refs)- }),- ok- end.---spec send_subscription_task_chunk([{integer(), list()}], tuple()) -> ok.-send_subscription_task_chunk(Tasks, Ctx) ->- lists:foreach(fun(Task) -> send_subscription_task(Task, Ctx) end, Tasks),- ok.--spec send_subscription_task({integer(), list()}, tuple()) -> ok.send_subscription_task({UserId, Subscriptions}, Ctx) ->push_sender:send_to_user_subscriptions(@@ -473,7 +825,7 @@ send_context(UserId, Ctx) ->-ifdef(TEST).-include_lib("eunit/include/eunit.hrl").-send_subscription_tasks_sends_all_users_with_bounded_workers_test() ->+deliver_subscription_tasks_sends_every_user_its_badge_count_test() ->Self = self(),ok = meck:new(push_sender, [passthrough, no_link]),try@@ -486,7 +838,7 @@ send_subscription_tasks_sends_all_users_with_bounded_workers_test() ->okend),Tasks = [{1, [sub1]}, {2, [sub2]}, {3, [sub3]}],- ?assertEqual(ok, send_subscription_tasks(Tasks, test_send_ctx(), 2)),+ ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())),?assertEqual([{1, [sub1], 5}, {2, [sub2], 6}, {3, [sub3], 7}],lists:sort(collect_sent_subscription_tasks(3, []))@@ -503,54 +855,290 @@ chunk_subscription_tasks_uses_bounded_chunk_size_test() ->chunk_subscription_tasks(Tasks, 2, [])).-chunk_user_ids_uses_bounded_chunk_size_test() ->- ?assertEqual([], chunk_user_ids([], 2, [])),- ?assertEqual([[1, 2]], chunk_user_ids([1, 2], 2, [])),- ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])).+bounded_worker_count_scales_with_recipients_and_stays_bounded_test() ->+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),+ ?assertEqual(3, bounded_worker_count(3, 8, 64)),+ ?assertEqual(8, bounded_worker_count(100, 8, 64)),+ ?assertEqual(20, bounded_worker_count(2500, 8, 64)),+ ?assertEqual(64, bounded_worker_count(10000, 8, 64)),+ ?assertEqual(8, bounded_worker_count(10000, 8, 8)).++bounded_worker_count_treats_the_bound_as_a_real_ceiling_test() ->+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),+ ?assertEqual(4, bounded_worker_count(10000, 8, 4)),+ ?assertEqual(1, bounded_worker_count(10000, 8, 1)),+ ?assertEqual(2, bounded_worker_count(2500, 8, 2)).-add_fetched_user_subscription_task_caches_empty_result_test() ->- push_ets_cache:init(),- push_ets_cache:delete_subscriptions(4242),- ?assertEqual([], add_fetched_user_subscription_task(4242, #{}, [])),- ?assertEqual([], push_ets_cache:get_subscriptions(4242)),- push_ets_cache:delete_subscriptions(4242).+shared_ctx_payload_carries_no_badge_counts_test() ->+ Ctx = test_send_ctx(),+ Shared = shared_ctx_payload(Ctx),+ ?assert(is_binary(Shared)),+ ?assertEqual(#{}, element(?CTX_BADGE_COUNTS, binary_to_term(Shared))),+ ?assert(byte_size(Shared) < byte_size(term_to_binary(Ctx))).++restore_chunk_ctx_rebuilds_the_ctx_with_only_the_chunk_badges_test() ->+ Ctx = test_send_ctx(),+ Shared = shared_ctx_payload(Ctx),+ Chunk = [{2, [sub]}],+ Restored = restore_chunk_ctx(Shared, chunk_badge_counts(Chunk, ctx_badge_counts(Ctx))),+ ?assertEqual(9, tuple_size(Restored)),+ ?assertEqual(#{2 => 6}, element(?CTX_BADGE_COUNTS, Restored)),+ ?assertEqual(element(1, Ctx), element(1, Restored)),+ ?assertEqual(element(7, Ctx), element(7, Restored)),+ ?assertEqual(element(8, Ctx), element(8, Restored)),+ ?assertEqual(#{}, element(?CTX_BADGE_COUNTS, restore_chunk_ctx(Shared, #{}))).++bounded_fanout_isolates_a_raising_task_and_counts_it_test() ->+ Self = self(),+ ok = meck:new(push_sender, [passthrough, no_link]),+ ensure_test_counter_table(),+ Before = counter_value(delivery_task_failures),+ application:set_env(fluxer_gateway, push_delivery_tasks_per_worker, 1),+ try+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) ->+ case UserId of+ 3 -> error(deliberate_task_crash);+ _ -> Self ! {bounded_sent, UserId}+ end,+ ok+ end),+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 5)],+ ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())),+ ?assertEqual([1, 2, 4, 5], lists:sort(collect_bounded_sent(4, []))),+ ?assertEqual(Before + 1, counter_value(delivery_task_failures))+ after+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),+ meck:unload(push_sender)+ end.-fetch_missing_subscriptions_chunks_and_merges_test() ->+bounded_fanout_returns_reserved_slots_to_the_push_budget_test() ->+ ok = meck:new(push_sender, [passthrough, no_link]),+ ensure_test_counter_table(),+ ok = reset_test_counter(?CNT_LIVE_WORKERS),+ try+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(_UserId, _Subs, _Ctx) ->+ ok+ end),+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 40)],+ ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())),+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS))+ after+ meck:unload(push_sender)+ end.++bounded_fanout_releases_reserved_slots_when_workers_are_killed_test() ->+ ok = meck:new(push_sender, [passthrough, no_link]),+ ensure_test_counter_table(),+ ok = reset_test_counter(?CNT_LIVE_WORKERS),+ ok = reset_test_counter(delivery_recipients_not_attempted),+ application:set_env(fluxer_gateway, push_delivery_tasks_per_worker, 1),+ try+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(_UserId, _Subs, _Ctx) ->+ exit(self(), kill)+ end),+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 4)],+ ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())),+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)),+ ?assertEqual(4, counter_value(delivery_recipients_not_attempted))+ after+ application:unset_env(fluxer_gateway, push_delivery_tasks_per_worker),+ meck:unload(push_sender)+ end.++claim_delivery_slots_reserves_before_the_next_job_reads_the_budget_test() ->+ ensure_test_counter_table(),+ ok = reset_test_counter(?CNT_LIVE_WORKERS),+ ?assertEqual(8, claim_delivery_slots(8, 10)),+ ?assertEqual(8, counter_value(?CNT_LIVE_WORKERS)),+ ?assertEqual(2, claim_delivery_slots(8, 10)),+ ?assertEqual(10, counter_value(?CNT_LIVE_WORKERS)),+ ?assertEqual(0, claim_delivery_slots(4, 10)),+ ?assertEqual(10, counter_value(?CNT_LIVE_WORKERS)),+ ok = release_delivery_slots(10),+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)),+ ok = release_delivery_slots(5),+ ?assertEqual(0, counter_value(?CNT_LIVE_WORKERS)).++bounded_fanout_delivers_inline_when_the_push_budget_is_exhausted_test() ->+ Self = self(),+ ok = meck:new(push_sender, [passthrough, no_link]),+ ensure_test_counter_table(),+ ok = reset_test_counter(delivery_budget_exhausted),+ Budget = gateway_http_client:push_max_concurrency(),+ ok = reset_test_counter(?CNT_LIVE_WORKERS),+ _ = claim_delivery_slots(Budget, Budget),+ try+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) ->+ Self ! {bounded_sent, UserId},+ ok+ end),+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 6)],+ ?assertEqual(ok, send_bounded_subscription_tasks(Tasks, test_send_ctx())),+ ?assertEqual(lists:seq(1, 6), lists:sort(collect_bounded_sent(6, []))),+ ?assertEqual(1, counter_value(delivery_budget_exhausted)),+ ?assertEqual(Budget, counter_value(?CNT_LIVE_WORKERS))+ after+ ok = release_delivery_slots(Budget),+ meck:unload(push_sender)+ end.++delivery_stats_exposes_cumulative_loss_without_logs_test() ->+ ensure_test_counter_table(),+ Before = maps:get(delivery_recipients_not_attempted, delivery_stats()),+ ok = bump_counter(delivery_recipients_not_attempted, 3),+ Stats = delivery_stats(),+ ?assertEqual(Before + 3, maps:get(delivery_recipients_not_attempted, Stats)),+ ?assertEqual(+ lists:sort([delivery_last_crash | delivery_counter_keys()]),+ lists:sort(maps:keys(Stats))+ ).++bump_counter_never_creates_the_counter_table_test() ->+ case drop_test_counter_table() of+ ok ->+ ?assertEqual(ok, bump_counter(delivery_recipients_not_attempted, 1)),+ ?assertEqual(0, counter_value(delivery_recipients_not_attempted)),+ ?assertEqual(undefined, ets:info(?PUSH_COUNTERS, name));+ skip ->+ ok+ end.++ensure_test_counter_table() ->+ case ets:info(?PUSH_COUNTERS, name) of+ undefined ->+ _ = ets:new(?PUSH_COUNTERS, [+ named_table, public, set, {write_concurrency, true}+ ]),+ ok;+ _ ->+ ok+ end.++drop_test_counter_table() ->+ case ets:info(?PUSH_COUNTERS, name) of+ undefined ->+ ok;+ _ ->+ try ets:delete(?PUSH_COUNTERS) of+ _ -> ok+ catch+ error:badarg -> skip+ end+ end.++reset_test_counter(Key) ->+ try ets:insert(?PUSH_COUNTERS, {Key, 0}) of+ _ -> ok+ catch+ error:badarg -> ok+ end.++bounded_fanout_delivers_every_recipient_test() ->+ Self = self(),+ ok = meck:new(push_sender, [passthrough, no_link]),+ try+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, _Subs, _Ctx) ->+ Self ! {bounded_sent, UserId},+ ok+ end),+ Tasks = [{UserId, [sub]} || UserId <- lists:seq(1, 200)],+ ?assertEqual(ok, deliver_subscription_tasks(Tasks, test_send_ctx())),+ ?assertEqual(lists:seq(1, 200), lists:sort(collect_bounded_sent(200, [])))+ after+ meck:unload(push_sender)+ end.++chunk_user_ids_uses_bounded_batches_test() ->+ ?assertEqual([[1, 2], [3, 4], [5]], chunk_user_ids([1, 2, 3, 4, 5], 2, [])),+ ?assertEqual([], chunk_user_ids([], 2, [])).++fetch_subscription_batches_stops_after_consecutive_failures_test() ->ok = meck:new(rpc_client, [passthrough, no_link]),try- ok = meck:expect(rpc_client, call, fun subscriptions_chunk_stub/1),- UserIds = lists:seq(1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2),- Expected = #{- <<"1">> => [{chunk, ?SUBSCRIPTION_FETCH_CHUNK_SIZE}],- integer_to_binary(?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1) => [{chunk, 2}]- },- ?assertEqual({UserIds, Expected}, fetch_missing_subscriptions(UserIds)),- ?assertEqual(2, meck:num_calls(rpc_client, call, '_'))+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),+ Batches = [[N] || N <- lists:seq(1, 10)],+ ?assertEqual(+ {[], 10, 10, ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES},+ fetch_subscription_batches(Batches, {[], 0, 0, 0})+ ),+ ?assertEqual(+ ?SUBSCRIPTION_FETCH_MAX_CONSECUTIVE_FAILURES,+ length(meck:history(rpc_client))+ )aftermeck:unload(rpc_client)end.-fetch_missing_subscriptions_keeps_other_chunks_when_one_fails_test() ->+fetch_missing_in_batches_sends_only_successful_batches_test() ->+ Self = self(),+ ok = meck:new(push_sender, [passthrough, no_link]),ok = meck:new(rpc_client, [passthrough, no_link]),+ ok = meck:new(push_ets_cache, [passthrough, no_link]),+ application:set_env(fluxer_gateway, push_subscription_fetch_batch_size, 2),try- ok = meck:expect(rpc_client, call, fun failing_first_chunk_stub/1),- UserIds = lists:seq(1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2),- TailUserIds = [?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1, ?SUBSCRIPTION_FETCH_CHUNK_SIZE + 2],- Expected = #{- integer_to_binary(?SUBSCRIPTION_FETCH_CHUNK_SIZE + 1) => [{chunk, 2}]- },- ?assertEqual({TailUserIds, Expected}, fetch_missing_subscriptions(UserIds))+ ok = meck:expect(push_ets_cache, put_subscriptions, fun(_UserId, _Subs) -> ok end),+ ok = meck:expect(rpc_client, call, fun(#{<<"user_ids">> := Ids}) ->+ case Ids of+ [<<"1">>, <<"2">>] -> {ok, #{<<"1">> => [sub1], <<"2">> => [sub2]}};+ _ -> {error, unavailable}+ end+ end),+ ok = meck:expect(push_sender, send_to_user_subscriptions, fun(UserId, Subs, _Ctx) ->+ Self ! {bounded_sent, {UserId, Subs}},+ ok+ end),+ ?assertEqual(+ ok, fetch_and_send_missing_subscriptions([1, 2, 3, 4], test_send_ctx())+ ),+ ?assertEqual([{1, [sub1]}, {2, [sub2]}], lists:sort(collect_bounded_sent(2, [])))+ after+ application:unset_env(fluxer_gateway, push_subscription_fetch_batch_size),+ meck:unload(push_ets_cache),+ meck:unload(rpc_client),+ meck:unload(push_sender)+ end.++subscription_fetch_failure_counts_dropped_recipients_test() ->+ ok = meck:new(rpc_client, [passthrough, no_link]),+ ensure_test_counter_table(),+ ok = reset_test_counter(subscription_fetch_calls_failed),+ ok = reset_test_counter(subscription_fetch_users_dropped),+ try+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),+ ?assertEqual(+ ok, fetch_and_send_missing_subscriptions([1, 2, 3], test_send_ctx())+ ),+ ?assertEqual(1, counter_value(subscription_fetch_calls_failed)),+ ?assertEqual(3, counter_value(subscription_fetch_users_dropped))aftermeck:unload(rpc_client)end.-subscriptions_chunk_stub(#{<<"user_ids">> := [FirstUserIdBin | _] = UserIdBins}) ->- {ok, #{FirstUserIdBin => [{chunk, length(UserIdBins)}]}}.+clear_notification_fetch_failure_counts_the_dropped_recipient_test() ->+ ok = meck:new(rpc_client, [passthrough, no_link]),+ ensure_test_counter_table(),+ ok = reset_test_counter(subscription_fetch_calls_failed),+ ok = reset_test_counter(subscription_fetch_users_dropped),+ try+ ok = meck:expect(rpc_client, call, fun(_Req) -> {error, unavailable} end),+ ?assertEqual(+ ok, fetch_and_send_clear_notification_from_rpc(7, 8, 9, 0)+ ),+ ?assertEqual(1, counter_value(subscription_fetch_calls_failed)),+ ?assertEqual(1, counter_value(subscription_fetch_users_dropped))+ after+ meck:unload(rpc_client)+ end.-failing_first_chunk_stub(#{<<"user_ids">> := [<<"1">> | _]}) ->- {error, timeout};-failing_first_chunk_stub(Request) ->- subscriptions_chunk_stub(Request).+collect_bounded_sent(0, Acc) ->+ Acc;+collect_bounded_sent(Count, Acc) ->+ receive+ {bounded_sent, Sent} ->+ collect_bounded_sent(Count - 1, [Sent | Acc])+ after 5000 ->+ ?assert(false)+ end.collect_sent_subscription_tasks(0, Acc) ->Acc;diff --git a/fluxer_gateway/src/session/session_init.erl b/fluxer_gateway/src/session/session_init.erlindex 0270fbb43..cd0156745 100644--- a/fluxer_gateway/src/session/session_init.erl+++ b/fluxer_gateway/src/session/session_init.erl@@ -63,9 +63,21 @@ normalize_seq(_) -> 0.-spec normalize_buffer(term()) -> [map()].normalize_buffer(Buffer) when is_list(Buffer) ->lists:filtermap(fun normalize_buffer_event/1, Buffer);+normalize_buffer(Buffer) when is_map(Buffer) ->+ lists:filtermap(fun normalize_buffer_event/1, deque_to_list(Buffer));normalize_buffer(_) ->[].+-spec deque_to_list(map()) -> [term()].+deque_to_list(Buffer) ->+ try limited_deque:to_list(eqwalizer:dynamic_cast(Buffer)) of+ List when is_list(List) -> List;+ _ -> []+ catch+ error:_Reason -> [];+ exit:_Reason -> []+ end.+-spec normalize_buffer_event(term()) -> {true, map()} | false.normalize_buffer_event(Event) when is_map(Event) ->case maps:get(seq, Event, -1) of@@ -75,19 +87,43 @@ normalize_buffer_event(Event) when is_map(Event) ->normalize_buffer_event(_) ->false.+-spec init_buffer_bytes(term(), [map()]) -> non_neg_integer().+init_buffer_bytes(_Raw, []) ->+ 0;+init_buffer_bytes(Raw, Buffer) when is_map(Raw) ->+ restored_deque_bytes(Raw, Buffer);+init_buffer_bytes(_Raw, Buffer) ->+ replay_buffer_bytes(Buffer).++-spec restored_deque_bytes(map(), [map()]) -> non_neg_integer().+restored_deque_bytes(Raw, Buffer) ->+ try limited_deque:bytes(eqwalizer:dynamic_cast(Raw)) of+ Bytes when is_integer(Bytes), Bytes >= 0 -> Bytes;+ _ -> replay_buffer_bytes(Buffer)+ catch+ error:_Reason -> replay_buffer_bytes(Buffer);+ exit:_Reason -> replay_buffer_bytes(Buffer)+ end.+-spec replay_buffer_bytes([map()]) -> non_neg_integer().replay_buffer_bytes(Buffer) ->WordSize = erlang:system_info(wordsize),- trunc(- lists:foldl(- fun(Event, Acc) ->- Acc + erts_debug:flat_size(Event) * WordSize- end,- 0,- Buffer- )+ lists:foldl(+ fun(Event, Acc) -> Acc + erts_debug:flat_size(Event) * WordSize end,+ 0,+ Buffer).+-spec replay_payload_bytes([map()]) -> non_neg_integer().+replay_payload_bytes(Buffer) ->+ lists:foldl(fun(Event, Acc) -> Acc + entry_payload_bytes(Event) end, 0, Buffer).++-spec entry_payload_bytes(term()) -> non_neg_integer().+entry_payload_bytes(#{data := {pre_encoded, Payload}}) when is_binary(Payload) ->+ byte_size(Payload);+entry_payload_bytes(_Event) ->+ 0.+-spec build_ignored_events_map(term()) -> #{binary() => true}.build_ignored_events_map(Events) when is_list(Events) ->maps:from_list([{Event, true} || Event <- Events, is_binary(Event)]);@@ -193,7 +229,8 @@ extract_core_fields(token_hash => TokenHash,auth_session_id_hash => AuthSessionIdHash,buffer => Buffer,- buffer_bytes => replay_buffer_bytes(Buffer),+ buffer_bytes => init_buffer_bytes(maps:get(buffer, D, []), Buffer),+ replay_payload_bytes => replay_payload_bytes(Buffer),seq => Seq,ack_seq => AckSeq,properties => Properties,@@ -285,7 +322,7 @@ schedule_timers(#{bot := Bot, guilds := GuildsMap}) ->init_prewarm_guilds(Bot, GuildIds),lists:foreach(fun(Gid) -> self() ! {guild_connect, Gid, 0} end, GuildIds),erlang:send_after(5000, self(), premature_readiness),- erlang:send_after(200, self(), enable_presence_updates),+ erlang:send_after(10000, self(), enable_presence_updates),erlang:send_after(60000, self(), check_ack_lag),ok.@@ -468,4 +505,68 @@ normalize_buffer_filters_invalid_entries_test() ->?assertEqual([#{seq => 1}], normalize_buffer(Buffer)),?assertEqual([], normalize_buffer(undefined)).+normalize_buffer_accepts_deque_test() ->+ Deque = test_deque([#{seq => 1}, #{seq => 2}]),+ ?assertEqual([#{seq => 1}, #{seq => 2}], normalize_buffer(Deque)).++normalize_buffer_filters_invalid_deque_entries_test() ->+ Deque = test_deque([#{seq => 1}, #{seq => -1}, #{no_seq => true}, <<"bad">>]),+ ?assertEqual([#{seq => 1}], normalize_buffer(Deque)).++normalize_buffer_ignores_non_deque_map_test() ->+ ?assertEqual([], normalize_buffer(#{seq => 1})),+ ?assertEqual([], normalize_buffer(#{})),+ ?assertEqual([], normalize_buffer(undefined)).++build_state_preserves_transferred_deque_buffer_test() ->+ Data = transferred_session_data(test_deque([#{seq => 13}, #{seq => 14}])),+ State = build_state(Data),+ ?assertEqual([#{seq => 13}, #{seq => 14}], maps:get(buffer, State)),+ ?assertEqual(+ replay_buffer_bytes([#{seq => 13}, #{seq => 14}]),+ maps:get(buffer_bytes, State)+ ),+ ?assertEqual(14, maps:get(seq, State)),+ ?assertEqual(12, maps:get(ack_seq, State)).++build_state_reuses_deque_byte_total_test() ->+ Events = [#{seq => 21}, #{seq => 22}],+ Data = transferred_session_data(test_deque(Events)),+ State = build_state(Data),+ ?assertEqual(limited_deque:bytes(test_deque(Events)), maps:get(buffer_bytes, State)),+ ?assertEqual(replay_buffer_bytes(Events), maps:get(buffer_bytes, State)).++replay_buffer_bytes_agrees_with_deque_accounting_test() ->+ Entries = [+ #{seq => 1, event => message_create, data => #{<<"content">> => <<"hi">>}},+ #{seq => 2, event => message_create, data => {pre_encoded, binary:copy(<<"x">>, 70000)}}+ ],+ lists:foreach(+ fun(Entry) ->+ ?assertEqual(+ limited_deque:bytes(limited_deque:from_list([Entry], 4096, 0)),+ replay_buffer_bytes([Entry])+ )+ end,+ Entries+ ).++build_state_walks_list_buffer_bytes_test() ->+ Events = [#{seq => 13}, #{seq => 14}],+ Data = transferred_session_data(Events),+ State = build_state(Data),+ ?assertEqual(replay_buffer_bytes(Events), maps:get(buffer_bytes, State)).++build_state_zeroes_bytes_for_non_deque_map_buffer_test() ->+ Data = transferred_session_data(#{seq => 13}),+ State = build_state(Data),+ ?assertEqual([], maps:get(buffer, State)),+ ?assertEqual(0, maps:get(buffer_bytes, State)).++transferred_session_data(Buffer) ->+ (base_session_data(#{}))#{buffer => Buffer, seq => 14, ack_seq => 12}.++test_deque(Events) ->+ limited_deque:from_list(Events, 4096, 16777216).+-endif.diff --git a/fluxer_gateway/src/session/session_lifecycle.erl b/fluxer_gateway/src/session/session_lifecycle.erlindex 8d77ae453..54f2577dc 100644--- a/fluxer_gateway/src/session/session_lifecycle.erl+++ b/fluxer_gateway/src/session/session_lifecycle.erl@@ -34,6 +34,9 @@-type status() :: session:status().-type voice_state_entry() :: {binary(), map()}.+-define(MAX_INLINE_RESUME_REPLAY, 256).+-define(MAX_TRANSFER_PRE_ENCODED_BYTES, 4194304).+-spec handle_terminate_call([binary()], session_state()) ->{stop, normal, terminated, session_state()} | {reply, ignored, session_state()}.handle_terminate_call(Hashes, #{auth_session_id_hash := AuthHash} = State) ->@@ -287,10 +290,25 @@ handle_heartbeat_ack(Seq, #{buffer := Buffer} = State) ->true -> session_init:replay_buffer_bytes(NewBuffer);false -> limited_deque:bytes(NewBuffer)end,- NewState0 = State#{ack_seq => AckedSeq, buffer => NewBuffer, buffer_bytes => NewBytes},+ NewState0 = ack_state(State, AckedSeq, NewBuffer, NewBytes),NewState = session_connection_guild:repair_stalled_guild_connects(NewState0),{reply, true, NewState}.+-spec ack_state(session_state(), seq(), term(), non_neg_integer()) -> session_state().+ack_state(State, AckedSeq, NewBuffer, NewBytes) ->+ Base = State#{ack_seq => AckedSeq, buffer => NewBuffer, buffer_bytes => NewBytes},+ Base#{replay_payload_bytes => buffer_payload_bytes(NewBuffer)}.++-spec buffer_payload_bytes(term()) -> non_neg_integer().+buffer_payload_bytes(Buffer) when is_list(Buffer) ->+ lists:foldl(fun(Entry, Acc) -> Acc + entry_payload_bytes(Entry) end, 0, Buffer);+buffer_payload_bytes(Buffer) ->+ try limited_deque:to_list(Buffer) of+ Entries -> buffer_payload_bytes(Entries)+ catch+ _:_ -> 0+ end.+-spec drop_acked_buffer(seq(), eqwalizer:dynamic(limited_deque:deque() | [map()])) -> eqwalizer:dynamic(limited_deque:deque() | [map()]).@@ -325,11 +343,41 @@ handle_resume(Seq, SocketPid, #{seq := CurrentSeq} = State) ->MissedEvents = events_after_seq(Seq, BufferList),NewState0 = cancel_offline_timer(cancel_resume_timer(State)),NewState1 = replace_socket(SocketPid, NewState0),+ ReplyEvents = replay_missed_events_inline(MissedEvents, SocketPid),NewState = NewState1#{status => ResumeStatus, resume_status => ResumeStatus},NewState2 = ensure_presence_attached_on_resume(NewState, SessionId, ResumeStatus, Afk, Mobile),- {reply, {ok, MissedEvents, CurrentSeq}, NewState2}.+ {reply, {ok, ReplyEvents, CurrentSeq}, NewState2}.++-spec replay_missed_events_inline([map()], pid()) -> [map()].+replay_missed_events_inline(MissedEvents, SocketPid) ->+ case inline_replay_allowed(MissedEvents) of+ true ->+ lists:foreach(fun(Event) -> send_missed_event(SocketPid, Event) end, MissedEvents),+ [];+ false ->+ MissedEvents+ end.++-spec inline_replay_allowed([map()]) -> boolean().+inline_replay_allowed(MissedEvents) ->+ within_inline_replay_limit(MissedEvents, ?MAX_INLINE_RESUME_REPLAY).++-spec within_inline_replay_limit([map()], non_neg_integer()) -> boolean().+within_inline_replay_limit([], _Remaining) ->+ true;+within_inline_replay_limit(_MissedEvents, 0) ->+ false;+within_inline_replay_limit([_Event | Rest], Remaining) ->+ within_inline_replay_limit(Rest, Remaining - 1).++-spec send_missed_event(pid(), map()) -> ok.+send_missed_event(SocketPid, #{event := Event, data := Data, seq := EventSeq}) ->+ SocketPid ! {dispatch, Event, guild_data_wire:payload(Data), EventSeq},+ ok;+send_missed_event(_SocketPid, _Event) ->+ ok.-spec ensure_presence_attached_on_resume(session_state(), session_id(), status(), boolean(), boolean()@@ -509,7 +557,6 @@ serialize_state(State) ->resume_status => maps:get(resume_status, State, maps:get(status, State)),afk => maps:get(afk, State),mobile => maps:get(mobile, State),- buffer => maps:get(buffer, State),ready => maps:get(ready, State),bot => maps:get(bot, State, false),shard => maps:get(shard, State, undefined),@@ -553,20 +600,340 @@ serialize_transfer_identity(State) ->debounce_reactions => maps:get(debounce_reactions, State, false)}.--spec transfer_buffer(eqwalizer:dynamic(limited_deque:deque() | [map()])) -> [term()].-transfer_buffer(Buffer) when is_list(Buffer) -> Buffer;-transfer_buffer(Buffer) -> limited_deque:to_list(Buffer).--spec serialize_transfer_runtime(session_state()) -> map().serialize_transfer_runtime(State) ->+ {Buffer, AckSeq} = transfer_buffer(+ maps:get(buffer, State, []), maps:get(ack_seq, State, 0)+ ),#{channels => maps:get(channels, State, #{}),relationships => maps:get(relationships, State, #{}),seq => maps:get(seq, State, 0),- ack_seq => maps:get(ack_seq, State, 0),- buffer => transfer_buffer(maps:get(buffer, State, [])),+ ack_seq => AckSeq,+ buffer => Buffer,collected_guild_states => maps:get(collected_guild_states, State, []),collected_sessions => maps:get(collected_sessions, State, []),collected_presences => maps:get(collected_presences, State, []),guild_subscription_state => maps:get(guild_subscription_state, State, #{})}.++%% Always normalises to a list before trimming: a limited_deque left in the transfer+%% map does not survive the handoff to the receiving node's session_init.+-spec transfer_buffer(+ eqwalizer:dynamic(limited_deque:deque() | [map()]), seq()+) -> {[map()], seq()}.+transfer_buffer(Buffer, AckSeq) when is_list(Buffer) ->+ trim_transfer_list(Buffer, transfer_payload_bytes(Buffer), AckSeq);+transfer_buffer(Buffer, AckSeq) ->+ ListBuffer = limited_deque:to_list(Buffer),+ trim_transfer_list(ListBuffer, transfer_payload_bytes(ListBuffer), AckSeq).++-spec trim_transfer_list([map()], non_neg_integer(), seq()) -> {[map()], seq()}.+trim_transfer_list(Buffer, Bytes, AckSeq) when Bytes =< ?MAX_TRANSFER_PRE_ENCODED_BYTES ->+ {Buffer, AckSeq};+trim_transfer_list([], _Bytes, AckSeq) ->+ {[], AckSeq};+trim_transfer_list([Dropped | Rest], Bytes, AckSeq) ->+ trim_transfer_list(+ Rest,+ max(0, Bytes - entry_payload_bytes(Dropped)),+ dropped_ack_seq(Dropped, AckSeq)+ ).++-spec transfer_payload_bytes([term()]) -> non_neg_integer().+transfer_payload_bytes(Entries) ->+ lists:foldl(fun(Entry, Acc) -> Acc + entry_payload_bytes(Entry) end, 0, Entries).++-spec entry_payload_bytes(term()) -> non_neg_integer().+entry_payload_bytes(#{data := {pre_encoded, Payload}}) when is_binary(Payload) ->+ byte_size(Payload);+entry_payload_bytes(_Entry) ->+ 0.++-spec dropped_ack_seq(term(), seq()) -> seq().+dropped_ack_seq(#{seq := Seq}, AckSeq) when is_integer(Seq), Seq > AckSeq ->+ Seq;+dropped_ack_seq(_Dropped, AckSeq) ->+ AckSeq.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++resume_flag_state(Overrides) ->+ maps:merge(+ #{+ id => <<"session-resume-flag-test">>,+ status => online,+ afk => false,+ mobile => false,+ presence_pid => undefined,+ presence_mref => undefined,+ socket_pid => undefined,+ socket_mref => undefined,+ resume_timer => undefined,+ offline_timer => undefined,+ ack_seq => 0,+ buffer => [],+ seq => 0+ },+ Overrides+ ).++buffered_event(Seq) ->+ #{seq => Seq, event => message_create, data => #{<<"id">> => Seq}}.++with_env(Key, Value, Fun) ->+ Previous = application:get_env(fluxer_gateway, Key),+ application:set_env(fluxer_gateway, Key, Value),+ try+ Fun()+ after+ restore_env(Key, Previous)+ end.++restore_env(Key, {ok, Previous}) ->+ application:set_env(fluxer_gateway, Key, Previous);+restore_env(Key, undefined) ->+ application:unset_env(fluxer_gateway, Key).++next_dispatch() ->+ receive+ {dispatch, _Event, _Data, _Seq} = Msg -> Msg+ after 200 -> no_dispatch+ end.++drain_dispatches() ->+ receive+ {dispatch, _Event, _Data, _Seq} -> drain_dispatches()+ after 0 -> ok+ end.++count_dispatches() ->+ count_dispatches(0).++count_dispatches(Count) ->+ receive+ {dispatch, _Event, _Data, _Seq} -> count_dispatches(Count + 1)+ after 0 -> Count+ end.++inline_replay_sends_backlog_and_replies_empty_test() ->+ drain_dispatches(),+ State0 = resume_flag_state(#{seq => 2, buffer => [buffered_event(1), buffered_event(2)]}),+ {reply, {ok, [], 2}, _State1} = handle_resume(0, self(), State0),+ First = next_dispatch(),+ Second = next_dispatch(),+ ?assertEqual(+ {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 1}), 1}, First+ ),+ ?assertEqual(+ {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 2}), 2}, Second+ ),+ ?assertEqual(no_dispatch, next_dispatch()).++inline_replay_sends_wire_payload_test() ->+ drain_dispatches(),+ State0 = resume_flag_state(#{seq => 1, buffer => [buffered_event(1)]}),+ {reply, {ok, [], 1}, _State1} = handle_resume(0, self(), State0),+ {dispatch, message_create, Data, 1} = next_dispatch(),+ ?assertEqual(#{<<"id">> => <<"1">>}, Data).++inline_replay_skips_malformed_buffer_entry_test() ->+ drain_dispatches(),+ State0 = resume_flag_state(#{seq => 2, buffer => [#{seq => 1}, buffered_event(2)]}),+ {reply, {ok, [], 2}, _State1} = handle_resume(0, self(), State0),+ ?assertEqual(+ {dispatch, message_create, guild_data_wire:payload(#{<<"id">> => 2}), 2},+ next_dispatch()+ ),+ ?assertEqual(no_dispatch, next_dispatch()).++inline_replay_sends_nothing_when_backlog_empty_test() ->+ drain_dispatches(),+ State0 = resume_flag_state(#{seq => 1, ack_seq => 1, buffer => [buffered_event(1)]}),+ {reply, {ok, [], 1}, _State1} = handle_resume(1, self(), State0),+ ?assertEqual(no_dispatch, next_dispatch()).++inline_replay_bounded_batch_is_sent_inline_test() ->+ drain_dispatches(),+ Events = [buffered_event(Seq) || Seq <- lists:seq(1, ?MAX_INLINE_RESUME_REPLAY)],+ State0 = resume_flag_state(#{seq => ?MAX_INLINE_RESUME_REPLAY, buffer => Events}),+ {reply, {ok, [], ?MAX_INLINE_RESUME_REPLAY}, _State1} = handle_resume(0, self(), State0),+ ?assertEqual(?MAX_INLINE_RESUME_REPLAY, count_dispatches()).++inline_replay_oversized_batch_falls_back_to_reply_test() ->+ drain_dispatches(),+ Over = ?MAX_INLINE_RESUME_REPLAY + 1,+ Events = [buffered_event(Seq) || Seq <- lists:seq(1, Over)],+ State0 = resume_flag_state(#{seq => Over, buffer => Events}),+ {reply, {ok, Missed, Over}, _State1} = handle_resume(0, self(), State0),+ ?assertEqual(Over, length(Missed)),+ ?assertEqual(no_dispatch, next_dispatch()).++inline_replay_oversized_batch_still_attaches_socket_test() ->+ drain_dispatches(),+ Over = ?MAX_INLINE_RESUME_REPLAY + 1,+ Events = [buffered_event(Seq) || Seq <- lists:seq(1, Over)],+ State0 = resume_flag_state(#{seq => Over, buffer => Events}),+ {reply, {ok, _Missed, Over}, State1} = handle_resume(0, self(), State0),+ ?assertEqual(self(), maps:get(socket_pid, State1)).++pre_encoded_buffered_event(Seq) ->+ #{seq => Seq, event => message_create, data => {pre_encoded, <<"{}">>}}.++heartbeat_ack_trims_pre_encoded_entries_test() ->+ Buffer = limited_deque:from_list(+ [pre_encoded_buffered_event(1), pre_encoded_buffered_event(2)], 4096, 16777216+ ),+ State0 = resume_flag_state(#{seq => 2, buffer => Buffer}),+ {reply, true, State1} = handle_heartbeat_ack(1, State0),+ Remaining = limited_deque:to_list(maps:get(buffer, State1)),+ ?assertEqual([2], [maps:get(seq, Event) || Event <- Remaining]).++heartbeat_ack_keeps_trimming_pre_encoded_head_test() ->+ Buffer = limited_deque:from_list(+ [pre_encoded_buffered_event(Seq) || Seq <- lists:seq(1, 4)], 4096, 16777216+ ),+ State0 = resume_flag_state(#{seq => 4, buffer => Buffer}),+ {reply, true, State1} = handle_heartbeat_ack(4, State0),+ ?assertEqual(0, limited_deque:size(maps:get(buffer, State1))).++inline_replay_sends_pre_encoded_entry_unchanged_test() ->+ drain_dispatches(),+ Entry = pre_encoded_buffered_event(1),+ State0 = resume_flag_state(#{seq => 1, buffer => [Entry]}),+ {reply, {ok, [], 1}, _State1} = handle_resume(0, self(), State0),+ ?assertEqual({dispatch, message_create, maps:get(data, Entry), 1}, next_dispatch()).++oversized_replay_returns_pre_encoded_entries_unchanged_test() ->+ drain_dispatches(),+ Over = ?MAX_INLINE_RESUME_REPLAY + 1,+ Entries = [pre_encoded_buffered_event(Seq) || Seq <- lists:seq(1, Over)],+ State0 = resume_flag_state(#{seq => Over, buffer => Entries}),+ {reply, {ok, Missed, Over}, _State1} = handle_resume(0, self(), State0),+ ?assertEqual(Entries, Missed),+ ?assertEqual(no_dispatch, next_dispatch()).++resume_ignores_withdrawn_buffer_floor_env_test() ->+ drain_dispatches(),+ State0 = resume_flag_state(#{seq => 5, buffer => [buffered_event(4), buffered_event(5)]}),+ with_env(session_resume_buffer_floor, true, fun() ->+ {reply, {ok, [], 5}, _State1} = handle_resume(2, self(), State0),+ ?assertMatch({dispatch, _, _, 4}, next_dispatch()),+ ?assertMatch({dispatch, _, _, 5}, next_dispatch()),+ ?assertEqual(no_dispatch, next_dispatch())+ end).++serialize_state_omits_replay_buffer_test() ->+ State = resume_flag_state(#{+ user_id => 12345,+ user_data => #{},+ version => 9,+ properties => #{},+ ready => undefined,+ guilds => #{},+ collected_guild_states => [],+ collected_sessions => [],+ buffer => [buffered_event(1)]+ }),+ Serialized = serialize_state(State),+ ?assertEqual(false, maps:is_key(buffer, Serialized)),+ ?assertEqual(false, maps:is_key(buffer_bytes, Serialized)),+ ?assertEqual(0, maps:get(seq, Serialized)).++transfer_entry(Seq, PayloadBytes) ->+ #{+ seq => Seq,+ event => message_create,+ data => {pre_encoded, binary:copy(<<"x">>, PayloadBytes)}+ }.++transfer_deque(Entries) ->+ limited_deque:from_list(Entries, 4096, 16777216).++transfer_runtime_state(Buffer, AckSeq) ->+ #{+ channels => #{},+ relationships => #{},+ seq => 4,+ ack_seq => AckSeq,+ buffer => Buffer,+ collected_guild_states => [],+ collected_sessions => [],+ collected_presences => [],+ guild_subscription_state => #{}+ }.++oversized_transfer_entries() ->+ [transfer_entry(Seq, 1500000) || Seq <- lists:seq(1, 4)].++transfer_buffer_is_bounded_test() ->+ Deque = transfer_deque(oversized_transfer_entries()),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),+ Entries = maps:get(buffer, Runtime),+ ?assert(transfer_payload_bytes(Entries) =< ?MAX_TRANSFER_PRE_ENCODED_BYTES),+ ?assertEqual([3, 4], [maps:get(seq, Entry) || Entry <- Entries]),+ ?assertEqual(2, maps:get(ack_seq, Runtime)).++transfer_buffer_admits_one_full_size_event_test() ->+ Single = transfer_entry(1, 2097152),+ Deque = transfer_deque([Single]),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),+ ?assertEqual([Single], maps:get(buffer, Runtime)),+ ?assertEqual(0, maps:get(ack_seq, Runtime)).++transfer_buffer_bound_keeps_entry_representation_test() ->+ Deque = transfer_deque(oversized_transfer_entries()),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),+ Entries = maps:get(buffer, Runtime),+ ?assertEqual(Entries, session_init:normalize_buffer(Entries)),+ ?assertEqual([transfer_entry(3, 1500000), transfer_entry(4, 1500000)], Entries).++transfer_buffer_bound_leaves_small_buffers_alone_test() ->+ Deque = transfer_deque([buffered_event(1), buffered_event(2)]),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),+ ?assertEqual(limited_deque:to_list(Deque), maps:get(buffer, Runtime)),+ ?assertEqual(0, maps:get(ack_seq, Runtime)).++fat_map_entry(Seq) ->+ #{+ seq => Seq,+ event => message_create,+ data => #{<<"content">> => lists:duplicate(200000, $x)}+ }.++transfer_buffer_bound_never_truncates_map_entries_test() ->+ Entries = [fat_map_entry(Seq) || Seq <- lists:seq(1, 2)],+ Deque = transfer_deque(Entries),+ ?assert(limited_deque:bytes(Deque) > ?MAX_TRANSFER_PRE_ENCODED_BYTES),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),+ ?assertEqual(limited_deque:to_list(Deque), maps:get(buffer, Runtime)),+ ?assertEqual(0, maps:get(ack_seq, Runtime)).++transfer_buffer_bound_never_lowers_ack_seq_test() ->+ Deque = transfer_deque([buffered_event(9)]),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 8)),+ ?assertEqual(8, maps:get(ack_seq, Runtime)).++transfer_buffer_bounds_list_buffers_test() ->+ Entries = oversized_transfer_entries(),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Entries, 0)),+ ?assertEqual([3, 4], [maps:get(seq, Entry) || Entry <- maps:get(buffer, Runtime)]),+ ?assertEqual(2, maps:get(ack_seq, Runtime)).++transfer_buffer_bound_makes_stale_resume_invalid_test() ->+ Deque = transfer_deque(oversized_transfer_entries()),+ Runtime = serialize_transfer_runtime(transfer_runtime_state(Deque, 0)),+ Restored = resume_flag_state(#{+ seq => 4,+ ack_seq => maps:get(ack_seq, Runtime),+ buffer => maps:get(buffer, Runtime)+ }),+ ?assertMatch({reply, invalid_seq, _}, handle_resume(1, self(), Restored)),+ drain_dispatches(),+ {reply, {ok, [], 4}, _} = handle_resume(2, self(), Restored),+ ?assertMatch({dispatch, _, _, 3}, next_dispatch()),+ ?assertMatch({dispatch, _, _, 4}, next_dispatch()),+ ?assertEqual(no_dispatch, next_dispatch()).++-endif.diff --git a/fluxer_gateway/src/session/session_ready_dispatch.erl b/fluxer_gateway/src/session/session_ready_dispatch.erlindex 910c564b6..905c30d9a 100644--- a/fluxer_gateway/src/session/session_ready_dispatch.erl+++ b/fluxer_gateway/src/session/session_ready_dispatch.erl@@ -70,7 +70,8 @@ dispatch_ready_to_socket(State) ->IsBot, CollectedGuilds, Guilds, StateAfterReady),schedule_call_creates(StateAfterGuilds, SessionId),- FinalState = StateAfterGuilds#{+ StateAfterPresences = release_pending_presences(StateAfterGuilds, CollectedPresences),+ FinalState = StateAfterPresences#{ready => undefined,collected_guild_states => [],collected_sessions => [],@@ -79,6 +80,65 @@ dispatch_ready_to_socket(State) ->erlang:garbage_collect(),{noreply, FinalState}.+-spec release_pending_presences(session_state(), [map()]) -> session_state().+release_pending_presences(State, CollectedPresences) ->+ Covered = covered_presence_ids(CollectedPresences),+ Pruned = drop_covered_pending_presences(State, Covered),+ Flushed = session_dispatch:flush_all_pending_presences(Pruned),+ Flushed#{suppress_presence_updates => false}.++-spec covered_presence_ids([map()]) -> #{integer() => true}.+covered_presence_ids(CollectedPresences) ->+ lists:foldl(+ fun(Presence, Acc) ->+ case presence_user_id(Presence) of+ undefined -> Acc;+ UserId -> Acc#{UserId => true}+ end+ end,+ #{},+ CollectedPresences+ ).++-spec drop_covered_pending_presences(session_state(), #{integer() => true}) -> session_state().+drop_covered_pending_presences(State, Covered) when map_size(Covered) =:= 0 ->+ State;+drop_covered_pending_presences(State, Covered) ->+ case maps:get(pending_presences, State, undefined) of+ undefined ->+ State;+ Pending ->+ State#{pending_presences => filter_pending_presences(Pending, Covered)}+ end.++-spec filter_pending_presences(term(), #{integer() => true}) -> term().+filter_pending_presences(Pending, Covered) ->+ Keep = fun(Entry) -> not covered_entry(Entry, Covered) end,+ try queue:filter(Keep, Pending) of+ Filtered -> Filtered+ catch+ error:_ when is_list(Pending) -> lists:filter(Keep, Pending);+ error:_ -> Pending+ end.++-spec covered_entry(term(), #{integer() => true}) -> boolean().+covered_entry(#{user_id := UserId}, Covered) when is_integer(UserId) ->+ maps:is_key(UserId, Covered);+covered_entry(_Entry, _Covered) ->+ false.++-spec presence_user_id(term()) -> integer() | undefined.+presence_user_id(Presence) when is_map(Presence) ->+ User = maps:get(<<"user">>, Presence, #{}),+ presence_id(User);+presence_user_id(_Presence) ->+ undefined.++-spec presence_id(term()) -> integer() | undefined.+presence_id(User) when is_map(User) ->+ snowflake_id:parse_maybe(maps:get(<<"id">>, User, undefined));+presence_id(_User) ->+ undefined.-spec is_staff_session(session_state()) -> boolean().is_staff_session(#{is_staff := true}) ->true;diff --git a/fluxer_gateway/src/utils/custom_status_expiry.erl b/fluxer_gateway/src/utils/custom_status_expiry.erlnew file mode 100644index 000000000..b0f47974a--- /dev/null+++ b/fluxer_gateway/src/utils/custom_status_expiry.erl@@ -0,0 +1,406 @@+%% SPDX-License-Identifier: AGPL-3.0-or-later++-module(custom_status_expiry).+-typing([eqwalizer]).++-export([+ enabled/0,+ clear_if_expired/1,+ clear_if_expired/2,+ remaining_ms/2,+ next_wakeup_ms/1,+ reconcile_message/0,+ repair/1+]).++-export_type([custom_status/0]).++-type custom_status() :: map() | null.+-type user_id() :: integer().+-type wakeup() :: {ok, pos_integer()} | none.+-type offset() :: {ok, integer()} | none.++-define(ENABLED_KEY, custom_status_expiry_enabled).+-define(MAX_REPAIR_WINDOW_SECONDS, 86400).+-define(MAX_TIMER_MS, 86400000).+-define(WAKEUP_JITTER_MS, 5000).++-spec enabled() -> boolean().+enabled() ->+ application:get_env(fluxer_gateway, ?ENABLED_KEY, false) =:= true.++-spec clear_if_expired(term()) -> custom_status().+clear_if_expired(CustomStatus) ->+ clear_when_enabled(enabled(), CustomStatus).++-spec clear_when_enabled(boolean(), term()) -> custom_status().+clear_when_enabled(true, CustomStatus) ->+ clear_if_expired(CustomStatus, erlang:system_time(millisecond));+clear_when_enabled(false, CustomStatus) when is_map(CustomStatus) ->+ CustomStatus;+clear_when_enabled(false, _CustomStatus) ->+ null.++-spec clear_if_expired(term(), term()) -> custom_status().+clear_if_expired(CustomStatus, NowMs) when is_map(CustomStatus), is_integer(NowMs) ->+ keep_or_clear(expired(CustomStatus, NowMs), CustomStatus);+clear_if_expired(CustomStatus, _NowMs) when is_map(CustomStatus) ->+ CustomStatus;+clear_if_expired(_CustomStatus, _NowMs) ->+ null.++-spec keep_or_clear(boolean(), map()) -> custom_status().+keep_or_clear(true, _CustomStatus) -> null;+keep_or_clear(false, CustomStatus) -> CustomStatus.++-spec expired(map(), integer()) -> boolean().+expired(CustomStatus, NowMs) ->+ case remaining_ms(CustomStatus, NowMs) of+ {ok, RemainingMs} -> RemainingMs =< 0;+ none -> false+ end.++-spec remaining_ms(term(), term()) -> offset().+remaining_ms(CustomStatus, NowMs) when is_map(CustomStatus), is_integer(NowMs) ->+ offset_from(expires_at_ms(maps:get(<<"expires_at">>, CustomStatus, undefined)), NowMs);+remaining_ms(_CustomStatus, _NowMs) ->+ none.++-spec offset_from(offset(), integer()) -> offset().+offset_from({ok, ExpiryMs}, NowMs) -> {ok, ExpiryMs - NowMs};+offset_from(none, _NowMs) -> none.++-spec expires_at_ms(term()) -> offset().+expires_at_ms(Value) when is_binary(Value) ->+ parse_rfc3339(binary_to_list(Value));+expires_at_ms(Value) when is_list(Value) ->+ parse_rfc3339(Value);+expires_at_ms(_Value) ->+ none.++-spec parse_rfc3339(term()) -> offset().+parse_rfc3339(Chars) ->+ try calendar:rfc3339_to_system_time(Chars, [{unit, millisecond}]) of+ Ms when is_integer(Ms) -> {ok, Ms};+ _Other -> none+ catch+ _Class:_Reason -> none+ end.++-spec next_wakeup_ms(term()) -> wakeup().+next_wakeup_ms(CustomStatus) ->+ next_wakeup_ms(enabled(), CustomStatus).++-spec next_wakeup_ms(boolean(), term()) -> wakeup().+next_wakeup_ms(false, _CustomStatus) ->+ none;+next_wakeup_ms(true, CustomStatus) ->+ wakeup_delay(remaining_ms(CustomStatus, erlang:system_time(millisecond))).++-spec wakeup_delay(offset()) -> wakeup().+wakeup_delay(none) ->+ none;+wakeup_delay({ok, RemainingMs}) when RemainingMs =< 0 ->+ none;+wakeup_delay({ok, RemainingMs}) when RemainingMs > ?MAX_TIMER_MS ->+ {ok, jittered(?MAX_TIMER_MS)};+wakeup_delay({ok, RemainingMs}) ->+ {ok, jittered(RemainingMs)}.++-spec jittered(pos_integer()) -> pos_integer().+jittered(DelayMs) ->+ DelayMs + rand:uniform(?WAKEUP_JITTER_MS).++-spec reconcile_message() -> {'$gen_cast', reconcile_flattened_presence}.+reconcile_message() ->+ {'$gen_cast', reconcile_flattened_presence}.++-spec repair(term()) -> map().+repair(WindowSeconds) when+ is_integer(WindowSeconds), WindowSeconds >= 0, WindowSeconds =< ?MAX_REPAIR_WINDOW_SECONDS+->+ Considered = local_presence_user_ids(),+ Pids = expiring_presence_pids(Considered),+ Total = length(Pids),+ ok = schedule_reconciles(Pids, Total, WindowSeconds * 1000),+ #{+ scheduled => Total,+ considered => length(Considered),+ window_seconds => WindowSeconds,+ enabled => enabled()+ };+repair(_WindowSeconds) ->+ #{error => invalid_window_seconds}.++%% Only presences whose cached payload actually carries an expires_at are worth+%% nudging. Nudging every local presence would cost one payload rebuild and one+%% replicated cache write each, thousands per node, to correct a few dozen.+%% Filtering on "has an expiry" rather than "is expired" is deliberate: it also+%% arms the ongoing wakeup for statuses that have not expired yet.+-spec expiring_presence_pids([integer()]) -> [pid()].+expiring_presence_pids(UserIds) ->+ [+ Pid+ || UserId <- UserIds,+ has_expires_at(UserId),+ Pid <- [presence_pid(UserId)],+ is_pid(Pid)+ ].++-spec has_expires_at(integer()) -> boolean().+has_expires_at(UserId) ->+ case catch presence_cache:get(UserId) of+ {ok, Presence} when is_map(Presence) ->+ presence_carries_expiry(Presence);+ _ ->+ false+ end.++%% presence_cache:get/1 returns {ok, map()} | not_found, never a bare map, and+%% expires_at_ms/1 takes the timestamp value rather than the custom_status map.+-spec presence_carries_expiry(map()) -> boolean().+presence_carries_expiry(Presence) ->+ case maps:get(<<"custom_status">>, Presence, null) of+ CustomStatus when is_map(CustomStatus) ->+ expires_at_ms(maps:get(<<"expires_at">>, CustomStatus, null)) =/= none;+ _ ->+ false+ end.++-spec local_presence_user_ids() -> [user_id()].+local_presence_user_ids() ->+ try presence_manager_shards:determine_count() of+ {ShardCount, _Source} -> shard_user_ids(ShardCount)+ catch+ _Class:_Reason -> []+ end.++-spec shard_user_ids(pos_integer()) -> [user_id()].+shard_user_ids(ShardCount) ->+ lists:append([+ presence_manager:get_shard_user_ids(Index)+ || Index <- lists:seq(0, ShardCount - 1)+ ]).++-spec presence_pid(user_id()) -> pid() | undefined.+presence_pid(UserId) ->+ process_registry:registry_whereis(process_registry:build_process_key(presence, UserId)).++-spec schedule_reconciles([pid()], non_neg_integer(), non_neg_integer()) -> ok.+schedule_reconciles(Pids, Total, WindowMs) ->+ Message = reconcile_message(),+ _ = lists:foldl(+ fun(Pid, Index) ->+ _ = schedule_one(Pid, spread_ms(Index, Total, WindowMs), Message),+ Index + 1+ end,+ 0,+ Pids+ ),+ ok.++%% A single unreachable destination must not abort the sweep and leave the+%% operator unable to tell how many nudges were actually scheduled.+-spec schedule_one(pid(), non_neg_integer(), term()) -> ok.+schedule_one(Pid, Delay, Message) ->+ try erlang:send_after(Delay, Pid, Message) of+ _Ref -> ok+ catch+ _Class:_Reason -> ok+ end.++-spec spread_ms(non_neg_integer(), non_neg_integer(), non_neg_integer()) -> non_neg_integer().+spread_ms(_Index, 0, _WindowMs) -> 0;+spread_ms(_Index, _Total, 0) -> 0;+spread_ms(Index, Total, WindowMs) -> (Index * WindowMs) div Total.++-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++presence_carries_expiry_detects_an_expiry_test() ->+ ?assert(+ presence_carries_expiry(#{+ <<"custom_status">> => #{<<"expires_at">> => <<"2026-05-13T13:02:27.497Z">>}+ })+ ),+ ?assertNot(presence_carries_expiry(#{<<"custom_status">> => #{<<"text">> => <<"hi">>}})),+ ?assertNot(presence_carries_expiry(#{<<"custom_status">> => null})),+ ?assertNot(presence_carries_expiry(#{})).++-define(LIVE_EXPIRES_AT, <<"2026-05-13T13:02:27.497Z">>).++epoch_ms(Iso) ->+ calendar:rfc3339_to_system_time(binary_to_list(Iso), [{unit, millisecond}]).++rfc3339(EpochMs) ->+ list_to_binary(+ calendar:system_time_to_rfc3339(EpochMs, [{unit, millisecond}, {offset, "Z"}])+ ).++expiry_ms() ->+ epoch_ms(?LIVE_EXPIRES_AT).++status(ExpiresAt) ->+ #{+ <<"emoji_animated">> => false,+ <<"emoji_name">> => null,+ <<"expires_at">> => ExpiresAt,+ <<"text">> => <<"brb">>+ }.++status_without_expiry() ->+ #{<<"emoji_animated">> => false, <<"emoji_name">> => null, <<"text">> => <<"brb">>}.++parser_anchors_on_the_unix_epoch_test() ->+ ?assertEqual(0, epoch_ms(<<"1970-01-01T00:00:00.000Z">>)),+ ?assertEqual(1500, epoch_ms(<<"1970-01-01T00:00:01.500Z">>)).++parser_keeps_milliseconds_test() ->+ ?assertEqual(497, expiry_ms() rem 1000).++not_expired_test() ->+ Status = status(?LIVE_EXPIRES_AT),+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 60000)).++expires_one_second_in_the_future_test() ->+ Status = status(?LIVE_EXPIRES_AT),+ ?assertEqual({ok, 1000}, remaining_ms(Status, expiry_ms() - 1000)),+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 1000)).++boundary_is_expired_test() ->+ Status = status(?LIVE_EXPIRES_AT),+ ?assertEqual({ok, 0}, remaining_ms(Status, expiry_ms())),+ ?assertEqual(null, clear_if_expired(Status, expiry_ms())).++one_millisecond_before_boundary_is_kept_test() ->+ Status = status(?LIVE_EXPIRES_AT),+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms() - 1)).++expired_test() ->+ Status = status(?LIVE_EXPIRES_AT),+ ?assertEqual(null, clear_if_expired(Status, expiry_ms() + 1)),+ ?assertEqual(null, clear_if_expired(Status, expiry_ms() + 194 * 86400000)).++absent_expires_at_is_kept_test() ->+ Status = status_without_expiry(),+ ?assertEqual(none, remaining_ms(Status, expiry_ms())),+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms())).++null_expires_at_is_kept_test() ->+ Status = status(null),+ ?assertEqual(none, remaining_ms(Status, expiry_ms())),+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms())).++malformed_expires_at_is_kept_test() ->+ lists:foreach(+ fun(Value) ->+ Status = status(Value),+ ?assertEqual(none, remaining_ms(Status, expiry_ms())),+ ?assertEqual(Status, clear_if_expired(Status, expiry_ms()))+ end,+ [<<"">>, <<"not-a-date">>, <<"2026-13-45T99:99:99Z">>, 1747141347, [], #{}, true]+ ).++custom_status_null_test() ->+ ?assertEqual(none, remaining_ms(null, expiry_ms())),+ ?assertEqual(null, clear_if_expired(null, expiry_ms())).++custom_status_absent_test() ->+ ?assertEqual(none, remaining_ms(undefined, expiry_ms())),+ ?assertEqual(null, clear_if_expired(undefined, expiry_ms())).++non_map_custom_status_test() ->+ ?assertEqual(null, clear_if_expired(<<"garbage">>, expiry_ms())),+ ?assertEqual(null, clear_if_expired(123, expiry_ms())).++non_integer_reference_time_keeps_status_test() ->+ Status = status(?LIVE_EXPIRES_AT),+ ?assertEqual(Status, clear_if_expired(Status, not_a_time)).++wakeup_none_without_expiry_test() ->+ ?assertEqual(none, wakeup_delay(remaining_ms(status_without_expiry(), expiry_ms()))),+ ?assertEqual(none, wakeup_delay(remaining_ms(null, expiry_ms()))).++wakeup_none_when_already_expired_test() ->+ ?assertEqual(none, wakeup_delay({ok, 0})),+ ?assertEqual(none, wakeup_delay({ok, -1000})).++wakeup_is_never_early_test() ->+ {ok, DelayMs} = wakeup_delay({ok, 1000}),+ ?assert(DelayMs > 1000),+ ?assert(DelayMs =< 1000 + ?WAKEUP_JITTER_MS).++wakeup_clamps_to_the_timer_limit_test() ->+ {ok, DelayMs} = wakeup_delay({ok, 400 * ?MAX_TIMER_MS}),+ ?assert(DelayMs > ?MAX_TIMER_MS),+ ?assert(DelayMs =< ?MAX_TIMER_MS + ?WAKEUP_JITTER_MS).++disabled_by_default_test() ->+ application:unset_env(fluxer_gateway, ?ENABLED_KEY),+ Status = status(?LIVE_EXPIRES_AT),+ ?assertEqual(false, enabled()),+ ?assertEqual(Status, clear_if_expired(Status)),+ ?assertEqual(none, next_wakeup_ms(future_status())).++enabled_clears_expired_and_arms_future_test() ->+ Live = future_status(),+ with_expiry_enabled(fun() ->+ ?assertEqual(true, enabled()),+ ?assertEqual(null, clear_if_expired(status(?LIVE_EXPIRES_AT))),+ ?assertEqual(Live, clear_if_expired(Live)),+ ?assertMatch({ok, _}, next_wakeup_ms(Live))+ end).++future_status() ->+ status(rfc3339(erlang:system_time(millisecond) + 3600000)).++reconcile_message_is_a_gen_server_cast_test() ->+ ?assertEqual({'$gen_cast', reconcile_flattened_presence}, reconcile_message()).++spread_ms_test() ->+ ?assertEqual(0, spread_ms(0, 0, 60000)),+ ?assertEqual(0, spread_ms(5, 10, 0)),+ ?assertEqual(0, spread_ms(0, 10, 60000)),+ ?assertEqual(30000, spread_ms(5, 10, 60000)),+ ?assertEqual(54000, spread_ms(9, 10, 60000)).++repair_rejects_a_bad_window_test() ->+ ?assertEqual(#{error => invalid_window_seconds}, repair(-1)),+ ?assertEqual(#{error => invalid_window_seconds}, repair(<<"900">>)).++repair_reports_what_it_scheduled_test() ->+ Result = repair(0),+ ?assert(is_integer(maps:get(scheduled, Result))),+ ?assert(maps:get(scheduled, Result) >= 0),+ ?assertEqual(0, maps:get(window_seconds, Result)),+ ?assertEqual(enabled(), maps:get(enabled, Result)).++schedule_reconciles_delivers_the_cast_test() ->+ flush_mailbox(),+ ok = schedule_reconciles([self()], 1, 0),+ receive+ Message -> ?assertEqual(reconcile_message(), Message)+ after 1000 -> ?assert(false)+ end.++flush_mailbox() ->+ receive+ _Any -> flush_mailbox()+ after 0 -> ok+ end.++with_expiry_enabled(Fun) ->+ Previous = application:get_env(fluxer_gateway, ?ENABLED_KEY),+ application:set_env(fluxer_gateway, ?ENABLED_KEY, true),+ try+ Fun()+ after+ restore_expiry_env(Previous)+ end.++restore_expiry_env(undefined) ->+ application:unset_env(fluxer_gateway, ?ENABLED_KEY);+restore_expiry_env({ok, Value}) ->+ application:set_env(fluxer_gateway, ?ENABLED_KEY, Value).++-endif.diff --git a/fluxer_gateway/test/guild_member_list_tests.erl b/fluxer_gateway/test/guild_member_list_tests.erlindex 2d609390c..dfbb11d59 100644--- a/fluxer_gateway/test/guild_member_list_tests.erl+++ b/fluxer_gateway/test/guild_member_list_tests.erl@@ -118,6 +118,7 @@ run_broadcast_channel_list_dispatches_sync_immediately() -><<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>},<<"roles">> => []},+ ok = guild_member_list_engine:bulk_load(Ref, [{1, <<"one">>, [], true}], []),OldState = channel_list_state(Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), [Member]),@@ -144,6 +145,9 @@ run_broadcast_channel_list_fans_out_repeated_syncs() ->#{<<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>}},#{<<"user">> => #{<<"id">> => <<"2">>, <<"username">> => <<"two">>}}],+ ok = guild_member_list_engine:bulk_load(+ Ref, [{1, <<"one">>, [], true}, {2, <<"two">>, [], true}], []+ ),OldState = channel_list_state(Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), Members),@@ -408,6 +412,7 @@ presence_delta_state(Ref) -><<"user">> => #{<<"id">> => <<"1">>, <<"username">> => <<"one">>},<<"roles">> => []},+ ok = guild_member_list_engine:bulk_load(Ref, [{1, <<"one">>, [], true}], []),channel_list_state(Ref, make_subs_tab([{<<"500">>, <<"s1">>, [{0, 99}]}]), [Member]).presence_map(Status, Mobile, CustomStatus) ->diff --git a/fluxer_gateway/test/guild_voice_move_tests.erl b/fluxer_gateway/test/guild_voice_move_tests.erlindex a25ef606d..81629c7f2 100644--- a/fluxer_gateway/test/guild_voice_move_tests.erl+++ b/fluxer_gateway/test/guild_voice_move_tests.erl@@ -271,6 +271,9 @@ guild_server_stub(StateData, TestPid) ->guild_server_stub_loop(StateData, TestPid) ->receive+ {'$gen_call', From, {get_voice_guild_state}} ->+ gen:reply(From, undefined),+ guild_server_stub_loop(StateData, TestPid);{'$gen_call', From, {get_sessions}} ->gen:reply(From, StateData),guild_server_stub_loop(StateData, TestPid);diff --git a/fluxer_gateway/test/session_lifecycle_tests.erl b/fluxer_gateway/test/session_lifecycle_tests.erlindex ca825e4f5..2da65a101 100644--- a/fluxer_gateway/test/session_lifecycle_tests.erl+++ b/fluxer_gateway/test/session_lifecycle_tests.erl@@ -171,8 +171,8 @@ handle_resume_clamps_truncated_gap_in_replay_buffer_test() ->#{seq => 5, event => message_create, data => #{}}]}),- {reply, {ok, Missed, 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),- ?assertEqual([4, 5], [maps:get(seq, Event) || Event <- Missed]),+ {reply, {ok, [], 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),+ ?assertEqual([4, 5], collect_dispatched_seqs(2)),?assertEqual(self(), maps:get(socket_pid, State1)).handle_resume_clamps_skipped_event_hole_test() ->@@ -183,8 +183,8 @@ handle_resume_clamps_skipped_event_hole_test() ->#{seq => 5, event => message_create, data => #{}}]}),- {reply, {ok, Missed, 5}, _State1} = session_lifecycle:handle_resume(2, self(), State0),- ?assertEqual([3, 5], [maps:get(seq, Event) || Event <- Missed]).+ {reply, {ok, [], 5}, _State1} = session_lifecycle:handle_resume(2, self(), State0),+ ?assertEqual([3, 5], collect_dispatched_seqs(2)).handle_resume_rejects_seq_ahead_of_current_test() ->State0 = resume_test_state(#{@@ -211,8 +211,8 @@ handle_resume_accepts_seq_at_ack_seq_test() ->#{seq => 10, event => message_create, data => #{}}]}),- {reply, {ok, Missed, 10}, _State1} = session_lifecycle:handle_resume(8, self(), State0),- ?assertEqual([9, 10], [maps:get(seq, Event) || Event <- Missed]).+ {reply, {ok, [], 10}, _State1} = session_lifecycle:handle_resume(8, self(), State0),+ ?assertEqual([9, 10], collect_dispatched_seqs(2)).handle_resume_accepts_contiguous_replay_buffer_test() ->State0 = resume_test_state(#{@@ -223,8 +223,8 @@ handle_resume_accepts_contiguous_replay_buffer_test() ->#{seq => 5, event => message_create, data => #{}}]}),- {reply, {ok, Missed, 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),- ?assertEqual([3, 4, 5], [maps:get(seq, Event) || Event <- Missed]),+ {reply, {ok, [], 5}, State1} = session_lifecycle:handle_resume(2, self(), State0),+ ?assertEqual([3, 4, 5], collect_dispatched_seqs(3)),?assertEqual(self(), maps:get(socket_pid, State1)).handle_resume_cancels_pending_resume_timer_test() ->@@ -445,6 +445,19 @@ resume_test_state(Overrides) ->Overrides).+%% Small replay buffers are delivered inline (dispatched straight to the socket) rather+%% than returned in the resume reply, so these tests read the missed events back off+%% self()'s mailbox instead of the (now empty) reply list.+collect_dispatched_seqs(Count) ->+ lists:sort([collect_dispatched_seq() || _ <- lists:seq(1, Count)]).++collect_dispatched_seq() ->+ receive+ {dispatch, _Event, _Payload, Seq} -> Seq+ after 200 ->+ ?assert(false)+ end.+fenced_terminate_releases_the_user_session_count_test() ->ok = session_abuse_protection:ensure_tables(),UserId = 900201,