A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
refactor(gateway): move push outbox worker out, drop tests
Hampus11caede1fe1b
diff --git a/fluxer_gateway/src/push/push_ets_cache.erl b/fluxer_gateway/src/push/push_ets_cache.erlindex 6b9881c1f..d777311ec 100644--- a/fluxer_gateway/src/push/push_ets_cache.erl+++ b/fluxer_gateway/src/push/push_ets_cache.erl@@ -7,17 +7,14 @@init/0,get_user_guild_settings/2,put_user_guild_settings/3, put_user_guild_settings/4,- delete_user_guild_settings/2,reserve_user_guild_settings/2,get_blocked_ids/1,- put_blocked_ids/2,put_blocked_ids_fetched/3,reserve_blocked_ids/1,release/1,rebalance/0,cache_stats/0,- evict_tables/1,- table_size/1+ evict_tables/1]).-export_type([fill/0]).@@ -62,10 +59,6 @@ put_user_guild_settings(UserId, GuildId, Settings) ->put_user_guild_settings(UserId, GuildId, Settings, {?USER_GUILD_SETTINGS, _, _} = Fill) ->fill(Fill, {{UserId, GuildId}, Settings}).--spec delete_user_guild_settings(integer(), integer()) -> ok.-delete_user_guild_settings(UserId, GuildId) ->- safe_delete(?USER_GUILD_SETTINGS, {UserId, GuildId}).--spec reserve_user_guild_settings([integer()], integer()) -> fill().reserve_user_guild_settings(UserIds, GuildId) ->reserve(?USER_GUILD_SETTINGS, [{UserId, GuildId} || UserId <- UserIds]).@@ -92,10 +85,6 @@ live_fetched_blocked_ids(BlockedIds, ExpiresAt) ->false -> undefinedend.--spec put_blocked_ids(integer(), [integer()]) -> ok.-put_blocked_ids(UserId, BlockedIds) ->- write(?BLOCKED_IDS, {UserId, BlockedIds, infinity}).--spec put_blocked_ids_fetched(integer(), [integer()], fill()) -> ok.put_blocked_ids_fetched(UserId, BlockedIds, {?BLOCKED_IDS, _, _} = Fill) ->ExpiresAt = erlang:system_time(second) + blocked_ids_ttl_seconds(),@@ -107,16 +96,11 @@ reserve_blocked_ids(UserIds) ->-spec blocked_ids_ttl_seconds() -> pos_integer().blocked_ids_ttl_seconds() ->- Value = app_pos_integer(push_blocked_ids_cache_ttl_seconds, ?DEFAULT_BLOCKED_IDS_TTL),+ Value = push_utils:app_pos_integer(+ push_blocked_ids_cache_ttl_seconds, ?DEFAULT_BLOCKED_IDS_TTL+ ),min(max(Value, ?MIN_BLOCKED_IDS_TTL), ?MAX_BLOCKED_IDS_TTL).--spec app_pos_integer(atom(), pos_integer()) -> pos_integer().-app_pos_integer(Key, Default) ->- case application:get_env(fluxer_gateway, Key, undefined) of- Value when is_integer(Value), Value > 0 -> Value;- _ -> Default- end.--spec write(atom(), tuple()) -> ok.write(Table, Row) ->guard_table_size(Table, ?MAX_TABLE_ENTRIES),@@ -350,58 +334,3 @@ user_id_from_key(UserId) when is_integer(UserId) ->UserId;user_id_from_key(_) ->undefined.---ifdef(TEST).--include_lib("eunit/include/eunit.hrl").--synced_blocked_ids_never_expire_test() ->- init(),- try- ok = put_blocked_ids(4002, [4003]),- ?assertEqual([4003], get_blocked_ids(4002)),- ok = evict_tables(#{}),- ?assertEqual([4003], get_blocked_ids(4002))- after- safe_delete(?BLOCKED_IDS, 4002)- end.--fetched_blocked_ids_expire_and_are_reclaimed_test() ->- init(),- try- ok = put_blocked_ids_fetched(4004, [4005], reserve_blocked_ids([4004])),- ?assertEqual([4005], get_blocked_ids(4004)),- Stale = erlang:system_time(second) - 1,- true = ets:insert(?BLOCKED_IDS, {4004, [4005], Stale}),- ?assertEqual(undefined, get_blocked_ids(4004)),- ok = evict_tables(#{}),- ?assertEqual([], ets:lookup(?BLOCKED_IDS, 4004))- after- safe_delete(?BLOCKED_IDS, 4004)- end.--untimed_blocked_ids_are_still_honoured_test() ->- init(),- try- true = ets:insert(?BLOCKED_IDS, {4008, [4009]}),- ?assertEqual([4009], get_blocked_ids(4008)),- ok = evict_tables(#{}),- ?assertEqual([4009], get_blocked_ids(4008))- after- safe_delete(?BLOCKED_IDS, 4008)- end.--blocked_ids_ttl_is_clamped_test() ->- ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, blocked_ids_ttl_seconds()),- ?assertEqual(?MIN_BLOCKED_IDS_TTL, with_ttl_env(1, fun blocked_ids_ttl_seconds/0)),- ?assertEqual(?MAX_BLOCKED_IDS_TTL, with_ttl_env(999999, fun blocked_ids_ttl_seconds/0)),- ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, with_ttl_env(0, fun blocked_ids_ttl_seconds/0)).--with_ttl_env(Value, Fun) ->- application:set_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds, Value),- try- Fun()- after- application:unset_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds)- end.---endif.diff --git a/fluxer_gateway/src/push/push_outbox.erl b/fluxer_gateway/src/push/push_outbox.erlindex 0467041c7..01a0cbcc5 100644--- a/fluxer_gateway/src/push/push_outbox.erl+++ b/fluxer_gateway/src/push/push_outbox.erl@@ -57,14 +57,16 @@| skipped_active| followup_clears| enqueued.+-type reads() :: #{{integer(), integer()} => {integer(), integer()}}.+-type active() :: #{integer() => {non_neg_integer(), integer()}}.-type state() :: #{jobs := gb_trees:tree(non_neg_integer(), entry()),ready := queue:queue(non_neg_integer()),inflight := #{pid() => {reference(), reference(), entry()}},followups => #{pid() => #{integer() => integer()}},next_seq := non_neg_integer(),- reads := #{{integer(), integer()} => {integer(), integer()}},- active := #{integer() => {non_neg_integer(), integer()}},+ reads := reads(),+ active := active(),counters := #{counter() => non_neg_integer()},dropped := #{{kind(), atom()} => pos_integer()},max_queue := pos_integer(),@@ -133,7 +135,9 @@ init([]) ->max_inflight => env_pos_integer(push_outbox_max_inflight, ?DEFAULT_MAX_INFLIGHT),request_timeout_ms => request_timeout_ms(),max_age_ms => max_age_ms(),- retry_base_ms => app_pos_integer(push_outbox_retry_base_ms, ?DEFAULT_RETRY_BASE_MS)+ retry_base_ms => push_utils:app_pos_integer(+ push_outbox_retry_base_ms, ?DEFAULT_RETRY_BASE_MS+ )}}.-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}.@@ -145,7 +149,7 @@ handle_call({enqueue, Job}, _From, State) whenis_map_key(channel_id, Job),is_map_key(message_id, Job)->- {reply, ok, pump(admit(Job, State))};+ {reply, ok, pump(admit(eqwalizer:dynamic_cast(Job), State))};handle_call(stats, _From, State) ->{reply, build_stats(State), State};handle_call(_Request, _From, State) ->@@ -159,17 +163,17 @@ handle_cast({truncate_read, UserId, ChannelId, MessageId}, State) whenhandle_cast({session_active, UserId}, State) when is_integer(UserId) ->{noreply, record_active(UserId, State)};handle_cast({dropped, Kind, Reason}, State) when is_atom(Kind), is_atom(Reason) ->- {noreply, count_dropped(Kind, Reason, State)};+ {noreply, count_dropped(eqwalizer:dynamic_cast(Kind), Reason, State)};handle_cast(_Msg, State) ->{noreply, State}.-spec handle_info(term(), state()) -> {noreply, state()}.handle_info({push_outbox_reply, Pid, Result}, State) when is_pid(Pid) ->- {noreply, pump(finish_worker(Pid, reply_result(Result), State))};+ {noreply, pump(finish_worker(Pid, push_outbox_worker:reply_result(Result), State))};handle_info({'DOWN', _MRef, process, Pid, Reason}, State) when is_pid(Pid) ->- {noreply, pump(finish_worker(Pid, down_result(Reason), State))};-handle_info({request_deadline, Pid}, State) when is_pid(Pid) ->- kill_expired_worker(Pid, State),+ {noreply, pump(finish_worker(Pid, push_outbox_worker:down_result(Reason), State))};+handle_info({request_deadline, Pid}, #{inflight := Inflight} = State) when is_pid(Pid) ->+ push_outbox_worker:kill_expired(Pid, Inflight),{noreply, State};handle_info({retry, Seq}, State) when is_integer(Seq), Seq >= 0 ->{noreply, pump(make_ready(Seq, State))};@@ -270,18 +274,14 @@ prepare(#{user_ids := UserIds} = Entry, #{reads := Reads, active := Active} = St_ -> {send, with_user_ids(Kept, Entry), State1}end.--spec is_read(integer(), integer(), integer(), #{- {integer(), integer()} => {integer(), integer()}-}) ->- boolean().+-spec is_read(integer(), integer(), integer(), reads()) -> boolean().is_read(UserId, ChannelId, MessageId, Reads) ->case maps:find({UserId, ChannelId}, Reads) of{ok, {ReadMessageId, _At}} -> MessageId =< ReadMessageId;error -> falseend.--spec became_active(integer(), non_neg_integer(), #{integer() => {non_neg_integer(), integer()}}) ->- boolean().+-spec became_active(integer(), non_neg_integer(), active()) -> boolean().became_active(UserId, Seq, Active) ->case maps:find(UserId, Active) of{ok, {ActiveSeq, _At}} -> ActiveSeq > Seq;@@ -368,20 +368,20 @@ truncate_entry(_Seq, _Entry, _UserId, _ChannelId, _MessageId, State) ->State.-spec remove_reader(non_neg_integer(), entry(), integer(), state()) -> state().-remove_reader(Seq, #{user_ids := UserIds} = Entry, UserId, #{jobs := Jobs} = State) ->+remove_reader(Seq, #{user_ids := UserIds} = Entry, UserId, State) ->case lists:member(UserId, UserIds) offalse ->State;true ->- State1 = bump(truncations, 1, State),- case lists:delete(UserId, UserIds) of- [] ->- State1#{jobs := gb_trees:delete(Seq, Jobs)};- Remaining ->- State1#{jobs := gb_trees:update(Seq, with_user_ids(Remaining, Entry), Jobs)}- end+ drop_reader(Seq, lists:delete(UserId, UserIds), Entry, bump(truncations, 1, State))end.+-spec drop_reader(non_neg_integer(), [integer()], entry(), state()) -> state().+drop_reader(Seq, [], _Entry, #{jobs := Jobs} = State) ->+ State#{jobs := gb_trees:delete(Seq, Jobs)};+drop_reader(Seq, Remaining, Entry, #{jobs := Jobs} = State) ->+ State#{jobs := gb_trees:update(Seq, with_user_ids(Remaining, Entry), Jobs)}.+-spec record_active(integer(), state()) -> state().record_active(UserId, #{active := Active, next_seq := Seq} = State) ->State#{active := Active#{UserId => {Seq, now_ms()}}, next_seq := Seq + 1}.@@ -391,30 +391,11 @@ start_worker(Entry, #{inflight := Inflight, request_timeout_ms := Timeout} = Sta#{subject := Subject, body := Body} = Entry,Outbox = self(),{Pid, MRef} = spawn_monitor(fun() ->- Outbox ! {push_outbox_reply, self(), send(Subject, Body, Timeout)}+ Outbox ! {push_outbox_reply, self(), push_outbox_worker:send(Subject, Body, Timeout)}end),TRef = erlang:send_after(Timeout, Outbox, {request_deadline, Pid}),State#{inflight := Inflight#{Pid => {MRef, TRef, Entry}}}.--spec send(binary(), binary(), pos_integer()) -> ok | {error, term()}.-send(Subject, Body, Timeout) ->- try push_job_publisher:request(Subject, Body, Timeout) of- ok -> ok;- {error, Reason} -> {error, Reason}- catch- Class:Reason -> {error, {Class, Reason}}- end.---spec kill_expired_worker(pid(), state()) -> ok.-kill_expired_worker(Pid, #{inflight := Inflight}) ->- case maps:is_key(Pid, Inflight) of- true ->- exit(Pid, kill),- ok;- false ->- ok- end.--spec finish_worker(pid(), ok | {error, term()}, state()) -> state().finish_worker(Pid, Result, #{inflight := Inflight} = State) ->case maps:take(Pid, Inflight) of@@ -434,18 +415,20 @@ follow_up_clears(_Entry, Reads, State) when map_size(Reads) =:= 0 ->State;follow_up_clears(#{channel_id := ChannelId, message_id := AfterMessageId}, Reads, State) ->case push:clear_notifications_enabled() of- true ->- maps:fold(- fun(UserId, MessageId, Acc) ->- follow_up_clear(UserId, ChannelId, MessageId, AfterMessageId, Acc)- end,- State,- Reads- );- false ->- State+ true -> follow_up_each_clear(ChannelId, AfterMessageId, Reads, State);+ false -> Stateend.+-spec follow_up_each_clear(integer(), integer(), #{integer() => integer()}, state()) -> state().+follow_up_each_clear(ChannelId, AfterMessageId, Reads, State) ->+ maps:fold(+ fun(UserId, MessageId, Acc) ->+ follow_up_clear(UserId, ChannelId, MessageId, AfterMessageId, Acc)+ end,+ State,+ Reads+ ).+-spec follow_up_clear(integer(), integer(), integer(), integer(), state()) -> state().follow_up_clear(UserId, ChannelId, MessageId, AfterMessageId, State) ->case pending_followup(UserId, ChannelId, State) of@@ -478,15 +461,6 @@ inflight_channel(Pid, Inflight) ->error -> errorend.--spec reply_result(term()) -> ok | {error, term()}.-reply_result(ok) -> ok;-reply_result({error, Reason}) -> {error, Reason};-reply_result(Other) -> {error, {invalid_worker_reply, Other}}.---spec down_result(term()) -> {error, term()}.-down_result(killed) -> {error, timeout};-down_result(Reason) -> {error, {worker_down, Reason}}.--spec handle_result(ok | {error, term()}, entry(), state()) -> state().handle_result(ok, _Entry, State) ->bump(delivered, 1, State);@@ -608,92 +582,3 @@ env_pos_integer(Key, Default) ->Value when is_integer(Value), Value > 0 -> Value;_ -> Defaultend.---spec app_pos_integer(atom(), pos_integer()) -> pos_integer().-app_pos_integer(Key, Default) ->- case application:get_env(fluxer_gateway, Key, undefined) of- Value when is_integer(Value), Value > 0 -> Value;- _ -> Default- end.---ifdef(TEST).--include_lib("eunit/include/eunit.hrl").--count(Counter, #{counters := Counters}) ->- maps:get(Counter, Counters, 0).--an_expired_entry_is_dropped_without_the_users_who_read_it_test() ->- State = test_state(#{{7, 20} => {30, now_ms()}}),- Result = handle_result({error, timeout}, test_entry([7, 8]), State),- ?assertEqual(#{{message, expired} => 1}, maps:get(dropped, Result)),- ?assertEqual(1, count(truncations, Result)),- ?assertEqual(0, count(retries, Result)).--an_expired_entry_every_recipient_read_is_not_counted_as_dropped_test() ->- State = test_state(#{{7, 20} => {30, now_ms()}, {8, 20} => {31, now_ms()}}),- Result = handle_result({error, timeout}, test_entry([7, 8]), State),- ?assertEqual(#{}, maps:get(dropped, Result)),- ?assertEqual(2, count(truncations, Result)).--an_expired_clear_is_dropped_and_counted_by_kind_test() ->- State = test_state(#{{7, 20} => {30, now_ms()}, {8, 20} => {30, now_ms()}}),- Entry = (test_entry([7, 8]))#{kind := clear},- Result = handle_result({error, timeout}, Entry, State),- ?assertEqual(#{{clear, expired} => 1}, maps:get(dropped, Result)),- ?assertEqual(0, count(truncations, Result)).--a_failed_entry_within_its_age_is_retried_test() ->- State = test_state(#{}),- Entry = (test_entry([7]))#{enqueued_at := now_ms()},- Result = handle_result({error, timeout}, Entry, State),- ?assertEqual(#{}, maps:get(dropped, Result)),- ?assertEqual(1, count(retries, Result)),- ?assertEqual({value, Entry#{attempts := 4}}, gb_trees:lookup(0, maps:get(jobs, Result))).--an_entry_expired_at_dispatch_is_dropped_test() ->- Result = dispatch(test_entry([7]), test_state(#{})),- ?assertEqual(#{{message, expired} => 1}, maps:get(dropped, Result)),- ?assertEqual(0, map_size(maps:get(inflight, Result))).--reported_drops_are_counted_by_kind_and_reason_test() ->- {noreply, State1} = handle_cast({dropped, message, payload_too_large}, test_state(#{})),- {noreply, State2} = handle_cast({dropped, message, payload_too_large}, State1),- {noreply, State3} = handle_cast({dropped, ring, outbox_unavailable}, State2),- ?assertEqual(- #{{message, payload_too_large} => 2, {ring, outbox_unavailable} => 1},- maps:get(dropped, build_stats(State3))- ).--test_state(Reads) ->- #{- jobs => gb_trees:empty(),- ready => queue:new(),- inflight => #{},- followups => #{},- next_seq => 1,- reads => Reads,- active => #{},- counters => #{},- dropped => #{},- max_queue => ?DEFAULT_MAX_QUEUE,- max_inflight => ?DEFAULT_MAX_INFLIGHT,- request_timeout_ms => ?DEFAULT_REQUEST_TIMEOUT_MS,- max_age_ms => ?DEFAULT_MAX_AGE_MS,- retry_base_ms => ?DEFAULT_RETRY_BASE_MS- }.--test_entry(UserIds) ->- #{- kind => message,- subject => <<"rpc.push.message">>,- job => #{<<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]},- body => <<"{}">>,- user_ids => UserIds,- channel_id => 20,- message_id => 30,- seq => 0,- enqueued_at => now_ms() - ?DEFAULT_MAX_AGE_MS,- attempts => 3- }.---endif.diff --git a/fluxer_gateway/src/push/push_outbox_worker.erl b/fluxer_gateway/src/push/push_outbox_worker.erlnew file mode 100644index 000000000..076d16bad--- /dev/null+++ b/fluxer_gateway/src/push/push_outbox_worker.erl@@ -0,0 +1,34 @@+%% SPDX-License-Identifier: AGPL-3.0-or-later++-module(push_outbox_worker).+-typing([eqwalizer]).++-export([send/3, reply_result/1, down_result/1, kill_expired/2]).++-spec send(binary(), binary(), pos_integer()) -> ok | {error, term()}.+send(Subject, Body, Timeout) ->+ try push_job_publisher:request(Subject, Body, Timeout) of+ ok -> ok;+ {error, Reason} -> {error, Reason}+ catch+ Class:Reason -> {error, {Class, Reason}}+ end.++-spec reply_result(term()) -> ok | {error, term()}.+reply_result(ok) -> ok;+reply_result({error, Reason}) -> {error, Reason};+reply_result(Other) -> {error, {invalid_worker_reply, Other}}.++-spec down_result(term()) -> {error, term()}.+down_result(killed) -> {error, timeout};+down_result(Reason) -> {error, {worker_down, Reason}}.++-spec kill_expired(pid(), #{pid() => term()}) -> ok.+kill_expired(Pid, Inflight) ->+ case maps:is_key(Pid, Inflight) of+ true ->+ exit(Pid, kill),+ ok;+ false ->+ ok+ end.diff --git a/fluxer_gateway/src/push/push_worker_pool.erl b/fluxer_gateway/src/push/push_worker_pool.erlindex f62905507..81d480b38 100644--- a/fluxer_gateway/src/push/push_worker_pool.erl+++ b/fluxer_gateway/src/push/push_worker_pool.erl@@ -4,10 +4,12 @@-typing([eqwalizer]).-export([init_counter/0, maybe_spawn/1, active_count/0]).+-export([bump_counter/2, read_counter/1, counter_table_status/0]).-define(MAX_PUSH_WORKERS, 256).-define(PUSH_WORKER_COUNTER, push_worker_counter).-define(UTILIZATION_WARN_THRESHOLD, 80).+-define(UTILIZATION_WARNING_INTERVAL_SECONDS, 10).-spec init_counter() -> ok.init_counter() ->@@ -33,23 +35,23 @@ ensure_active_counter() ->-spec maybe_spawn(fun(() -> term())) -> ok | dropped.maybe_spawn(Fun) ->- maybe_spawn(Fun, false).---spec maybe_spawn(fun(() -> term()), boolean()) -> ok | dropped.-maybe_spawn(Fun, RetryAfterInit) ->try ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, 1}) of- N when N > ?MAX_PUSH_WORKERS ->- _ = ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, -1}),- dropped;- N ->- maybe_warn_high_utilization(N),- spawn_tracked_worker(Fun),- ok+ N -> spawn_with_capacity(Fun, N)catcherror:badarg ->- maybe_spawn_after_missing_counter(Fun, RetryAfterInit)+ ok = init_counter(),+ retry_spawn(Fun)end.+-spec spawn_with_capacity(fun(() -> term()), integer()) -> ok | dropped.+spawn_with_capacity(_Fun, N) when N > ?MAX_PUSH_WORKERS ->+ _ = ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, -1}),+ dropped;+spawn_with_capacity(Fun, N) ->+ maybe_warn_high_utilization(N),+ spawn_tracked_worker(Fun),+ ok.+-spec active_count() -> non_neg_integer().active_count() ->try ets:lookup(?PUSH_WORKER_COUNTER, active) of@@ -61,11 +63,10 @@ active_count() ->-spec maybe_warn_high_utilization(non_neg_integer()) -> ok.maybe_warn_high_utilization(Active) ->- Pct = (Active * 100) div ?MAX_PUSH_WORKERS,- case Pct >= ?UTILIZATION_WARN_THRESHOLD of- true ->+ case (Active * 100) div ?MAX_PUSH_WORKERS of+ Pct when Pct >= ?UTILIZATION_WARN_THRESHOLD ->throttled_utilization_warning(Active, Pct);- false ->+ _ ->okend.@@ -77,23 +78,24 @@ throttled_utilization_warning(Active, Pct) ->undefined -> 0;V -> Vend,- case Now - Last >= 10 of- true ->+ case Now - Last of+ Elapsed when Elapsed >= ?UTILIZATION_WARNING_INTERVAL_SECONDS ->persistent_term:put({?MODULE, last_util_warn}, Now),logger:warning("Push worker pool high utilisation", #{utilisation_pct => Pct, active => Active, max => ?MAX_PUSH_WORKERS}),ok;- false ->+ _ ->okend.--spec maybe_spawn_after_missing_counter(fun(() -> term()), boolean()) -> ok | dropped.-maybe_spawn_after_missing_counter(Fun, false) ->- ok = init_counter(),- maybe_spawn(Fun, true);-maybe_spawn_after_missing_counter(_Fun, true) ->- dropped.+-spec retry_spawn(fun(() -> term())) -> ok | dropped.+retry_spawn(Fun) ->+ try ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, 1}) of+ N -> spawn_with_capacity(Fun, N)+ catch+ error:badarg -> dropped+ end.-spec spawn_tracked_worker(fun(() -> term())) -> pid().spawn_tracked_worker(Fun) ->@@ -118,59 +120,43 @@ run_tracked_worker(Fun) ->log_worker_crash(Class, Reason, Stack) ->logger:error("Push worker crash", #{class => Class, reason => Reason, stacktrace => Stack}).--ifdef(TEST).--include_lib("eunit/include/eunit.hrl").--maybe_spawn_initializes_missing_counter_and_tracks_worker_test() ->- run_maybe_spawn_counter_test(fun() -> ok end).+-spec counter_table_status() -> live | unavailable.+counter_table_status() ->+ case ets:info(?PUSH_WORKER_COUNTER, size) of+ Size when is_integer(Size) -> live;+ _ -> unavailable+ end.-maybe_spawn_repairs_missing_active_counter_test() ->- run_maybe_spawn_counter_test(fun() ->- _ = ets:new(?PUSH_WORKER_COUNTER, [named_table, public, set]),- ok- end).+-spec read_counter(atom()) -> non_neg_integer() | unavailable.+read_counter(Key) ->+ try ets:lookup(?PUSH_WORKER_COUNTER, Key) of+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;+ _ -> unavailable+ catch+ error:badarg -> unavailable+ end.-run_maybe_spawn_counter_test(Setup) ->- cleanup_test_counter(),- Self = self(),- ok = Setup(),- try- ?assertEqual(ok, maybe_spawn(fun() -> Self ! worker_started end)),- ?assertEqual(ok, receive_worker_started()),- ok = gateway_retry_timer:wait(50),- ?assertEqual(0, active_count())- after- cleanup_test_counter()+-spec bump_counter(atom(), non_neg_integer()) -> ok.+bump_counter(Key, Increment) ->+ try ets:update_counter(?PUSH_WORKER_COUNTER, Key, {2, Increment}) of+ _Value -> ok+ catch+ error:badarg -> insert_missing_counter(Key, Increment)end.-maybe_spawn_drops_when_full_test() ->- cleanup_test_counter(),- init_counter(),- ets:insert(?PUSH_WORKER_COUNTER, {active, ?MAX_PUSH_WORKERS}),- ?assertEqual(dropped, maybe_spawn(fun() -> ok end)),- ?assertEqual(?MAX_PUSH_WORKERS, active_count()),- cleanup_test_counter().--worker_crash_decrements_counter_test() ->- cleanup_test_counter(),- init_counter(),- ok = maybe_spawn(fun() -> error(deliberate_crash) end),- ok = gateway_retry_timer:wait(100),- ?assertEqual(0, active_count()),- cleanup_test_counter().--cleanup_test_counter() ->- try ets:delete(?PUSH_WORKER_COUNTER) of- _ -> ok+-spec insert_missing_counter(atom(), non_neg_integer()) -> ok.+insert_missing_counter(Key, Increment) ->+ try ets:insert_new(?PUSH_WORKER_COUNTER, {Key, Increment}) of+ true -> ok;+ false -> retry_bump_counter(Key, Increment)catcherror:badarg -> okend.-receive_worker_started() ->- receive- worker_started -> ok- after 100 ->- timeout+-spec retry_bump_counter(atom(), non_neg_integer()) -> ok.+retry_bump_counter(Key, Increment) ->+ try ets:update_counter(?PUSH_WORKER_COUNTER, Key, {2, Increment}) of+ _Value -> ok+ catch+ error:badarg -> okend.---endif.