Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

refactor(gateway): move push outbox worker out, drop tests

Hampus11caede1fe1b
diff --git a/fluxer_gateway/src/push/push_ets_cache.erl b/fluxer_gateway/src/push/push_ets_cache.erl
index 6b9881c1f..d777311ec 100644
--- a/fluxer_gateway/src/push/push_ets_cache.erl
+++ b/fluxer_gateway/src/push/push_ets_cache.erl
@@ -7,17 +7,14 @@
init/0,
get_user_guild_settings/2,
put_user_guild_settings/3, put_user_guild_settings/4,
- delete_user_guild_settings/2,
reserve_user_guild_settings/2,
get_blocked_ids/1,
- put_blocked_ids/2,
put_blocked_ids_fetched/3,
reserve_blocked_ids/1,
release/1,
rebalance/0,
cache_stats/0,
- evict_tables/1,
- table_size/1
+ evict_tables/1
]).
-export_type([fill/0]).
@@ -62,10 +59,6 @@ put_user_guild_settings(UserId, GuildId, Settings) ->
put_user_guild_settings(UserId, GuildId, Settings, {?USER_GUILD_SETTINGS, _, _} = Fill) ->
fill(Fill, {{UserId, GuildId}, Settings}).
--spec delete_user_guild_settings(integer(), integer()) -> ok.
-delete_user_guild_settings(UserId, GuildId) ->
- safe_delete(?USER_GUILD_SETTINGS, {UserId, GuildId}).
-
-spec reserve_user_guild_settings([integer()], integer()) -> fill().
reserve_user_guild_settings(UserIds, GuildId) ->
reserve(?USER_GUILD_SETTINGS, [{UserId, GuildId} || UserId <- UserIds]).
@@ -92,10 +85,6 @@ live_fetched_blocked_ids(BlockedIds, ExpiresAt) ->
false -> undefined
end.
--spec put_blocked_ids(integer(), [integer()]) -> ok.
-put_blocked_ids(UserId, BlockedIds) ->
- write(?BLOCKED_IDS, {UserId, BlockedIds, infinity}).
-
-spec put_blocked_ids_fetched(integer(), [integer()], fill()) -> ok.
put_blocked_ids_fetched(UserId, BlockedIds, {?BLOCKED_IDS, _, _} = Fill) ->
ExpiresAt = erlang:system_time(second) + blocked_ids_ttl_seconds(),
@@ -107,16 +96,11 @@ reserve_blocked_ids(UserIds) ->
-spec blocked_ids_ttl_seconds() -> pos_integer().
blocked_ids_ttl_seconds() ->
- Value = app_pos_integer(push_blocked_ids_cache_ttl_seconds, ?DEFAULT_BLOCKED_IDS_TTL),
+ Value = push_utils:app_pos_integer(
+ push_blocked_ids_cache_ttl_seconds, ?DEFAULT_BLOCKED_IDS_TTL
+ ),
min(max(Value, ?MIN_BLOCKED_IDS_TTL), ?MAX_BLOCKED_IDS_TTL).
--spec app_pos_integer(atom(), pos_integer()) -> pos_integer().
-app_pos_integer(Key, Default) ->
- case application:get_env(fluxer_gateway, Key, undefined) of
- Value when is_integer(Value), Value > 0 -> Value;
- _ -> Default
- end.
-
-spec write(atom(), tuple()) -> ok.
write(Table, Row) ->
guard_table_size(Table, ?MAX_TABLE_ENTRIES),
@@ -350,58 +334,3 @@ user_id_from_key(UserId) when is_integer(UserId) ->
UserId;
user_id_from_key(_) ->
undefined.
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-synced_blocked_ids_never_expire_test() ->
- init(),
- try
- ok = put_blocked_ids(4002, [4003]),
- ?assertEqual([4003], get_blocked_ids(4002)),
- ok = evict_tables(#{}),
- ?assertEqual([4003], get_blocked_ids(4002))
- after
- safe_delete(?BLOCKED_IDS, 4002)
- end.
-
-fetched_blocked_ids_expire_and_are_reclaimed_test() ->
- init(),
- try
- ok = put_blocked_ids_fetched(4004, [4005], reserve_blocked_ids([4004])),
- ?assertEqual([4005], get_blocked_ids(4004)),
- Stale = erlang:system_time(second) - 1,
- true = ets:insert(?BLOCKED_IDS, {4004, [4005], Stale}),
- ?assertEqual(undefined, get_blocked_ids(4004)),
- ok = evict_tables(#{}),
- ?assertEqual([], ets:lookup(?BLOCKED_IDS, 4004))
- after
- safe_delete(?BLOCKED_IDS, 4004)
- end.
-
-untimed_blocked_ids_are_still_honoured_test() ->
- init(),
- try
- true = ets:insert(?BLOCKED_IDS, {4008, [4009]}),
- ?assertEqual([4009], get_blocked_ids(4008)),
- ok = evict_tables(#{}),
- ?assertEqual([4009], get_blocked_ids(4008))
- after
- safe_delete(?BLOCKED_IDS, 4008)
- end.
-
-blocked_ids_ttl_is_clamped_test() ->
- ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, blocked_ids_ttl_seconds()),
- ?assertEqual(?MIN_BLOCKED_IDS_TTL, with_ttl_env(1, fun blocked_ids_ttl_seconds/0)),
- ?assertEqual(?MAX_BLOCKED_IDS_TTL, with_ttl_env(999999, fun blocked_ids_ttl_seconds/0)),
- ?assertEqual(?DEFAULT_BLOCKED_IDS_TTL, with_ttl_env(0, fun blocked_ids_ttl_seconds/0)).
-
-with_ttl_env(Value, Fun) ->
- application:set_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds, Value),
- try
- Fun()
- after
- application:unset_env(fluxer_gateway, push_blocked_ids_cache_ttl_seconds)
- end.
-
--endif.
diff --git a/fluxer_gateway/src/push/push_outbox.erl b/fluxer_gateway/src/push/push_outbox.erl
index 0467041c7..01a0cbcc5 100644
--- a/fluxer_gateway/src/push/push_outbox.erl
+++ b/fluxer_gateway/src/push/push_outbox.erl
@@ -57,14 +57,16 @@
| skipped_active
| followup_clears
| enqueued.
+-type reads() :: #{{integer(), integer()} => {integer(), integer()}}.
+-type active() :: #{integer() => {non_neg_integer(), integer()}}.
-type state() :: #{
jobs := gb_trees:tree(non_neg_integer(), entry()),
ready := queue:queue(non_neg_integer()),
inflight := #{pid() => {reference(), reference(), entry()}},
followups => #{pid() => #{integer() => integer()}},
next_seq := non_neg_integer(),
- reads := #{{integer(), integer()} => {integer(), integer()}},
- active := #{integer() => {non_neg_integer(), integer()}},
+ reads := reads(),
+ active := active(),
counters := #{counter() => non_neg_integer()},
dropped := #{{kind(), atom()} => pos_integer()},
max_queue := pos_integer(),
@@ -133,7 +135,9 @@ init([]) ->
max_inflight => env_pos_integer(push_outbox_max_inflight, ?DEFAULT_MAX_INFLIGHT),
request_timeout_ms => request_timeout_ms(),
max_age_ms => max_age_ms(),
- retry_base_ms => app_pos_integer(push_outbox_retry_base_ms, ?DEFAULT_RETRY_BASE_MS)
+ retry_base_ms => push_utils:app_pos_integer(
+ push_outbox_retry_base_ms, ?DEFAULT_RETRY_BASE_MS
+ )
}}.
-spec handle_call(term(), gen_server:from(), state()) -> {reply, term(), state()}.
@@ -145,7 +149,7 @@ handle_call({enqueue, Job}, _From, State) when
is_map_key(channel_id, Job),
is_map_key(message_id, Job)
->
- {reply, ok, pump(admit(Job, State))};
+ {reply, ok, pump(admit(eqwalizer:dynamic_cast(Job), State))};
handle_call(stats, _From, State) ->
{reply, build_stats(State), State};
handle_call(_Request, _From, State) ->
@@ -159,17 +163,17 @@ handle_cast({truncate_read, UserId, ChannelId, MessageId}, State) when
handle_cast({session_active, UserId}, State) when is_integer(UserId) ->
{noreply, record_active(UserId, State)};
handle_cast({dropped, Kind, Reason}, State) when is_atom(Kind), is_atom(Reason) ->
- {noreply, count_dropped(Kind, Reason, State)};
+ {noreply, count_dropped(eqwalizer:dynamic_cast(Kind), Reason, State)};
handle_cast(_Msg, State) ->
{noreply, State}.
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info({push_outbox_reply, Pid, Result}, State) when is_pid(Pid) ->
- {noreply, pump(finish_worker(Pid, reply_result(Result), State))};
+ {noreply, pump(finish_worker(Pid, push_outbox_worker:reply_result(Result), State))};
handle_info({'DOWN', _MRef, process, Pid, Reason}, State) when is_pid(Pid) ->
- {noreply, pump(finish_worker(Pid, down_result(Reason), State))};
-handle_info({request_deadline, Pid}, State) when is_pid(Pid) ->
- kill_expired_worker(Pid, State),
+ {noreply, pump(finish_worker(Pid, push_outbox_worker:down_result(Reason), State))};
+handle_info({request_deadline, Pid}, #{inflight := Inflight} = State) when is_pid(Pid) ->
+ push_outbox_worker:kill_expired(Pid, Inflight),
{noreply, State};
handle_info({retry, Seq}, State) when is_integer(Seq), Seq >= 0 ->
{noreply, pump(make_ready(Seq, State))};
@@ -270,18 +274,14 @@ prepare(#{user_ids := UserIds} = Entry, #{reads := Reads, active := Active} = St
_ -> {send, with_user_ids(Kept, Entry), State1}
end.
--spec is_read(integer(), integer(), integer(), #{
- {integer(), integer()} => {integer(), integer()}
-}) ->
- boolean().
+-spec is_read(integer(), integer(), integer(), reads()) -> boolean().
is_read(UserId, ChannelId, MessageId, Reads) ->
case maps:find({UserId, ChannelId}, Reads) of
{ok, {ReadMessageId, _At}} -> MessageId =< ReadMessageId;
error -> false
end.
--spec became_active(integer(), non_neg_integer(), #{integer() => {non_neg_integer(), integer()}}) ->
- boolean().
+-spec became_active(integer(), non_neg_integer(), active()) -> boolean().
became_active(UserId, Seq, Active) ->
case maps:find(UserId, Active) of
{ok, {ActiveSeq, _At}} -> ActiveSeq > Seq;
@@ -368,20 +368,20 @@ truncate_entry(_Seq, _Entry, _UserId, _ChannelId, _MessageId, State) ->
State.
-spec remove_reader(non_neg_integer(), entry(), integer(), state()) -> state().
-remove_reader(Seq, #{user_ids := UserIds} = Entry, UserId, #{jobs := Jobs} = State) ->
+remove_reader(Seq, #{user_ids := UserIds} = Entry, UserId, State) ->
case lists:member(UserId, UserIds) of
false ->
State;
true ->
- State1 = bump(truncations, 1, State),
- case lists:delete(UserId, UserIds) of
- [] ->
- State1#{jobs := gb_trees:delete(Seq, Jobs)};
- Remaining ->
- State1#{jobs := gb_trees:update(Seq, with_user_ids(Remaining, Entry), Jobs)}
- end
+ drop_reader(Seq, lists:delete(UserId, UserIds), Entry, bump(truncations, 1, State))
end.
+-spec drop_reader(non_neg_integer(), [integer()], entry(), state()) -> state().
+drop_reader(Seq, [], _Entry, #{jobs := Jobs} = State) ->
+ State#{jobs := gb_trees:delete(Seq, Jobs)};
+drop_reader(Seq, Remaining, Entry, #{jobs := Jobs} = State) ->
+ State#{jobs := gb_trees:update(Seq, with_user_ids(Remaining, Entry), Jobs)}.
+
-spec record_active(integer(), state()) -> state().
record_active(UserId, #{active := Active, next_seq := Seq} = State) ->
State#{active := Active#{UserId => {Seq, now_ms()}}, next_seq := Seq + 1}.
@@ -391,30 +391,11 @@ start_worker(Entry, #{inflight := Inflight, request_timeout_ms := Timeout} = Sta
#{subject := Subject, body := Body} = Entry,
Outbox = self(),
{Pid, MRef} = spawn_monitor(fun() ->
- Outbox ! {push_outbox_reply, self(), send(Subject, Body, Timeout)}
+ Outbox ! {push_outbox_reply, self(), push_outbox_worker:send(Subject, Body, Timeout)}
end),
TRef = erlang:send_after(Timeout, Outbox, {request_deadline, Pid}),
State#{inflight := Inflight#{Pid => {MRef, TRef, Entry}}}.
--spec send(binary(), binary(), pos_integer()) -> ok | {error, term()}.
-send(Subject, Body, Timeout) ->
- try push_job_publisher:request(Subject, Body, Timeout) of
- ok -> ok;
- {error, Reason} -> {error, Reason}
- catch
- Class:Reason -> {error, {Class, Reason}}
- end.
-
--spec kill_expired_worker(pid(), state()) -> ok.
-kill_expired_worker(Pid, #{inflight := Inflight}) ->
- case maps:is_key(Pid, Inflight) of
- true ->
- exit(Pid, kill),
- ok;
- false ->
- ok
- end.
-
-spec finish_worker(pid(), ok | {error, term()}, state()) -> state().
finish_worker(Pid, Result, #{inflight := Inflight} = State) ->
case maps:take(Pid, Inflight) of
@@ -434,18 +415,20 @@ follow_up_clears(_Entry, Reads, State) when map_size(Reads) =:= 0 ->
State;
follow_up_clears(#{channel_id := ChannelId, message_id := AfterMessageId}, Reads, State) ->
case push:clear_notifications_enabled() of
- true ->
- maps:fold(
- fun(UserId, MessageId, Acc) ->
- follow_up_clear(UserId, ChannelId, MessageId, AfterMessageId, Acc)
- end,
- State,
- Reads
- );
- false ->
- State
+ true -> follow_up_each_clear(ChannelId, AfterMessageId, Reads, State);
+ false -> State
end.
+-spec follow_up_each_clear(integer(), integer(), #{integer() => integer()}, state()) -> state().
+follow_up_each_clear(ChannelId, AfterMessageId, Reads, State) ->
+ maps:fold(
+ fun(UserId, MessageId, Acc) ->
+ follow_up_clear(UserId, ChannelId, MessageId, AfterMessageId, Acc)
+ end,
+ State,
+ Reads
+ ).
+
-spec follow_up_clear(integer(), integer(), integer(), integer(), state()) -> state().
follow_up_clear(UserId, ChannelId, MessageId, AfterMessageId, State) ->
case pending_followup(UserId, ChannelId, State) of
@@ -478,15 +461,6 @@ inflight_channel(Pid, Inflight) ->
error -> error
end.
--spec reply_result(term()) -> ok | {error, term()}.
-reply_result(ok) -> ok;
-reply_result({error, Reason}) -> {error, Reason};
-reply_result(Other) -> {error, {invalid_worker_reply, Other}}.
-
--spec down_result(term()) -> {error, term()}.
-down_result(killed) -> {error, timeout};
-down_result(Reason) -> {error, {worker_down, Reason}}.
-
-spec handle_result(ok | {error, term()}, entry(), state()) -> state().
handle_result(ok, _Entry, State) ->
bump(delivered, 1, State);
@@ -608,92 +582,3 @@ env_pos_integer(Key, Default) ->
Value when is_integer(Value), Value > 0 -> Value;
_ -> Default
end.
-
--spec app_pos_integer(atom(), pos_integer()) -> pos_integer().
-app_pos_integer(Key, Default) ->
- case application:get_env(fluxer_gateway, Key, undefined) of
- Value when is_integer(Value), Value > 0 -> Value;
- _ -> Default
- end.
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-count(Counter, #{counters := Counters}) ->
- maps:get(Counter, Counters, 0).
-
-an_expired_entry_is_dropped_without_the_users_who_read_it_test() ->
- State = test_state(#{{7, 20} => {30, now_ms()}}),
- Result = handle_result({error, timeout}, test_entry([7, 8]), State),
- ?assertEqual(#{{message, expired} => 1}, maps:get(dropped, Result)),
- ?assertEqual(1, count(truncations, Result)),
- ?assertEqual(0, count(retries, Result)).
-
-an_expired_entry_every_recipient_read_is_not_counted_as_dropped_test() ->
- State = test_state(#{{7, 20} => {30, now_ms()}, {8, 20} => {31, now_ms()}}),
- Result = handle_result({error, timeout}, test_entry([7, 8]), State),
- ?assertEqual(#{}, maps:get(dropped, Result)),
- ?assertEqual(2, count(truncations, Result)).
-
-an_expired_clear_is_dropped_and_counted_by_kind_test() ->
- State = test_state(#{{7, 20} => {30, now_ms()}, {8, 20} => {30, now_ms()}}),
- Entry = (test_entry([7, 8]))#{kind := clear},
- Result = handle_result({error, timeout}, Entry, State),
- ?assertEqual(#{{clear, expired} => 1}, maps:get(dropped, Result)),
- ?assertEqual(0, count(truncations, Result)).
-
-a_failed_entry_within_its_age_is_retried_test() ->
- State = test_state(#{}),
- Entry = (test_entry([7]))#{enqueued_at := now_ms()},
- Result = handle_result({error, timeout}, Entry, State),
- ?assertEqual(#{}, maps:get(dropped, Result)),
- ?assertEqual(1, count(retries, Result)),
- ?assertEqual({value, Entry#{attempts := 4}}, gb_trees:lookup(0, maps:get(jobs, Result))).
-
-an_entry_expired_at_dispatch_is_dropped_test() ->
- Result = dispatch(test_entry([7]), test_state(#{})),
- ?assertEqual(#{{message, expired} => 1}, maps:get(dropped, Result)),
- ?assertEqual(0, map_size(maps:get(inflight, Result))).
-
-reported_drops_are_counted_by_kind_and_reason_test() ->
- {noreply, State1} = handle_cast({dropped, message, payload_too_large}, test_state(#{})),
- {noreply, State2} = handle_cast({dropped, message, payload_too_large}, State1),
- {noreply, State3} = handle_cast({dropped, ring, outbox_unavailable}, State2),
- ?assertEqual(
- #{{message, payload_too_large} => 2, {ring, outbox_unavailable} => 1},
- maps:get(dropped, build_stats(State3))
- ).
-
-test_state(Reads) ->
- #{
- jobs => gb_trees:empty(),
- ready => queue:new(),
- inflight => #{},
- followups => #{},
- next_seq => 1,
- reads => Reads,
- active => #{},
- counters => #{},
- dropped => #{},
- max_queue => ?DEFAULT_MAX_QUEUE,
- max_inflight => ?DEFAULT_MAX_INFLIGHT,
- request_timeout_ms => ?DEFAULT_REQUEST_TIMEOUT_MS,
- max_age_ms => ?DEFAULT_MAX_AGE_MS,
- retry_base_ms => ?DEFAULT_RETRY_BASE_MS
- }.
-
-test_entry(UserIds) ->
- #{
- kind => message,
- subject => <<"rpc.push.message">>,
- job => #{<<"user_ids">> => [integer_to_binary(UserId) || UserId <- UserIds]},
- body => <<"{}">>,
- user_ids => UserIds,
- channel_id => 20,
- message_id => 30,
- seq => 0,
- enqueued_at => now_ms() - ?DEFAULT_MAX_AGE_MS,
- attempts => 3
- }.
-
--endif.
diff --git a/fluxer_gateway/src/push/push_outbox_worker.erl b/fluxer_gateway/src/push/push_outbox_worker.erl
new file mode 100644
index 000000000..076d16bad
--- /dev/null
+++ b/fluxer_gateway/src/push/push_outbox_worker.erl
@@ -0,0 +1,34 @@
+%% SPDX-License-Identifier: AGPL-3.0-or-later
+
+-module(push_outbox_worker).
+-typing([eqwalizer]).
+
+-export([send/3, reply_result/1, down_result/1, kill_expired/2]).
+
+-spec send(binary(), binary(), pos_integer()) -> ok | {error, term()}.
+send(Subject, Body, Timeout) ->
+ try push_job_publisher:request(Subject, Body, Timeout) of
+ ok -> ok;
+ {error, Reason} -> {error, Reason}
+ catch
+ Class:Reason -> {error, {Class, Reason}}
+ end.
+
+-spec reply_result(term()) -> ok | {error, term()}.
+reply_result(ok) -> ok;
+reply_result({error, Reason}) -> {error, Reason};
+reply_result(Other) -> {error, {invalid_worker_reply, Other}}.
+
+-spec down_result(term()) -> {error, term()}.
+down_result(killed) -> {error, timeout};
+down_result(Reason) -> {error, {worker_down, Reason}}.
+
+-spec kill_expired(pid(), #{pid() => term()}) -> ok.
+kill_expired(Pid, Inflight) ->
+ case maps:is_key(Pid, Inflight) of
+ true ->
+ exit(Pid, kill),
+ ok;
+ false ->
+ ok
+ end.
diff --git a/fluxer_gateway/src/push/push_worker_pool.erl b/fluxer_gateway/src/push/push_worker_pool.erl
index f62905507..81d480b38 100644
--- a/fluxer_gateway/src/push/push_worker_pool.erl
+++ b/fluxer_gateway/src/push/push_worker_pool.erl
@@ -4,10 +4,12 @@
-typing([eqwalizer]).
-export([init_counter/0, maybe_spawn/1, active_count/0]).
+-export([bump_counter/2, read_counter/1, counter_table_status/0]).
-define(MAX_PUSH_WORKERS, 256).
-define(PUSH_WORKER_COUNTER, push_worker_counter).
-define(UTILIZATION_WARN_THRESHOLD, 80).
+-define(UTILIZATION_WARNING_INTERVAL_SECONDS, 10).
-spec init_counter() -> ok.
init_counter() ->
@@ -33,23 +35,23 @@ ensure_active_counter() ->
-spec maybe_spawn(fun(() -> term())) -> ok | dropped.
maybe_spawn(Fun) ->
- maybe_spawn(Fun, false).
-
--spec maybe_spawn(fun(() -> term()), boolean()) -> ok | dropped.
-maybe_spawn(Fun, RetryAfterInit) ->
try ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, 1}) of
- N when N > ?MAX_PUSH_WORKERS ->
- _ = ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, -1}),
- dropped;
- N ->
- maybe_warn_high_utilization(N),
- spawn_tracked_worker(Fun),
- ok
+ N -> spawn_with_capacity(Fun, N)
catch
error:badarg ->
- maybe_spawn_after_missing_counter(Fun, RetryAfterInit)
+ ok = init_counter(),
+ retry_spawn(Fun)
end.
+-spec spawn_with_capacity(fun(() -> term()), integer()) -> ok | dropped.
+spawn_with_capacity(_Fun, N) when N > ?MAX_PUSH_WORKERS ->
+ _ = ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, -1}),
+ dropped;
+spawn_with_capacity(Fun, N) ->
+ maybe_warn_high_utilization(N),
+ spawn_tracked_worker(Fun),
+ ok.
+
-spec active_count() -> non_neg_integer().
active_count() ->
try ets:lookup(?PUSH_WORKER_COUNTER, active) of
@@ -61,11 +63,10 @@ active_count() ->
-spec maybe_warn_high_utilization(non_neg_integer()) -> ok.
maybe_warn_high_utilization(Active) ->
- Pct = (Active * 100) div ?MAX_PUSH_WORKERS,
- case Pct >= ?UTILIZATION_WARN_THRESHOLD of
- true ->
+ case (Active * 100) div ?MAX_PUSH_WORKERS of
+ Pct when Pct >= ?UTILIZATION_WARN_THRESHOLD ->
throttled_utilization_warning(Active, Pct);
- false ->
+ _ ->
ok
end.
@@ -77,23 +78,24 @@ throttled_utilization_warning(Active, Pct) ->
undefined -> 0;
V -> V
end,
- case Now - Last >= 10 of
- true ->
+ case Now - Last of
+ Elapsed when Elapsed >= ?UTILIZATION_WARNING_INTERVAL_SECONDS ->
persistent_term:put({?MODULE, last_util_warn}, Now),
logger:warning("Push worker pool high utilisation", #{
utilisation_pct => Pct, active => Active, max => ?MAX_PUSH_WORKERS
}),
ok;
- false ->
+ _ ->
ok
end.
--spec maybe_spawn_after_missing_counter(fun(() -> term()), boolean()) -> ok | dropped.
-maybe_spawn_after_missing_counter(Fun, false) ->
- ok = init_counter(),
- maybe_spawn(Fun, true);
-maybe_spawn_after_missing_counter(_Fun, true) ->
- dropped.
+-spec retry_spawn(fun(() -> term())) -> ok | dropped.
+retry_spawn(Fun) ->
+ try ets:update_counter(?PUSH_WORKER_COUNTER, active, {2, 1}) of
+ N -> spawn_with_capacity(Fun, N)
+ catch
+ error:badarg -> dropped
+ end.
-spec spawn_tracked_worker(fun(() -> term())) -> pid().
spawn_tracked_worker(Fun) ->
@@ -118,59 +120,43 @@ run_tracked_worker(Fun) ->
log_worker_crash(Class, Reason, Stack) ->
logger:error("Push worker crash", #{class => Class, reason => Reason, stacktrace => Stack}).
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-maybe_spawn_initializes_missing_counter_and_tracks_worker_test() ->
- run_maybe_spawn_counter_test(fun() -> ok end).
+-spec counter_table_status() -> live | unavailable.
+counter_table_status() ->
+ case ets:info(?PUSH_WORKER_COUNTER, size) of
+ Size when is_integer(Size) -> live;
+ _ -> unavailable
+ end.
-maybe_spawn_repairs_missing_active_counter_test() ->
- run_maybe_spawn_counter_test(fun() ->
- _ = ets:new(?PUSH_WORKER_COUNTER, [named_table, public, set]),
- ok
- end).
+-spec read_counter(atom()) -> non_neg_integer() | unavailable.
+read_counter(Key) ->
+ try ets:lookup(?PUSH_WORKER_COUNTER, Key) of
+ [{Key, Value}] when is_integer(Value), Value >= 0 -> Value;
+ _ -> unavailable
+ catch
+ error:badarg -> unavailable
+ end.
-run_maybe_spawn_counter_test(Setup) ->
- cleanup_test_counter(),
- Self = self(),
- ok = Setup(),
- try
- ?assertEqual(ok, maybe_spawn(fun() -> Self ! worker_started end)),
- ?assertEqual(ok, receive_worker_started()),
- ok = gateway_retry_timer:wait(50),
- ?assertEqual(0, active_count())
- after
- cleanup_test_counter()
+-spec bump_counter(atom(), non_neg_integer()) -> ok.
+bump_counter(Key, Increment) ->
+ try ets:update_counter(?PUSH_WORKER_COUNTER, Key, {2, Increment}) of
+ _Value -> ok
+ catch
+ error:badarg -> insert_missing_counter(Key, Increment)
end.
-maybe_spawn_drops_when_full_test() ->
- cleanup_test_counter(),
- init_counter(),
- ets:insert(?PUSH_WORKER_COUNTER, {active, ?MAX_PUSH_WORKERS}),
- ?assertEqual(dropped, maybe_spawn(fun() -> ok end)),
- ?assertEqual(?MAX_PUSH_WORKERS, active_count()),
- cleanup_test_counter().
-
-worker_crash_decrements_counter_test() ->
- cleanup_test_counter(),
- init_counter(),
- ok = maybe_spawn(fun() -> error(deliberate_crash) end),
- ok = gateway_retry_timer:wait(100),
- ?assertEqual(0, active_count()),
- cleanup_test_counter().
-
-cleanup_test_counter() ->
- try ets:delete(?PUSH_WORKER_COUNTER) of
- _ -> ok
+-spec insert_missing_counter(atom(), non_neg_integer()) -> ok.
+insert_missing_counter(Key, Increment) ->
+ try ets:insert_new(?PUSH_WORKER_COUNTER, {Key, Increment}) of
+ true -> ok;
+ false -> retry_bump_counter(Key, Increment)
catch
error:badarg -> ok
end.
-receive_worker_started() ->
- receive
- worker_started -> ok
- after 100 ->
- timeout
+-spec retry_bump_counter(atom(), non_neg_integer()) -> ok.
+retry_bump_counter(Key, Increment) ->
+ try ets:update_counter(?PUSH_WORKER_COUNTER, Key, {2, Increment}) of
+ _Value -> ok
+ catch
+ error:badarg -> ok
end.
-
--endif.