Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(gateway): recover NATS subscriptions after failed connects (#3690)

Hampusb27c4e9c1b48
diff --git a/fluxer_gateway/src/gateway/gateway_nats_pool.erl b/fluxer_gateway/src/gateway/gateway_nats_pool.erl
index 234ace3e2..8147ed5ce 100644
--- a/fluxer_gateway/src/gateway/gateway_nats_pool.erl
+++ b/fluxer_gateway/src/gateway/gateway_nats_pool.erl
@@ -5,6 +5,7 @@
-behaviour(gen_server).
-export([start_link/0, pub/2, pub_reply/2, reply_publish_failures/0, get_pool_status/0]).
+-export([has_ready_connection/0]).
-export([enable_rpc_subscription/0, disable_rpc_subscription/0]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
@@ -18,7 +19,8 @@
handle_connect_ok_entry/5,
accept_connect_ok/4,
ignore_stale_connect_ok/2,
- handle_ready/4,
+ handle_ready/2,
+ accept_connect_timeout/3,
handle_conn_event_down/2,
handle_slot_monitor_down/5,
conn_from_term/1
@@ -79,6 +81,13 @@ reply_failure_counter() ->
Ref
end.
+-spec has_ready_connection() -> boolean().
+has_ready_connection() ->
+ case gateway_nats_pool_conn:get_pool_conn() of
+ {ok, _Conn} -> true;
+ {error, _Reason} -> false
+ end.
+
-spec get_pool_status() -> map().
get_pool_status() ->
gen_server:call(?MODULE, get_pool_status).
@@ -107,6 +116,7 @@ init([]) ->
connecting => #{},
rpc_enabled => gateway_nats_pool_conn:nats_rpc_enabled(),
subs => #{},
+ sub_failures => #{},
handler_count => 0,
handler_refs => #{},
max_handlers => gateway_nats_pool_conn:max_handlers()
@@ -144,8 +154,10 @@ handle_info({reconnect_slot, Idx}, #{slots := Slots} = State) when
is_integer(Idx), Idx >= 0
->
{noreply, reconnect_slot(Idx, Slots, State)};
-handle_info({Conn, ready}, #{rpc_enabled := RpcEnabled, slots := Slots} = State) ->
- {noreply, handle_ready(conn_from_term(Conn), RpcEnabled, Slots, State)};
+handle_info({Conn, ready}, State) ->
+ {noreply, handle_ready(conn_from_term(Conn), State)};
+handle_info({resubscribe_slot, Idx}, State) when is_integer(Idx), Idx >= 0 ->
+ {noreply, resubscribe_slot(Idx, State)};
handle_info({_Conn, _Sid, {msg, Subject, Payload, MsgOpts}}, State) when
is_binary(Subject), is_binary(Payload), is_map(MsgOpts)
->
@@ -229,8 +241,17 @@ enable_subscriptions(Slots, State) ->
-spec enable_slot_subscription(non_neg_integer(), nats:conn() | undefined, map()) -> map().
enable_slot_subscription(_Idx, undefined, State) ->
State;
-enable_slot_subscription(Idx, Conn, State) ->
- gateway_nats_pool_conn:subscribe_slot(Idx, Conn, State).
+enable_slot_subscription(Idx, Conn, #{connecting := Connecting} = State) ->
+ case maps:is_key(Idx, Connecting) of
+ true -> State;
+ false -> gateway_nats_pool_conn:subscribe_slot(Idx, Conn, State)
+ end.
+
+-spec resubscribe_slot(non_neg_integer(), map()) -> map().
+resubscribe_slot(_Idx, #{rpc_enabled := false} = State) ->
+ State;
+resubscribe_slot(Idx, #{slots := Slots} = State) ->
+ enable_slot_subscription(Idx, maps:get(Idx, Slots, undefined), State).
-spec disable_subscriptions(map(), map()) -> ok.
disable_subscriptions(Slots, Subs) ->
@@ -273,25 +294,34 @@ handle_connect_ok_entry(Idx, Pid, Entry, Conn, State) ->
-spec accept_connect_ok(non_neg_integer(), map(), nats:conn(), map()) -> map().
accept_connect_ok(Idx, Entry, Conn, State) ->
- #{
- connecting := Connecting,
- slots := Slots,
- monitors := Monitors,
- pool_size := PoolSize
- } = State,
- cancel_connect_timer(Entry),
+ #{connecting := Connecting, slots := Slots, monitors := Monitors} = State,
cleanup_old_conn(maps:get(Idx, Slots, undefined)),
CleanMon = remove_slot_monitors(Idx, Monitors),
MRef = nats:monitor(Conn),
- NewSlots = Slots#{Idx => Conn},
- gateway_nats_pool_conn:update_persistent_term(NewSlots, PoolSize),
gateway_nats_pool_conn:reset_failure_count(Idx),
- logger:info("Gateway NATS pool slot connected", #{slot => Idx}),
- State#{
- slots => NewSlots,
+ NewState = State#{
+ slots => Slots#{Idx => Conn},
monitors => CleanMon#{MRef => Idx},
- connecting => maps:remove(Idx, Connecting)
- }.
+ connecting => Connecting#{Idx => connecting_entry_with_conn(Entry, Conn)}
+ },
+ case gateway_nats_pool_conn:conn_ready(Conn) of
+ true -> mark_slot_ready(Idx, NewState);
+ false -> NewState
+ end.
+
+-spec connecting_entry_with_conn(term(), nats:conn()) -> map().
+connecting_entry_with_conn(Entry, Conn) when is_map(Entry) ->
+ Entry#{conn => Conn};
+connecting_entry_with_conn(Pid, Conn) ->
+ #{pid => Pid, conn => Conn}.
+
+-spec mark_slot_ready(non_neg_integer(), map()) -> map().
+mark_slot_ready(Idx, #{connecting := Connecting} = State) ->
+ cancel_connect_timer(maps:get(Idx, Connecting, undefined)),
+ ReadyState = State#{connecting => maps:remove(Idx, Connecting)},
+ gateway_nats_pool_conn:publish_ready_slots(ReadyState),
+ logger:info("Gateway NATS pool slot connected", #{slot => Idx}),
+ resubscribe_slot(Idx, ReadyState).
-spec ignore_stale_connect_ok(nats:conn(), map()) -> map().
ignore_stale_connect_ok(Conn, State) ->
@@ -331,20 +361,12 @@ accept_connect_error(Idx, Entry, Reason, #{connecting := Connecting} = State) ->
gateway_nats_pool_conn:schedule_reconnect(Idx, self()),
State#{connecting => maps:remove(Idx, Connecting)}.
--spec handle_ready(nats:conn(), boolean(), map(), map()) -> map().
-handle_ready(_Conn, false, _Slots, State) ->
- State;
-handle_ready(Conn, true, Slots, State) ->
+-spec handle_ready(nats:conn(), map()) -> map().
+handle_ready(Conn, #{connecting := Connecting} = State) ->
case gateway_nats_pool_conn:find_slot_by_conn(Conn, State) of
undefined -> State;
- Idx -> handle_ready_slot(Idx, Conn, Slots, State)
- end.
-
--spec handle_ready_slot(non_neg_integer(), nats:conn(), map(), map()) -> map().
-handle_ready_slot(Idx, Conn, Slots, State) ->
- case maps:find(Idx, Slots) of
- {ok, Conn} -> gateway_nats_pool_conn:subscribe_slot(Idx, Conn, State);
- _ -> State
+ Idx when is_map_key(Idx, Connecting) -> mark_slot_ready(Idx, State);
+ Idx -> resubscribe_slot(Idx, State)
end.
-spec handle_rpc_msg(binary(), binary(), map(), map()) -> map().
@@ -372,7 +394,7 @@ dispatch_rpc_msg(Subject, Payload, MsgOpts, HC, MaxH, HRefs, State) ->
undefined ->
State;
ReplyTo when HC >= MaxH ->
- _ = send_overloaded(ReplyTo),
+ send_overloaded_async(ReplyTo),
State;
ReplyTo ->
{_Pid, MRef} = spawn_rpc_handler(Subject, Payload, ReplyTo),
@@ -385,6 +407,11 @@ spawn_rpc_handler(Subject, Payload, ReplyTo) ->
gateway_nats_rpc_handler:handle_rpc_request(Subject, Payload, ReplyTo)
end).
+-spec send_overloaded_async(binary()) -> ok.
+send_overloaded_async(ReplyTo) ->
+ _ = spawn(fun() -> send_overloaded(ReplyTo) end),
+ ok.
+
-spec send_overloaded(binary()) -> ok | {error, term()}.
send_overloaded(ReplyTo) ->
Resp = iolist_to_binary(
@@ -474,6 +501,9 @@ handle_legacy_connect_timeout(Idx, #{connecting := Connecting} = State) ->
end.
-spec accept_connect_timeout(non_neg_integer(), term(), map()) -> map().
+accept_connect_timeout(Idx, #{conn := Conn}, #{pool_size := PoolSize} = State) ->
+ logger:warning("Gateway NATS pool slot connection did not become ready", #{slot => Idx}),
+ gateway_nats_pool_conn:handle_conn_down(Idx, conn_from_term(Conn), PoolSize, State);
accept_connect_timeout(Idx, Entry, #{connecting := Connecting} = State) ->
gateway_nats_pool_conn:maybe_kill_connect_pid(connecting_entry_pid(Entry)),
logger:warning("Gateway NATS pool slot connect attempt timed out", #{slot => Idx}),
diff --git a/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erl b/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erl
index 32f93776b..68da05d58 100644
--- a/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erl
+++ b/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erl
@@ -12,7 +12,11 @@
update_persistent_term/2,
schedule_reconnect/2,
reset_failure_count/1,
+ publish_ready_slots/1,
subscribe_slot/3,
+ guarded_sub/3,
+ conn_ready/1,
+ subscribe_retry_delay/1,
unsubscribe_slot_sids/2,
is_rpc_subject/1,
pool_size/0,
@@ -37,6 +41,9 @@
-define(CIRCUIT_BREAKER_THRESHOLD, 5).
-define(CIRCUIT_BREAKER_COOLDOWN_MS, 15000).
-define(FAILURE_COUNTS_KEY, {gateway_nats_pool, failure_counts}).
+-define(SUBSCRIBE_RETRY_BASE_MS, 250).
+-define(SUBSCRIBE_RETRY_MAX_MS, 5000).
+-define(SUBSCRIBE_TIMEOUT_MS, 5000).
-spec get_pool_conn() -> {ok, nats:conn()} | {error, no_connection}.
get_pool_conn() ->
@@ -134,17 +141,37 @@ address_family(Host) ->
handle_conn_down(
Idx,
Conn,
- PoolSize,
- #{slots := Slots, subs := Subs, monitors := Monitors} = State
+ _PoolSize,
+ #{slots := Slots, subs := Subs, monitors := Monitors, connecting := Connecting} = State
) ->
logger:warning("Gateway NATS pool slot connection lost, reconnecting", #{slot => Idx}),
ignore_disconnect(Conn),
+ cancel_entry_timer(maps:get(Idx, Connecting, undefined)),
CleanMonitors = maps:filter(fun(_Ref, SlotIdx) -> SlotIdx =/= Idx end, Monitors),
- NewSlots = Slots#{Idx => undefined},
- NewSubs = maps:remove(Idx, Subs),
- update_persistent_term(NewSlots, PoolSize),
+ NewState = State#{
+ slots => Slots#{Idx => undefined},
+ subs => maps:remove(Idx, Subs),
+ monitors => CleanMonitors,
+ connecting => maps:remove(Idx, Connecting)
+ },
+ publish_ready_slots(NewState),
schedule_reconnect(Idx, self()),
- State#{slots => NewSlots, subs => NewSubs, monitors => CleanMonitors}.
+ NewState.
+
+-spec cancel_entry_timer(term()) -> ok.
+cancel_entry_timer(#{timer := TimerRef}) when is_reference(TimerRef) ->
+ _ = erlang:cancel_timer(TimerRef, [{async, true}, {info, false}]),
+ ok;
+cancel_entry_timer(_) ->
+ ok.
+
+-spec publish_ready_slots(map()) -> ok.
+publish_ready_slots(#{slots := Slots, connecting := Connecting, pool_size := PoolSize}) ->
+ Ready = maps:filter(
+ fun(Idx, Conn) -> Conn =/= undefined andalso not maps:is_key(Idx, Connecting) end,
+ Slots
+ ),
+ update_persistent_term(Ready, PoolSize).
-spec find_slot_by_conn(nats:conn(), map()) -> non_neg_integer() | undefined.
find_slot_by_conn(Conn, #{slots := Slots}) ->
@@ -206,37 +233,84 @@ subscribe_slot(Idx, Conn, #{subs := Subs} = State) ->
subscribe_new_slot(Idx, Conn, Subs, State) ->
Role = fluxer_gateway_sup:current_role(),
Subjects = gateway_nats_rpc_handler:rpc_subjects_for_role(Role),
- Sids = subscribe_slot_subjects(Idx, Conn, Role, Subjects),
- case Sids of
- [] -> State;
- _ -> State#{subs => Subs#{Idx => Sids}}
+ Results = [subscribe_slot_subject(Idx, Conn, Role, Subject) || Subject <- Subjects],
+ Sids = [Sid || {ok, Sid} <- Results],
+ case length(Sids) =:= length(Subjects) of
+ true when Sids =:= [] ->
+ State;
+ true ->
+ State#{subs => Subs#{Idx => Sids}, sub_failures => clear_sub_failures(Idx, State)};
+ false ->
+ unsubscribe_slot_sids(Conn, Sids),
+ schedule_slot_resubscribe(Idx, State)
end.
--spec subscribe_slot_subjects(non_neg_integer(), nats:conn(), atom(), [binary()]) -> [term()].
-subscribe_slot_subjects(Idx, Conn, Role, Subjects) ->
- lists:filtermap(
- fun(Subject) ->
- subscribe_slot_subject(Idx, Conn, Role, Subject)
- end,
- Subjects
- ).
-
-spec subscribe_slot_subject(non_neg_integer(), nats:conn(), atom(), binary()) ->
- {true, term()} | false.
+ {ok, nats:sid()} | {error, term()}.
subscribe_slot_subject(Idx, Conn, Role, Subject) ->
- case nats:sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of
+ case guarded_sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of
{ok, Sid} ->
logger:info(
"Gateway NATS pool slot ~p subscribed to ~s (role=~p)",
[Idx, Subject, Role]
),
- {true, Sid};
+ {ok, Sid};
{error, Reason} ->
logger:error(
"Gateway NATS pool slot ~p failed to subscribe to ~s: ~p",
[Idx, Subject, Reason]
),
- false
+ {error, Reason}
+ end.
+
+-spec clear_sub_failures(non_neg_integer(), map()) -> map().
+clear_sub_failures(Idx, State) ->
+ maps:remove(Idx, maps:get(sub_failures, State, #{})).
+
+-spec schedule_slot_resubscribe(non_neg_integer(), map()) -> map().
+schedule_slot_resubscribe(Idx, State) ->
+ Failures = maps:get(sub_failures, State, #{}),
+ Count = maps:get(Idx, Failures, 0) + 1,
+ Delay = subscribe_retry_delay(Count),
+ logger:warning("Gateway NATS pool slot will retry its subscriptions", #{
+ slot => Idx, attempt => Count, delay_ms => Delay
+ }),
+ erlang:send_after(Delay, self(), {resubscribe_slot, Idx}),
+ State#{sub_failures => Failures#{Idx => Count}}.
+
+-spec subscribe_retry_delay(term()) -> pos_integer().
+subscribe_retry_delay(Failures) when is_integer(Failures), Failures > 5 ->
+ ?SUBSCRIBE_RETRY_MAX_MS;
+subscribe_retry_delay(Failures) when is_integer(Failures), Failures > 0 ->
+ min(?SUBSCRIBE_RETRY_MAX_MS, ?SUBSCRIBE_RETRY_BASE_MS bsl (Failures - 1));
+subscribe_retry_delay(_Failures) ->
+ ?SUBSCRIBE_RETRY_BASE_MS.
+
+-spec guarded_sub(nats:conn(), binary(), map()) -> {ok, nats:sid()} | {error, term()}.
+guarded_sub(Conn, Subject, Opts) ->
+ Guard = spawn(fun() -> disconnect_unless_done(Conn) end),
+ try nats:sub(Conn, Subject, Opts) of
+ Result -> Result
+ catch
+ exit:Reason -> {error, {exit, Reason}}
+ after
+ Guard ! done
+ end.
+
+-spec disconnect_unless_done(nats:conn()) -> ok.
+disconnect_unless_done(Conn) ->
+ receive
+ done -> ok
+ after ?SUBSCRIBE_TIMEOUT_MS ->
+ logger:warning("Gateway NATS subscribe did not finish in time, closing the connection"),
+ ignore_disconnect(Conn)
+ end.
+
+-spec conn_ready(nats:conn()) -> boolean().
+conn_ready(Conn) ->
+ case nats:is_ready(Conn) of
+ true -> true;
+ _ -> false
end.
-spec unsubscribe_slot_sids(nats:conn(), term()) -> ok.
diff --git a/fluxer_gateway/src/gateway/gateway_nats_rpc.erl b/fluxer_gateway/src/gateway/gateway_nats_rpc.erl
index 4b9e4d4b9..e05275f05 100644
--- a/fluxer_gateway/src/gateway/gateway_nats_rpc.erl
+++ b/fluxer_gateway/src/gateway/gateway_nats_rpc.erl
@@ -5,7 +5,7 @@
-behaviour(gen_server).
-export([start_link/0, get_connection/0, subscribe/2]).
--export([enable_rpc_subscription/0, disable_rpc_subscription/0]).
+-export([enable_rpc_subscription/0, disable_rpc_subscription/0, nats_ready/0]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
@@ -13,10 +13,13 @@
{no_opaque, [
handle_info/2,
handle_connect_ok/3,
+ handle_connect_timeout/2,
conn_from_term/1
]}
).
+-define(STATUS_KEY, {?MODULE, subscription_status}).
+
-spec start_link() -> gen_server:start_ret().
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
@@ -37,45 +40,50 @@ enable_rpc_subscription() ->
disable_rpc_subscription() ->
gen_server:call(?MODULE, disable_rpc_subscription, 5000).
+-spec nats_ready() -> boolean().
+nats_ready() ->
+ case persistent_term:get(?STATUS_KEY, missing) of
+ not_required -> true;
+ ready -> gateway_nats_pool:has_ready_connection();
+ _ -> false
+ end.
+
-spec init([]) -> {ok, map()}.
init([]) ->
process_flag(trap_exit, true),
erlang:process_flag(fullsweep_after, 10),
RpcEnabled = gateway_nats_pool_conn:nats_rpc_enabled(),
self() ! connect,
- {ok, #{
- conn => undefined,
- sub => undefined,
- handler_count => 0,
- handler_refs => #{},
- max_handlers => gateway_nats_pool_conn:max_handlers(),
- monitor_ref => undefined,
- connecting_pid => undefined,
- connecting_token => undefined,
- connecting_timer => undefined,
- rpc_enabled => RpcEnabled,
- extra_subscriptions => #{}
- }}.
+ {ok,
+ publish_status(#{
+ conn => undefined,
+ ready => false,
+ rpc_subs => #{},
+ subscribe_timer => undefined,
+ subscribe_failures => 0,
+ handler_count => 0,
+ handler_refs => #{},
+ max_handlers => gateway_nats_pool_conn:max_handlers(),
+ monitor_ref => undefined,
+ connecting_pid => undefined,
+ connecting_token => undefined,
+ connecting_timer => undefined,
+ rpc_enabled => RpcEnabled,
+ extra_subscriptions => #{}
+ })}.
-spec handle_call(term(), gen_server:from(), map()) -> {reply, term(), map()}.
handle_call(get_connection, _From, #{conn := Conn} = State) ->
{reply, {ok, Conn}, State};
-handle_call({subscribe_subject, Subject, QueueGroup}, _From, #{conn := Conn} = State) when
- Conn =/= undefined, is_binary(Subject), is_binary(QueueGroup)
+handle_call({subscribe_subject, Subject, QueueGroup}, _From, State) when
+ is_binary(Subject), is_binary(QueueGroup)
->
- {Reply, NewState} = subscribe_extra_subject(Conn, Subject, QueueGroup, State),
+ {Reply, NewState} = subscribe_extra_subject(Subject, QueueGroup, State),
{reply, Reply, NewState};
-handle_call({subscribe_subject, _Subject, _QueueGroup}, _From, State) ->
- {reply, {error, not_connected}, State};
-handle_call(enable_rpc_subscription, _From, #{conn := Conn} = State) when
- Conn =/= undefined
-->
- NewState = gateway_nats_rpc_handler:do_subscribe(State#{rpc_enabled => true}),
- ignore_pool_call(fun gateway_nats_pool:enable_rpc_subscription/0),
- {reply, ok, NewState};
handle_call(enable_rpc_subscription, _From, State) ->
+ NewState = ensure_subscriptions(State#{rpc_enabled => true}),
ignore_pool_call(fun gateway_nats_pool:enable_rpc_subscription/0),
- {reply, ok, State#{rpc_enabled => true}};
+ {reply, ok, NewState};
handle_call(disable_rpc_subscription, _From, State) ->
{reply, ok, do_disable_rpc(State)};
handle_call(_Request, _From, State) ->
@@ -95,14 +103,21 @@ handle_info({nats_connect_result, Pid, {error, Reason}}, State) when is_pid(Pid)
handle_info({connect_timeout, Token}, State) when is_reference(Token) ->
{noreply, handle_connect_timeout(Token, State)};
handle_info({Conn, ready}, #{conn := Conn} = State) ->
- RpcState = gateway_nats_rpc_handler:do_subscribe(State),
- {noreply, subscribe_extra_subjects(Conn, RpcState)};
+ {noreply, handle_ready(State)};
+handle_info(resubscribe, State) ->
+ {noreply, ensure_subscriptions(State#{subscribe_timer => undefined})};
handle_info({Conn, closed}, #{conn := Conn} = State) ->
logger:warning("Gateway NATS RPC connection closed, reconnecting"),
{noreply, reconnect_after_disconnect(Conn, State)};
handle_info({Conn, {error, Reason}}, #{conn := Conn} = State) ->
logger:warning("Gateway NATS RPC connection error, reconnecting", #{reason => Reason}),
{noreply, reconnect_after_disconnect(Conn, State)};
+handle_info({Conn, closed}, State) when is_pid(Conn) ->
+ gateway_nats_pool_conn:ignore_disconnect(Conn),
+ {noreply, State};
+handle_info({Conn, {error, _Reason}}, State) when is_pid(Conn) ->
+ gateway_nats_pool_conn:ignore_disconnect(Conn),
+ {noreply, State};
handle_info({Conn, _Sid, {msg, Subject, Payload, MsgOpts}}, #{conn := Conn} = State) when
is_binary(Subject), is_binary(Payload), is_map(MsgOpts)
->
@@ -134,11 +149,13 @@ handle_handler_down(MRef, HRefs, HandlerCount, State) ->
-spec terminate(term(), map()) -> ok.
terminate(_Reason, #{conn := Conn} = State) when Conn =/= undefined ->
+ persistent_term:erase(?STATUS_KEY),
cleanup_connect_attempt(State),
gateway_nats_pool_conn:ignore_disconnect(Conn),
logger:info("Gateway NATS RPC subscriber stopped"),
ok;
terminate(_Reason, State) ->
+ persistent_term:erase(?STATUS_KEY),
cleanup_connect_attempt(State),
ok.
@@ -168,15 +185,16 @@ reconnect_after_disconnect(Conn, State) ->
schedule_conn_reconnect(State) ->
cancel_connect_timer(State),
gateway_nats_rpc_handler:schedule_reconnect(
- State#{
+ publish_status(State#{
conn => undefined,
- sub => undefined,
+ ready => false,
+ rpc_subs => #{},
monitor_ref => undefined,
connecting_pid => undefined,
connecting_token => undefined,
connecting_timer => undefined,
extra_subscriptions => clear_extra_subscription_sids(State)
- }
+ })
).
-spec handle_connect_ok(pid(), nats:conn(), map()) -> map().
@@ -184,16 +202,14 @@ handle_connect_ok(Pid, Conn, State) ->
case maps:find(connecting_pid, State) of
{ok, Pid} ->
cleanup_old(State),
- cancel_connect_timer(State),
MRef = nats:monitor(Conn),
- logger:info("Gateway NATS RPC connected"),
- State#{
+ OpenState = State#{
conn => Conn,
+ ready => false,
monitor_ref => MRef,
- connecting_pid => undefined,
- connecting_token => undefined,
- connecting_timer => undefined
- };
+ connecting_pid => undefined
+ },
+ ready_if_already(gateway_nats_pool_conn:conn_ready(Conn), OpenState);
_ ->
gateway_nats_pool_conn:ignore_disconnect(Conn),
State
@@ -245,49 +261,123 @@ handle_connect_error(Pid, Reason, State) ->
State
end.
+-spec ready_if_already(boolean(), map()) -> map().
+ready_if_already(true, State) ->
+ handle_ready(State);
+ready_if_already(false, State) ->
+ State.
+
+-spec handle_ready(map()) -> map().
+handle_ready(State) ->
+ cancel_connect_timer(State),
+ logger:info("Gateway NATS RPC connected"),
+ ensure_subscriptions(State#{
+ ready => true,
+ connecting_token => undefined,
+ connecting_timer => undefined
+ }).
+
-spec handle_connect_timeout(reference(), map()) -> map().
-handle_connect_timeout(Token, State) ->
- case maps:get(connecting_token, State, undefined) of
- Token ->
- Pid = maps:get(connecting_pid, State, undefined),
- gateway_nats_pool_conn:maybe_kill_connect_pid(Pid),
- logger:warning("Gateway NATS RPC connect attempt timed out, retrying"),
- gateway_nats_rpc_handler:do_connect(
- State#{
- connecting_pid => undefined,
- connecting_token => undefined,
- connecting_timer => undefined
- }
- );
- _ ->
- State
+handle_connect_timeout(Token, #{connecting_token := Token, conn := Conn} = State) when
+ Conn =/= undefined
+->
+ logger:warning("Gateway NATS RPC connection did not become ready, reconnecting"),
+ reconnect_after_disconnect(conn_from_term(Conn), State);
+handle_connect_timeout(Token, #{connecting_token := Token} = State) ->
+ Pid = maps:get(connecting_pid, State, undefined),
+ gateway_nats_pool_conn:maybe_kill_connect_pid(Pid),
+ logger:warning("Gateway NATS RPC connect attempt timed out, retrying"),
+ gateway_nats_rpc_handler:do_connect(
+ State#{
+ connecting_pid => undefined,
+ connecting_token => undefined,
+ connecting_timer => undefined
+ }
+ );
+handle_connect_timeout(_Token, State) ->
+ State.
+
+-spec ensure_subscriptions(map()) -> map().
+ensure_subscriptions(#{ready := true, conn := Conn} = State) when Conn =/= undefined ->
+ {RpcState, RpcDone} = gateway_nats_rpc_handler:do_subscribe(State),
+ {ExtraState, ExtraDone} = subscribe_extra_subjects(Conn, RpcState),
+ publish_status(subscribe_outcome(RpcDone andalso ExtraDone, ExtraState));
+ensure_subscriptions(State) ->
+ publish_status(State).
+
+-spec subscribe_outcome(boolean(), map()) -> map().
+subscribe_outcome(true, State) ->
+ State#{subscribe_failures => 0};
+subscribe_outcome(false, #{subscribe_timer := TimerRef} = State) when is_reference(TimerRef) ->
+ State;
+subscribe_outcome(false, State) ->
+ Failures = maps:get(subscribe_failures, State, 0) + 1,
+ Delay = gateway_nats_pool_conn:subscribe_retry_delay(Failures),
+ logger:warning("Gateway NATS RPC will retry its subscriptions", #{
+ attempt => Failures, delay_ms => Delay
+ }),
+ State#{
+ subscribe_failures => Failures,
+ subscribe_timer => erlang:send_after(Delay, self(), resubscribe)
+ }.
+
+-spec publish_status(map()) -> map().
+publish_status(State) ->
+ Status = subscription_status(State),
+ case persistent_term:get(?STATUS_KEY, undefined) of
+ Status -> ok;
+ _ -> persistent_term:put(?STATUS_KEY, Status)
+ end,
+ State.
+
+-spec subscription_status(map()) -> not_required | ready | missing.
+subscription_status(#{rpc_enabled := false}) ->
+ not_required;
+subscription_status(State) ->
+ Role = fluxer_gateway_sup:current_role(),
+ case gateway_nats_rpc_handler:rpc_subjects_for_role(Role) of
+ [] -> not_required;
+ Subjects -> subjects_status(Subjects, State)
end.
--spec subscribe_extra_subject(nats:conn(), binary(), binary(), map()) ->
+-spec subjects_status([binary()], map()) -> ready | missing.
+subjects_status(Subjects, #{ready := true, rpc_subs := Subs}) ->
+ Subscribed = lists:all(fun(Subject) -> maps:is_key(Subject, Subs) end, Subjects),
+ case Subscribed of
+ true -> ready;
+ false -> missing
+ end;
+subjects_status(_Subjects, _State) ->
+ missing.
+
+-spec subscribe_extra_subject(binary(), binary(), map()) ->
{{ok, term()} | {error, term()}, map()}.
-subscribe_extra_subject(Conn, Subject, QueueGroup, #{extra_subscriptions := Extra} = State) ->
+subscribe_extra_subject(Subject, QueueGroup, #{extra_subscriptions := Extra} = State) ->
Key = {Subject, QueueGroup},
case maps:get(Key, Extra, undefined) of
#{sid := Sid} when Sid =/= undefined ->
{{ok, Sid}, State};
_ ->
- subscribe_new_extra_subject(Conn, Subject, QueueGroup, Key, Extra, State)
+ Wanted = #{subject => Subject, queue_group => QueueGroup, sid => undefined},
+ subscribe_wanted_extra_subject(Key, State#{
+ extra_subscriptions => Extra#{Key => Wanted}
+ })
end.
--spec subscribe_new_extra_subject(nats:conn(), binary(), binary(), term(), map(), map()) ->
+-spec subscribe_wanted_extra_subject(term(), map()) ->
{{ok, term()} | {error, term()}, map()}.
-subscribe_new_extra_subject(Conn, Subject, QueueGroup, Key, Extra, State) ->
- case nats:sub(Conn, Subject, subscription_opts(QueueGroup)) of
- {ok, Sid} ->
- NewExtra = Extra#{
- Key => #{subject => Subject, queue_group => QueueGroup, sid => Sid}
- },
- {{ok, Sid}, State#{extra_subscriptions => NewExtra}};
- {error, Reason} ->
- {{error, Reason}, State}
- end.
-
--spec subscribe_extra_subjects(nats:conn(), map()) -> map().
+subscribe_wanted_extra_subject(Key, #{ready := true, conn := Conn} = State) when
+ Conn =/= undefined
+->
+ NewState = ensure_subscriptions(State),
+ case maps:get(Key, maps:get(extra_subscriptions, NewState)) of
+ #{sid := Sid} when Sid =/= undefined -> {{ok, Sid}, NewState};
+ _ -> {{error, subscribe_failed}, NewState}
+ end;
+subscribe_wanted_extra_subject(_Key, State) ->
+ {{error, not_connected}, State}.
+
+-spec subscribe_extra_subjects(nats:conn(), map()) -> {map(), boolean()}.
subscribe_extra_subjects(Conn, #{extra_subscriptions := Extra} = State) ->
NewExtra = maps:fold(
fun(Key, Sub, Acc) ->
@@ -296,7 +386,8 @@ subscribe_extra_subjects(Conn, #{extra_subscriptions := Extra} = State) ->
Extra,
Extra
),
- State#{extra_subscriptions => NewExtra}.
+ Done = lists:all(fun(#{sid := Sid}) -> Sid =/= undefined end, maps:values(NewExtra)),
+ {State#{extra_subscriptions => NewExtra}, Done}.
-spec subscribe_recorded_extra_subject(nats:conn(), term(), map(), map()) -> map().
subscribe_recorded_extra_subject(_Conn, _Key, #{sid := Sid}, Acc) when Sid =/= undefined ->
@@ -304,12 +395,12 @@ subscribe_recorded_extra_subject(_Conn, _Key, #{sid := Sid}, Acc) when Sid =/= u
subscribe_recorded_extra_subject(
Conn, Key, #{subject := Subject, queue_group := QueueGroup} = Sub, Acc
) ->
- case nats:sub(Conn, Subject, subscription_opts(QueueGroup)) of
+ case gateway_nats_pool_conn:guarded_sub(Conn, Subject, subscription_opts(QueueGroup)) of
{ok, Sid} ->
- notify_resubscribed(Subject),
- Acc#{Key => Sub#{sid => Sid}};
+ notify_if_resubscribed(maps:get(subscribed_before, Sub, false), Subject),
+ Acc#{Key => Sub#{sid => Sid, subscribed_before => true}};
{error, Reason} ->
- logger:error("Gateway NATS RPC failed to resubscribe extra subject", #{
+ logger:error("Gateway NATS RPC failed to subscribe extra subject", #{
subject => Subject, reason => Reason
}),
Acc#{Key => Sub#{sid => undefined}}
@@ -351,6 +442,12 @@ subject_owner(<<"config.gateway.rollout">>) -> gateway_rollout_config;
subject_owner(<<"config.channel.threads">>) -> channel_threads_config;
subject_owner(_Subject) -> undefined.
+-spec notify_if_resubscribed(boolean(), binary()) -> ok.
+notify_if_resubscribed(true, Subject) ->
+ notify_resubscribed(Subject);
+notify_if_resubscribed(false, _Subject) ->
+ ok.
+
-spec notify_resubscribed(binary()) -> ok.
notify_resubscribed(Subject) ->
case subject_owner(Subject) of
@@ -416,7 +513,7 @@ do_dispatch_rpc(ReplyTo, _Subject, _Payload, HC, MaxH, _HRefs, State) when HC >=
<<"ok">> => false, <<"error">> => <<"overloaded">>
})
),
- _ = gateway_nats_pool:pub_reply(ReplyTo, ErrResp),
+ _ = spawn(fun() -> gateway_nats_pool:pub_reply(ReplyTo, ErrResp) end),
State;
do_dispatch_rpc(ReplyTo, Subject, Payload, HC, _MaxH, HRefs, State) ->
{_Pid, MRef} = spawn_monitor(fun() ->
@@ -425,23 +522,16 @@ do_dispatch_rpc(ReplyTo, Subject, Payload, HC, _MaxH, HRefs, State) ->
State#{handler_count => HC + 1, handler_refs => HRefs#{MRef => true}}.
-spec do_disable_rpc(map()) -> map().
-do_disable_rpc(#{conn := Conn, sub := Sids} = State) when
- Conn =/= undefined, is_list(Sids)
-->
- [gateway_nats_pool_conn:ignore_unsub(Conn, Sid) || Sid <- Sids],
- ignore_pool_call(fun gateway_nats_pool:disable_rpc_subscription/0),
- logger:info("Gateway NATS RPC unsubscribed", #{subscription_count => length(Sids)}),
- State#{sub => undefined, rpc_enabled => false};
-do_disable_rpc(#{conn := Conn, sub := Sid} = State) when
- Conn =/= undefined, Sid =/= undefined
+do_disable_rpc(#{conn := Conn, rpc_subs := Subs} = State) when
+ Conn =/= undefined, map_size(Subs) > 0
->
- gateway_nats_pool_conn:ignore_unsub(Conn, Sid),
+ [gateway_nats_pool_conn:ignore_unsub(Conn, Sid) || Sid <- maps:values(Subs)],
ignore_pool_call(fun gateway_nats_pool:disable_rpc_subscription/0),
- logger:info("Gateway NATS RPC unsubscribed"),
- State#{sub => undefined, rpc_enabled => false};
+ logger:info("Gateway NATS RPC unsubscribed", #{subscription_count => map_size(Subs)}),
+ publish_status(State#{rpc_subs => #{}, rpc_enabled => false});
do_disable_rpc(State) ->
ignore_pool_call(fun gateway_nats_pool:disable_rpc_subscription/0),
- State#{rpc_enabled => false}.
+ publish_status(State#{rpc_subs => #{}, rpc_enabled => false}).
-spec conn_from_term(term()) -> nats:conn().
conn_from_term(Conn) when is_pid(Conn) ->
diff --git a/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl b/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl
index 983e1df4b..93b887be0 100644
--- a/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl
+++ b/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl
@@ -157,56 +157,54 @@ connect_and_notify(Host, Port, Opts, Parent) ->
end,
ok.
--spec do_subscribe(map()) -> map().
-do_subscribe(#{conn := Conn, rpc_enabled := RpcEnabled} = State) when
+-spec do_subscribe(map()) -> {map(), boolean()}.
+do_subscribe(#{conn := Conn, ready := true, rpc_enabled := true} = State) when
Conn =/= undefined
->
- case RpcEnabled of
- false ->
- logger:info(
- "Gateway NATS RPC connected but subscription disabled"
- " (FLUXER_GATEWAY_NATS_RPC_ENABLED=false)"
- ),
- State;
- _ ->
- subscribe_for_role(Conn, State)
- end;
+ subscribe_for_role(Conn, State);
+do_subscribe(#{conn := Conn, ready := true} = State) when Conn =/= undefined ->
+ logger:info(
+ "Gateway NATS RPC connected but subscription disabled"
+ " (FLUXER_GATEWAY_NATS_RPC_ENABLED=false)"
+ ),
+ {State, true};
do_subscribe(State) ->
- State.
+ {State, false}.
--spec subscribe_for_role(nats:conn(), map()) -> map().
+-spec subscribe_for_role(nats:conn(), map()) -> {map(), boolean()}.
subscribe_for_role(Conn, State) ->
Role = fluxer_gateway_sup:current_role(),
- Subjects = rpc_subjects_for_role(Role),
- case Subjects of
+ Subs = maps:get(rpc_subs, State, #{}),
+ case [Subject || Subject <- rpc_subjects_for_role(Role), not maps:is_key(Subject, Subs)] of
[] ->
- logger:info("Gateway NATS RPC connected but not subscribing", #{role => Role}),
- State;
- _ ->
- Sids = subscribe_subjects(Conn, Role, Subjects),
- State#{sub => Sids}
+ {State, true};
+ Missing ->
+ NewSubs = subscribe_missing(Conn, Role, Missing, Subs),
+ {
+ State#{rpc_subs => NewSubs},
+ map_size(NewSubs) - map_size(Subs) =:= length(Missing)
+ }
end.
--spec subscribe_subjects(nats:conn(), atom(), [binary()]) -> [term()].
-subscribe_subjects(Conn, Role, Subjects) ->
- lists:filtermap(
- fun(Subject) ->
- subscribe_subject(Conn, Role, Subject)
- end,
- Subjects
+-spec subscribe_missing(nats:conn(), atom(), [binary()], map()) -> map().
+subscribe_missing(Conn, Role, Missing, Subs) ->
+ lists:foldl(
+ fun(Subject, Acc) -> subscribe_subject(Conn, Role, Subject, Acc) end,
+ Subs,
+ Missing
).
--spec subscribe_subject(nats:conn(), atom(), binary()) -> {true, term()} | false.
-subscribe_subject(Conn, Role, Subject) ->
- case nats:sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of
+-spec subscribe_subject(nats:conn(), atom(), binary(), map()) -> map().
+subscribe_subject(Conn, Role, Subject, Subs) ->
+ case gateway_nats_pool_conn:guarded_sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of
{ok, Sid} ->
logger:info("Gateway NATS RPC subscribed", #{subject => Subject, role => Role}),
- {true, Sid};
+ Subs#{Subject => Sid};
{error, Reason} ->
logger:error("Gateway NATS RPC failed to subscribe", #{
subject => Subject, reason => Reason
}),
- false
+ Subs
end.
-spec schedule_reconnect(map()) -> map().
diff --git a/fluxer_gateway/src/gateway/health_handler.erl b/fluxer_gateway/src/gateway/health_handler.erl
index 25bb59271..f464920ec 100644
--- a/fluxer_gateway/src/gateway/health_handler.erl
+++ b/fluxer_gateway/src/gateway/health_handler.erl
@@ -41,14 +41,16 @@ readiness_response(Req) ->
false ->
{403, <<"FORBIDDEN">>};
true ->
- readiness_status(gateway_node_router:is_ready())
+ readiness_status(gateway_node_router:is_ready(), gateway_nats_rpc:nats_ready())
end.
--spec readiness_status(boolean()) -> {200 | 503, binary()}.
-readiness_status(true) ->
- {200, <<"OK">>};
-readiness_status(false) ->
- {503, <<"DRAINING">>}.
+-spec readiness_status(boolean(), boolean()) -> {200 | 503, binary()}.
+readiness_status(false, _NatsReady) ->
+ {503, <<"DRAINING">>};
+readiness_status(true, false) ->
+ {503, <<"NATS_UNAVAILABLE">>};
+readiness_status(true, true) ->
+ {200, <<"OK">>}.
-spec drain_response(cowboy_req:req()) -> {200 | 403, binary()}.
drain_response(Req) ->
diff --git a/fluxer_gateway/test/gateway_nats_resubscribe_tests.erl b/fluxer_gateway/test/gateway_nats_resubscribe_tests.erl
new file mode 100644
index 000000000..412dbb3f8
--- /dev/null
+++ b/fluxer_gateway/test/gateway_nats_resubscribe_tests.erl
@@ -0,0 +1,291 @@
+%% SPDX-License-Identifier: AGPL-3.0-or-later
+
+-module(gateway_nats_resubscribe_tests).
+-typing([eqwalizer]).
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+-define(CONFIG_KEY, {fluxer_gateway, runtime_config}).
+-define(RPC_SUBJECT, <<"rpc.gateway.>">>).
+-define(EXTRA_SUBJECT, <<"config.gateway.rollout">>).
+-define(INFO,
+ <<"INFO {\"server_id\":\"fake\",\"version\":\"2.10.0\",\"proto\":1,",
+ "\"max_payload\":1048576,\"headers\":true}\r\n">>
+).
+
+subscribe_retry_delay_backs_off_and_stops_growing_test() ->
+ Delays = [gateway_nats_pool_conn:subscribe_retry_delay(N) || N <- lists:seq(1, 8)],
+ ?assertEqual([250, 500, 1000, 2000, 4000, 5000, 5000, 5000], Delays),
+ ?assertEqual(250, gateway_nats_pool_conn:subscribe_retry_delay(0)).
+
+closed_notice_before_connect_result_is_not_lost_test_() ->
+ with_env(fun closed_notice_before_connect_result_is_not_lost/1).
+
+closed_notice_before_connect_result_is_not_lost(Port) ->
+ State0 = rpc_state(),
+ Conn = failed_conn(Port),
+ Attempt = dead_pid(),
+ {noreply, State1} = gateway_nats_rpc:handle_info(
+ {Conn, closed}, State0#{connecting_pid => Attempt}
+ ),
+ ?assertNot(is_process_alive(Conn)),
+ {noreply, State2} = gateway_nats_rpc:handle_info(
+ {nats_connect_result, Attempt, {ok, Conn}}, State1
+ ),
+ MRef = maps:get(monitor_ref, State2),
+ Down = receive_down(MRef, Conn),
+ {noreply, State3} = gateway_nats_rpc:handle_info(Down, State2),
+ ?assertEqual(undefined, maps:get(conn, State3)),
+ ?assertEqual(connect, receive_message(connect, 3000)).
+
+ready_notice_before_connect_result_still_subscribes_test_() ->
+ with_env(fun ready_notice_before_connect_result_still_subscribes/1).
+
+ready_notice_before_connect_result_still_subscribes(Port) ->
+ Server = start_fake_nats(Port),
+ State0 = rpc_state(),
+ {ok, Conn} = connect(Port),
+ ready = receive_message({Conn, ready}, 5000, ready),
+ Attempt = dead_pid(),
+ {noreply, State1} = gateway_nats_rpc:handle_info(
+ {nats_connect_result, Attempt, {ok, Conn}}, State0#{connecting_pid => Attempt}
+ ),
+ ?assert(maps:get(ready, State1)),
+ ?assert(maps:is_key(?RPC_SUBJECT, maps:get(rpc_subs, State1))),
+ ?assertEqual(ok, await_subs(?RPC_SUBJECT, 1, 5000)),
+ gateway_nats_pool_conn:ignore_disconnect(Conn),
+ stop_fake_nats(Server).
+
+connection_that_never_becomes_ready_is_replaced_test_() ->
+ with_env(fun connection_that_never_becomes_ready_is_replaced/1).
+
+connection_that_never_becomes_ready_is_replaced(Port) ->
+ State0 = rpc_state(),
+ Conn = failed_conn(Port),
+ Attempt = dead_pid(),
+ Token = make_ref(),
+ {noreply, State1} = gateway_nats_rpc:handle_info(
+ {nats_connect_result, Attempt, {ok, Conn}},
+ State0#{connecting_pid => Attempt, connecting_token => Token}
+ ),
+ ?assertEqual(Conn, maps:get(conn, State1)),
+ ?assertNot(maps:get(ready, State1)),
+ {noreply, State2} = gateway_nats_rpc:handle_info({connect_timeout, Token}, State1),
+ ?assertEqual(undefined, maps:get(conn, State2)),
+ ?assertNot(is_process_alive(Conn)),
+ ?assertEqual(connect, receive_message(connect, 3000)).
+
+subscribe_on_a_stuck_connection_gives_up_test_() ->
+ with_env(fun subscribe_on_a_stuck_connection_gives_up/1).
+
+subscribe_on_a_stuck_connection_gives_up(Port) ->
+ Conn = failed_conn(Port),
+ Result = gateway_nats_pool_conn:guarded_sub(Conn, ?RPC_SUBJECT, #{}),
+ ?assertMatch({error, _}, Result),
+ ?assertNot(is_process_alive(Conn)).
+
+gateway_recovers_when_nats_starts_late_and_restarts_test_() ->
+ with_env(fun gateway_recovers_when_nats_starts_late_and_restarts/1).
+
+gateway_recovers_when_nats_starts_late_and_restarts(Port) ->
+ {ok, Rpc} = gateway_nats_rpc:start_link(),
+ {ok, Pool} = gateway_nats_pool:start_link(),
+ timer:sleep(1500),
+ ?assertEqual({error, no_connection}, gateway_nats_pool_conn:get_pool_conn()),
+ ?assertNot(gateway_nats_rpc:nats_ready()),
+ ?assertEqual({error, not_connected}, gateway_nats_rpc:subscribe(?EXTRA_SUBJECT, <<>>)),
+ Server = start_fake_nats(Port),
+ ?assertEqual(ok, await_subs(?RPC_SUBJECT, 3, 10000)),
+ ?assertEqual(ok, await_subs(?EXTRA_SUBJECT, 1, 10000)),
+ ?assertEqual(ok, wait_until(fun gateway_nats_rpc:nats_ready/0, 5000)),
+ stop_fake_nats(Server),
+ ?assertEqual(ok, wait_until(fun() -> not gateway_nats_rpc:nats_ready() end, 5000)),
+ flush_subs(),
+ Restarted = start_fake_nats(Port),
+ ?assertEqual(ok, await_subs(?RPC_SUBJECT, 3, 10000)),
+ ?assertEqual(ok, await_subs(?EXTRA_SUBJECT, 1, 10000)),
+ ?assertEqual(ok, wait_until(fun gateway_nats_rpc:nats_ready/0, 5000)),
+ ok = gen_server:stop(Pool),
+ ok = gen_server:stop(Rpc),
+ stop_fake_nats(Restarted).
+
+with_env(Test) ->
+ {setup, fun setup/0, fun cleanup/1, fun({Port, _Saved}) ->
+ {timeout, 60, fun() -> run_trapping_exits(Test, Port) end}
+ end}.
+
+run_trapping_exits(Test, Port) ->
+ TrapExit = process_flag(trap_exit, true),
+ try
+ Test(Port)
+ after
+ process_flag(trap_exit, TrapExit),
+ flush()
+ end.
+
+setup() ->
+ Saved = #{
+ config => persistent_term:get(?CONFIG_KEY, undefined),
+ logger => logger:get_primary_config()
+ },
+ ok = logger:set_primary_config(level, none),
+ Port = free_port(),
+ persistent_term:put(?CONFIG_KEY, #{
+ nats_core_url => "nats://127.0.0.1:" ++ integer_to_list(Port),
+ nats_auth_token => "",
+ nats_rpc_enabled => true,
+ nats_pool_size => 2,
+ gateway_role => all,
+ gateway_nats_rpc_max_handlers => 16
+ }),
+ {Port, Saved}.
+
+cleanup({_Port, #{config := Config, logger := Logger}}) ->
+ restore_config(Config),
+ persistent_term:erase({gateway_nats_rpc, subscription_status}),
+ ok = logger:set_primary_config(Logger).
+
+restore_config(undefined) ->
+ persistent_term:erase(?CONFIG_KEY);
+restore_config(Config) ->
+ persistent_term:put(?CONFIG_KEY, Config).
+
+rpc_state() ->
+ {ok, State} = gateway_nats_rpc:init([]),
+ connect = receive_message(connect, 1000),
+ State.
+
+connect(Port) ->
+ Server = #{scheme => <<"nats">>, host => "127.0.0.1", port => Port, family => inet},
+ nats:connect(Server, #{buffer_size => 0}).
+
+failed_conn(Port) ->
+ {ok, Conn} = connect(Port),
+ closed = receive_message({Conn, closed}, 5000, closed),
+ flush_conn(Conn),
+ ?assert(is_process_alive(Conn)),
+ ?assertEqual(false, nats:is_ready(Conn)),
+ Conn.
+
+dead_pid() ->
+ {Pid, MRef} = spawn_monitor(fun() -> ok end),
+ receive
+ {'DOWN', MRef, process, Pid, _} -> Pid
+ end.
+
+free_port() ->
+ {ok, Socket} = gen_tcp:listen(0, [{ip, {127, 0, 0, 1}}]),
+ {ok, Port} = inet:port(Socket),
+ ok = gen_tcp:close(Socket),
+ Port.
+
+start_fake_nats(Port) ->
+ Owner = self(),
+ Pid = spawn(fun() -> listen(Port, Owner) end),
+ listening = receive_message({fake_nats_listening, Pid}, 5000, listening),
+ Pid.
+
+stop_fake_nats(Pid) ->
+ MRef = erlang:monitor(process, Pid),
+ exit(Pid, kill),
+ receive
+ {'DOWN', MRef, process, Pid, _} -> ok
+ end.
+
+listen(Port, Owner) ->
+ Opts = [binary, {packet, line}, {active, false}, {reuseaddr, true}, {ip, {127, 0, 0, 1}}],
+ {ok, Listen} = gen_tcp:listen(Port, Opts),
+ Owner ! {fake_nats_listening, self()},
+ accept_loop(Listen, Owner).
+
+accept_loop(Listen, Owner) ->
+ {ok, Socket} = gen_tcp:accept(Listen),
+ Handler = spawn_link(fun() -> serve(Owner) end),
+ ok = gen_tcp:controlling_process(Socket, Handler),
+ Handler ! {socket, Socket},
+ accept_loop(Listen, Owner).
+
+serve(Owner) ->
+ receive
+ {socket, Socket} ->
+ ok = gen_tcp:send(Socket, ?INFO),
+ serve_loop(Socket, Owner)
+ end.
+
+serve_loop(Socket, Owner) ->
+ case gen_tcp:recv(Socket, 0) of
+ {ok, Line} ->
+ handle_line(Socket, Owner, Line),
+ serve_loop(Socket, Owner);
+ {error, _Reason} ->
+ ok
+ end.
+
+handle_line(Socket, _Owner, <<"PING", _/binary>>) ->
+ gen_tcp:send(Socket, <<"PONG\r\n">>);
+handle_line(_Socket, Owner, <<"SUB ", Rest/binary>>) ->
+ [Subject | _] = binary:split(Rest, <<" ">>),
+ Owner ! {fake_nats_sub, Subject},
+ ok;
+handle_line(_Socket, _Owner, _Line) ->
+ ok.
+
+await_subs(_Subject, 0, _Timeout) ->
+ ok;
+await_subs(Subject, Count, Timeout) ->
+ receive
+ {fake_nats_sub, Subject} -> await_subs(Subject, Count - 1, Timeout)
+ after Timeout ->
+ {missing, Subject, Count}
+ end.
+
+wait_until(Check, Timeout) when Timeout =< 0 ->
+ {timeout, Check()};
+wait_until(Check, Timeout) ->
+ case Check() of
+ true ->
+ ok;
+ false ->
+ timer:sleep(50),
+ wait_until(Check, Timeout - 50)
+ end.
+
+receive_down(MRef, Conn) ->
+ receive
+ {'DOWN', MRef, process, Conn, _} = Down -> Down
+ after 3000 ->
+ error(no_down_message)
+ end.
+
+receive_message(Message, Timeout) ->
+ receive_message(Message, Timeout, Message).
+
+receive_message(Message, Timeout, Result) ->
+ receive
+ Message -> Result
+ after Timeout ->
+ {timeout, Message}
+ end.
+
+flush_conn(Conn) ->
+ receive
+ {Conn, _} -> flush_conn(Conn)
+ after 100 ->
+ ok
+ end.
+
+flush_subs() ->
+ receive
+ {fake_nats_sub, _Subject} -> flush_subs()
+ after 0 ->
+ ok
+ end.
+
+flush() ->
+ receive
+ _ -> flush()
+ after 0 ->
+ ok
+ end.
+
+-endif.
diff --git a/fluxer_rexctl/internal/stack/services_test.go b/fluxer_rexctl/internal/stack/services_test.go
index 37cf9fdc0..d7ee8c6d3 100644
--- a/fluxer_rexctl/internal/stack/services_test.go
+++ b/fluxer_rexctl/internal/stack/services_test.go
@@ -6,6 +6,8 @@ import (
"slices"
"strings"
"testing"
+
+ "go.yaml.in/yaml/v4"
)
func TestPortsDefaults(t *testing.T) {
@@ -61,3 +63,37 @@ func TestComposeVoicePortsShareOneVar(t *testing.T) {
t.Error("docker-compose.yml should keep the livekit signal port 7880 fixed and internal")
}
}
+
+func TestNatsClientsWaitForHealthyNats(t *testing.T) {
+ data, err := ReadFile(Embedded(), "docker-compose.yml")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var compose struct {
+ Services map[string]struct {
+ DependsOn map[string]struct {
+ Condition string `yaml:"condition"`
+ } `yaml:"depends_on"`
+ Healthcheck struct {
+ Test []string `yaml:"test"`
+ } `yaml:"healthcheck"`
+ } `yaml:"services"`
+ }
+ if err := yaml.Unmarshal(data, &compose); err != nil {
+ t.Fatal(err)
+ }
+ for _, name := range []string{"api", "worker", "gateway", "push", "media-proxy", "snowflakes", "users", "gifs", "messages", "unfurl"} {
+ service, ok := compose.Services[name]
+ if !ok {
+ t.Errorf("docker-compose.yml has no %s service", name)
+ continue
+ }
+ if got := service.DependsOn["nats"].Condition; got != "service_healthy" {
+ t.Errorf("%s depends on nats with condition %q, want service_healthy", name, got)
+ }
+ }
+ check := strings.Join(compose.Services["gateway"].Healthcheck.Test, " ")
+ if !strings.Contains(check, "GET /_health/ready ") {
+ t.Errorf("gateway healthcheck %q should ask /_health/ready, which fails while the gateway's NATS subscriptions are missing", check)
+ }
+}