A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(gateway): recover NATS subscriptions after failed connects (#3690)
Hampusb27c4e9c1b48
diff --git a/fluxer_gateway/src/gateway/gateway_nats_pool.erl b/fluxer_gateway/src/gateway/gateway_nats_pool.erlindex 234ace3e2..8147ed5ce 100644--- a/fluxer_gateway/src/gateway/gateway_nats_pool.erl+++ b/fluxer_gateway/src/gateway/gateway_nats_pool.erl@@ -5,6 +5,7 @@-behaviour(gen_server).-export([start_link/0, pub/2, pub_reply/2, reply_publish_failures/0, get_pool_status/0]).+-export([has_ready_connection/0]).-export([enable_rpc_subscription/0, disable_rpc_subscription/0]).-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).@@ -18,7 +19,8 @@handle_connect_ok_entry/5,accept_connect_ok/4,ignore_stale_connect_ok/2,- handle_ready/4,+ handle_ready/2,+ accept_connect_timeout/3,handle_conn_event_down/2,handle_slot_monitor_down/5,conn_from_term/1@@ -79,6 +81,13 @@ reply_failure_counter() ->Refend.+-spec has_ready_connection() -> boolean().+has_ready_connection() ->+ case gateway_nats_pool_conn:get_pool_conn() of+ {ok, _Conn} -> true;+ {error, _Reason} -> false+ end.+-spec get_pool_status() -> map().get_pool_status() ->gen_server:call(?MODULE, get_pool_status).@@ -107,6 +116,7 @@ init([]) ->connecting => #{},rpc_enabled => gateway_nats_pool_conn:nats_rpc_enabled(),subs => #{},+ sub_failures => #{},handler_count => 0,handler_refs => #{},max_handlers => gateway_nats_pool_conn:max_handlers()@@ -144,8 +154,10 @@ handle_info({reconnect_slot, Idx}, #{slots := Slots} = State) whenis_integer(Idx), Idx >= 0->{noreply, reconnect_slot(Idx, Slots, State)};-handle_info({Conn, ready}, #{rpc_enabled := RpcEnabled, slots := Slots} = State) ->- {noreply, handle_ready(conn_from_term(Conn), RpcEnabled, Slots, State)};+handle_info({Conn, ready}, State) ->+ {noreply, handle_ready(conn_from_term(Conn), State)};+handle_info({resubscribe_slot, Idx}, State) when is_integer(Idx), Idx >= 0 ->+ {noreply, resubscribe_slot(Idx, State)};handle_info({_Conn, _Sid, {msg, Subject, Payload, MsgOpts}}, State) whenis_binary(Subject), is_binary(Payload), is_map(MsgOpts)->@@ -229,8 +241,17 @@ enable_subscriptions(Slots, State) ->-spec enable_slot_subscription(non_neg_integer(), nats:conn() | undefined, map()) -> map().enable_slot_subscription(_Idx, undefined, State) ->State;-enable_slot_subscription(Idx, Conn, State) ->- gateway_nats_pool_conn:subscribe_slot(Idx, Conn, State).+enable_slot_subscription(Idx, Conn, #{connecting := Connecting} = State) ->+ case maps:is_key(Idx, Connecting) of+ true -> State;+ false -> gateway_nats_pool_conn:subscribe_slot(Idx, Conn, State)+ end.++-spec resubscribe_slot(non_neg_integer(), map()) -> map().+resubscribe_slot(_Idx, #{rpc_enabled := false} = State) ->+ State;+resubscribe_slot(Idx, #{slots := Slots} = State) ->+ enable_slot_subscription(Idx, maps:get(Idx, Slots, undefined), State).-spec disable_subscriptions(map(), map()) -> ok.disable_subscriptions(Slots, Subs) ->@@ -273,25 +294,34 @@ handle_connect_ok_entry(Idx, Pid, Entry, Conn, State) ->-spec accept_connect_ok(non_neg_integer(), map(), nats:conn(), map()) -> map().accept_connect_ok(Idx, Entry, Conn, State) ->- #{- connecting := Connecting,- slots := Slots,- monitors := Monitors,- pool_size := PoolSize- } = State,- cancel_connect_timer(Entry),+ #{connecting := Connecting, slots := Slots, monitors := Monitors} = State,cleanup_old_conn(maps:get(Idx, Slots, undefined)),CleanMon = remove_slot_monitors(Idx, Monitors),MRef = nats:monitor(Conn),- NewSlots = Slots#{Idx => Conn},- gateway_nats_pool_conn:update_persistent_term(NewSlots, PoolSize),gateway_nats_pool_conn:reset_failure_count(Idx),- logger:info("Gateway NATS pool slot connected", #{slot => Idx}),- State#{- slots => NewSlots,+ NewState = State#{+ slots => Slots#{Idx => Conn},monitors => CleanMon#{MRef => Idx},- connecting => maps:remove(Idx, Connecting)- }.+ connecting => Connecting#{Idx => connecting_entry_with_conn(Entry, Conn)}+ },+ case gateway_nats_pool_conn:conn_ready(Conn) of+ true -> mark_slot_ready(Idx, NewState);+ false -> NewState+ end.++-spec connecting_entry_with_conn(term(), nats:conn()) -> map().+connecting_entry_with_conn(Entry, Conn) when is_map(Entry) ->+ Entry#{conn => Conn};+connecting_entry_with_conn(Pid, Conn) ->+ #{pid => Pid, conn => Conn}.++-spec mark_slot_ready(non_neg_integer(), map()) -> map().+mark_slot_ready(Idx, #{connecting := Connecting} = State) ->+ cancel_connect_timer(maps:get(Idx, Connecting, undefined)),+ ReadyState = State#{connecting => maps:remove(Idx, Connecting)},+ gateway_nats_pool_conn:publish_ready_slots(ReadyState),+ logger:info("Gateway NATS pool slot connected", #{slot => Idx}),+ resubscribe_slot(Idx, ReadyState).-spec ignore_stale_connect_ok(nats:conn(), map()) -> map().ignore_stale_connect_ok(Conn, State) ->@@ -331,20 +361,12 @@ accept_connect_error(Idx, Entry, Reason, #{connecting := Connecting} = State) ->gateway_nats_pool_conn:schedule_reconnect(Idx, self()),State#{connecting => maps:remove(Idx, Connecting)}.--spec handle_ready(nats:conn(), boolean(), map(), map()) -> map().-handle_ready(_Conn, false, _Slots, State) ->- State;-handle_ready(Conn, true, Slots, State) ->+-spec handle_ready(nats:conn(), map()) -> map().+handle_ready(Conn, #{connecting := Connecting} = State) ->case gateway_nats_pool_conn:find_slot_by_conn(Conn, State) ofundefined -> State;- Idx -> handle_ready_slot(Idx, Conn, Slots, State)- end.---spec handle_ready_slot(non_neg_integer(), nats:conn(), map(), map()) -> map().-handle_ready_slot(Idx, Conn, Slots, State) ->- case maps:find(Idx, Slots) of- {ok, Conn} -> gateway_nats_pool_conn:subscribe_slot(Idx, Conn, State);- _ -> State+ Idx when is_map_key(Idx, Connecting) -> mark_slot_ready(Idx, State);+ Idx -> resubscribe_slot(Idx, State)end.-spec handle_rpc_msg(binary(), binary(), map(), map()) -> map().@@ -372,7 +394,7 @@ dispatch_rpc_msg(Subject, Payload, MsgOpts, HC, MaxH, HRefs, State) ->undefined ->State;ReplyTo when HC >= MaxH ->- _ = send_overloaded(ReplyTo),+ send_overloaded_async(ReplyTo),State;ReplyTo ->{_Pid, MRef} = spawn_rpc_handler(Subject, Payload, ReplyTo),@@ -385,6 +407,11 @@ spawn_rpc_handler(Subject, Payload, ReplyTo) ->gateway_nats_rpc_handler:handle_rpc_request(Subject, Payload, ReplyTo)end).+-spec send_overloaded_async(binary()) -> ok.+send_overloaded_async(ReplyTo) ->+ _ = spawn(fun() -> send_overloaded(ReplyTo) end),+ ok.+-spec send_overloaded(binary()) -> ok | {error, term()}.send_overloaded(ReplyTo) ->Resp = iolist_to_binary(@@ -474,6 +501,9 @@ handle_legacy_connect_timeout(Idx, #{connecting := Connecting} = State) ->end.-spec accept_connect_timeout(non_neg_integer(), term(), map()) -> map().+accept_connect_timeout(Idx, #{conn := Conn}, #{pool_size := PoolSize} = State) ->+ logger:warning("Gateway NATS pool slot connection did not become ready", #{slot => Idx}),+ gateway_nats_pool_conn:handle_conn_down(Idx, conn_from_term(Conn), PoolSize, State);accept_connect_timeout(Idx, Entry, #{connecting := Connecting} = State) ->gateway_nats_pool_conn:maybe_kill_connect_pid(connecting_entry_pid(Entry)),logger:warning("Gateway NATS pool slot connect attempt timed out", #{slot => Idx}),diff --git a/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erl b/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erlindex 32f93776b..68da05d58 100644--- a/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erl+++ b/fluxer_gateway/src/gateway/gateway_nats_pool_conn.erl@@ -12,7 +12,11 @@update_persistent_term/2,schedule_reconnect/2,reset_failure_count/1,+ publish_ready_slots/1,subscribe_slot/3,+ guarded_sub/3,+ conn_ready/1,+ subscribe_retry_delay/1,unsubscribe_slot_sids/2,is_rpc_subject/1,pool_size/0,@@ -37,6 +41,9 @@-define(CIRCUIT_BREAKER_THRESHOLD, 5).-define(CIRCUIT_BREAKER_COOLDOWN_MS, 15000).-define(FAILURE_COUNTS_KEY, {gateway_nats_pool, failure_counts}).+-define(SUBSCRIBE_RETRY_BASE_MS, 250).+-define(SUBSCRIBE_RETRY_MAX_MS, 5000).+-define(SUBSCRIBE_TIMEOUT_MS, 5000).-spec get_pool_conn() -> {ok, nats:conn()} | {error, no_connection}.get_pool_conn() ->@@ -134,17 +141,37 @@ address_family(Host) ->handle_conn_down(Idx,Conn,- PoolSize,- #{slots := Slots, subs := Subs, monitors := Monitors} = State+ _PoolSize,+ #{slots := Slots, subs := Subs, monitors := Monitors, connecting := Connecting} = State) ->logger:warning("Gateway NATS pool slot connection lost, reconnecting", #{slot => Idx}),ignore_disconnect(Conn),+ cancel_entry_timer(maps:get(Idx, Connecting, undefined)),CleanMonitors = maps:filter(fun(_Ref, SlotIdx) -> SlotIdx =/= Idx end, Monitors),- NewSlots = Slots#{Idx => undefined},- NewSubs = maps:remove(Idx, Subs),- update_persistent_term(NewSlots, PoolSize),+ NewState = State#{+ slots => Slots#{Idx => undefined},+ subs => maps:remove(Idx, Subs),+ monitors => CleanMonitors,+ connecting => maps:remove(Idx, Connecting)+ },+ publish_ready_slots(NewState),schedule_reconnect(Idx, self()),- State#{slots => NewSlots, subs => NewSubs, monitors => CleanMonitors}.+ NewState.++-spec cancel_entry_timer(term()) -> ok.+cancel_entry_timer(#{timer := TimerRef}) when is_reference(TimerRef) ->+ _ = erlang:cancel_timer(TimerRef, [{async, true}, {info, false}]),+ ok;+cancel_entry_timer(_) ->+ ok.++-spec publish_ready_slots(map()) -> ok.+publish_ready_slots(#{slots := Slots, connecting := Connecting, pool_size := PoolSize}) ->+ Ready = maps:filter(+ fun(Idx, Conn) -> Conn =/= undefined andalso not maps:is_key(Idx, Connecting) end,+ Slots+ ),+ update_persistent_term(Ready, PoolSize).-spec find_slot_by_conn(nats:conn(), map()) -> non_neg_integer() | undefined.find_slot_by_conn(Conn, #{slots := Slots}) ->@@ -206,37 +233,84 @@ subscribe_slot(Idx, Conn, #{subs := Subs} = State) ->subscribe_new_slot(Idx, Conn, Subs, State) ->Role = fluxer_gateway_sup:current_role(),Subjects = gateway_nats_rpc_handler:rpc_subjects_for_role(Role),- Sids = subscribe_slot_subjects(Idx, Conn, Role, Subjects),- case Sids of- [] -> State;- _ -> State#{subs => Subs#{Idx => Sids}}+ Results = [subscribe_slot_subject(Idx, Conn, Role, Subject) || Subject <- Subjects],+ Sids = [Sid || {ok, Sid} <- Results],+ case length(Sids) =:= length(Subjects) of+ true when Sids =:= [] ->+ State;+ true ->+ State#{subs => Subs#{Idx => Sids}, sub_failures => clear_sub_failures(Idx, State)};+ false ->+ unsubscribe_slot_sids(Conn, Sids),+ schedule_slot_resubscribe(Idx, State)end.--spec subscribe_slot_subjects(non_neg_integer(), nats:conn(), atom(), [binary()]) -> [term()].-subscribe_slot_subjects(Idx, Conn, Role, Subjects) ->- lists:filtermap(- fun(Subject) ->- subscribe_slot_subject(Idx, Conn, Role, Subject)- end,- Subjects- ).--spec subscribe_slot_subject(non_neg_integer(), nats:conn(), atom(), binary()) ->- {true, term()} | false.+ {ok, nats:sid()} | {error, term()}.subscribe_slot_subject(Idx, Conn, Role, Subject) ->- case nats:sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of+ case guarded_sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of{ok, Sid} ->logger:info("Gateway NATS pool slot ~p subscribed to ~s (role=~p)",[Idx, Subject, Role]),- {true, Sid};+ {ok, Sid};{error, Reason} ->logger:error("Gateway NATS pool slot ~p failed to subscribe to ~s: ~p",[Idx, Subject, Reason]),- false+ {error, Reason}+ end.++-spec clear_sub_failures(non_neg_integer(), map()) -> map().+clear_sub_failures(Idx, State) ->+ maps:remove(Idx, maps:get(sub_failures, State, #{})).++-spec schedule_slot_resubscribe(non_neg_integer(), map()) -> map().+schedule_slot_resubscribe(Idx, State) ->+ Failures = maps:get(sub_failures, State, #{}),+ Count = maps:get(Idx, Failures, 0) + 1,+ Delay = subscribe_retry_delay(Count),+ logger:warning("Gateway NATS pool slot will retry its subscriptions", #{+ slot => Idx, attempt => Count, delay_ms => Delay+ }),+ erlang:send_after(Delay, self(), {resubscribe_slot, Idx}),+ State#{sub_failures => Failures#{Idx => Count}}.++-spec subscribe_retry_delay(term()) -> pos_integer().+subscribe_retry_delay(Failures) when is_integer(Failures), Failures > 5 ->+ ?SUBSCRIBE_RETRY_MAX_MS;+subscribe_retry_delay(Failures) when is_integer(Failures), Failures > 0 ->+ min(?SUBSCRIBE_RETRY_MAX_MS, ?SUBSCRIBE_RETRY_BASE_MS bsl (Failures - 1));+subscribe_retry_delay(_Failures) ->+ ?SUBSCRIBE_RETRY_BASE_MS.++-spec guarded_sub(nats:conn(), binary(), map()) -> {ok, nats:sid()} | {error, term()}.+guarded_sub(Conn, Subject, Opts) ->+ Guard = spawn(fun() -> disconnect_unless_done(Conn) end),+ try nats:sub(Conn, Subject, Opts) of+ Result -> Result+ catch+ exit:Reason -> {error, {exit, Reason}}+ after+ Guard ! done+ end.++-spec disconnect_unless_done(nats:conn()) -> ok.+disconnect_unless_done(Conn) ->+ receive+ done -> ok+ after ?SUBSCRIBE_TIMEOUT_MS ->+ logger:warning("Gateway NATS subscribe did not finish in time, closing the connection"),+ ignore_disconnect(Conn)+ end.++-spec conn_ready(nats:conn()) -> boolean().+conn_ready(Conn) ->+ case nats:is_ready(Conn) of+ true -> true;+ _ -> falseend.-spec unsubscribe_slot_sids(nats:conn(), term()) -> ok.diff --git a/fluxer_gateway/src/gateway/gateway_nats_rpc.erl b/fluxer_gateway/src/gateway/gateway_nats_rpc.erlindex 4b9e4d4b9..e05275f05 100644--- a/fluxer_gateway/src/gateway/gateway_nats_rpc.erl+++ b/fluxer_gateway/src/gateway/gateway_nats_rpc.erl@@ -5,7 +5,7 @@-behaviour(gen_server).-export([start_link/0, get_connection/0, subscribe/2]).--export([enable_rpc_subscription/0, disable_rpc_subscription/0]).+-export([enable_rpc_subscription/0, disable_rpc_subscription/0, nats_ready/0]).-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).@@ -13,10 +13,13 @@{no_opaque, [handle_info/2,handle_connect_ok/3,+ handle_connect_timeout/2,conn_from_term/1]}).+-define(STATUS_KEY, {?MODULE, subscription_status}).+-spec start_link() -> gen_server:start_ret().start_link() ->gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).@@ -37,45 +40,50 @@ enable_rpc_subscription() ->disable_rpc_subscription() ->gen_server:call(?MODULE, disable_rpc_subscription, 5000).+-spec nats_ready() -> boolean().+nats_ready() ->+ case persistent_term:get(?STATUS_KEY, missing) of+ not_required -> true;+ ready -> gateway_nats_pool:has_ready_connection();+ _ -> false+ end.+-spec init([]) -> {ok, map()}.init([]) ->process_flag(trap_exit, true),erlang:process_flag(fullsweep_after, 10),RpcEnabled = gateway_nats_pool_conn:nats_rpc_enabled(),self() ! connect,- {ok, #{- conn => undefined,- sub => undefined,- handler_count => 0,- handler_refs => #{},- max_handlers => gateway_nats_pool_conn:max_handlers(),- monitor_ref => undefined,- connecting_pid => undefined,- connecting_token => undefined,- connecting_timer => undefined,- rpc_enabled => RpcEnabled,- extra_subscriptions => #{}- }}.+ {ok,+ publish_status(#{+ conn => undefined,+ ready => false,+ rpc_subs => #{},+ subscribe_timer => undefined,+ subscribe_failures => 0,+ handler_count => 0,+ handler_refs => #{},+ max_handlers => gateway_nats_pool_conn:max_handlers(),+ monitor_ref => undefined,+ connecting_pid => undefined,+ connecting_token => undefined,+ connecting_timer => undefined,+ rpc_enabled => RpcEnabled,+ extra_subscriptions => #{}+ })}.-spec handle_call(term(), gen_server:from(), map()) -> {reply, term(), map()}.handle_call(get_connection, _From, #{conn := Conn} = State) ->{reply, {ok, Conn}, State};-handle_call({subscribe_subject, Subject, QueueGroup}, _From, #{conn := Conn} = State) when- Conn =/= undefined, is_binary(Subject), is_binary(QueueGroup)+handle_call({subscribe_subject, Subject, QueueGroup}, _From, State) when+ is_binary(Subject), is_binary(QueueGroup)->- {Reply, NewState} = subscribe_extra_subject(Conn, Subject, QueueGroup, State),+ {Reply, NewState} = subscribe_extra_subject(Subject, QueueGroup, State),{reply, Reply, NewState};-handle_call({subscribe_subject, _Subject, _QueueGroup}, _From, State) ->- {reply, {error, not_connected}, State};-handle_call(enable_rpc_subscription, _From, #{conn := Conn} = State) when- Conn =/= undefined-->- NewState = gateway_nats_rpc_handler:do_subscribe(State#{rpc_enabled => true}),- ignore_pool_call(fun gateway_nats_pool:enable_rpc_subscription/0),- {reply, ok, NewState};handle_call(enable_rpc_subscription, _From, State) ->+ NewState = ensure_subscriptions(State#{rpc_enabled => true}),ignore_pool_call(fun gateway_nats_pool:enable_rpc_subscription/0),- {reply, ok, State#{rpc_enabled => true}};+ {reply, ok, NewState};handle_call(disable_rpc_subscription, _From, State) ->{reply, ok, do_disable_rpc(State)};handle_call(_Request, _From, State) ->@@ -95,14 +103,21 @@ handle_info({nats_connect_result, Pid, {error, Reason}}, State) when is_pid(Pid)handle_info({connect_timeout, Token}, State) when is_reference(Token) ->{noreply, handle_connect_timeout(Token, State)};handle_info({Conn, ready}, #{conn := Conn} = State) ->- RpcState = gateway_nats_rpc_handler:do_subscribe(State),- {noreply, subscribe_extra_subjects(Conn, RpcState)};+ {noreply, handle_ready(State)};+handle_info(resubscribe, State) ->+ {noreply, ensure_subscriptions(State#{subscribe_timer => undefined})};handle_info({Conn, closed}, #{conn := Conn} = State) ->logger:warning("Gateway NATS RPC connection closed, reconnecting"),{noreply, reconnect_after_disconnect(Conn, State)};handle_info({Conn, {error, Reason}}, #{conn := Conn} = State) ->logger:warning("Gateway NATS RPC connection error, reconnecting", #{reason => Reason}),{noreply, reconnect_after_disconnect(Conn, State)};+handle_info({Conn, closed}, State) when is_pid(Conn) ->+ gateway_nats_pool_conn:ignore_disconnect(Conn),+ {noreply, State};+handle_info({Conn, {error, _Reason}}, State) when is_pid(Conn) ->+ gateway_nats_pool_conn:ignore_disconnect(Conn),+ {noreply, State};handle_info({Conn, _Sid, {msg, Subject, Payload, MsgOpts}}, #{conn := Conn} = State) whenis_binary(Subject), is_binary(Payload), is_map(MsgOpts)->@@ -134,11 +149,13 @@ handle_handler_down(MRef, HRefs, HandlerCount, State) ->-spec terminate(term(), map()) -> ok.terminate(_Reason, #{conn := Conn} = State) when Conn =/= undefined ->+ persistent_term:erase(?STATUS_KEY),cleanup_connect_attempt(State),gateway_nats_pool_conn:ignore_disconnect(Conn),logger:info("Gateway NATS RPC subscriber stopped"),ok;terminate(_Reason, State) ->+ persistent_term:erase(?STATUS_KEY),cleanup_connect_attempt(State),ok.@@ -168,15 +185,16 @@ reconnect_after_disconnect(Conn, State) ->schedule_conn_reconnect(State) ->cancel_connect_timer(State),gateway_nats_rpc_handler:schedule_reconnect(- State#{+ publish_status(State#{conn => undefined,- sub => undefined,+ ready => false,+ rpc_subs => #{},monitor_ref => undefined,connecting_pid => undefined,connecting_token => undefined,connecting_timer => undefined,extra_subscriptions => clear_extra_subscription_sids(State)- }+ })).-spec handle_connect_ok(pid(), nats:conn(), map()) -> map().@@ -184,16 +202,14 @@ handle_connect_ok(Pid, Conn, State) ->case maps:find(connecting_pid, State) of{ok, Pid} ->cleanup_old(State),- cancel_connect_timer(State),MRef = nats:monitor(Conn),- logger:info("Gateway NATS RPC connected"),- State#{+ OpenState = State#{conn => Conn,+ ready => false,monitor_ref => MRef,- connecting_pid => undefined,- connecting_token => undefined,- connecting_timer => undefined- };+ connecting_pid => undefined+ },+ ready_if_already(gateway_nats_pool_conn:conn_ready(Conn), OpenState);_ ->gateway_nats_pool_conn:ignore_disconnect(Conn),State@@ -245,49 +261,123 @@ handle_connect_error(Pid, Reason, State) ->Stateend.+-spec ready_if_already(boolean(), map()) -> map().+ready_if_already(true, State) ->+ handle_ready(State);+ready_if_already(false, State) ->+ State.++-spec handle_ready(map()) -> map().+handle_ready(State) ->+ cancel_connect_timer(State),+ logger:info("Gateway NATS RPC connected"),+ ensure_subscriptions(State#{+ ready => true,+ connecting_token => undefined,+ connecting_timer => undefined+ }).+-spec handle_connect_timeout(reference(), map()) -> map().-handle_connect_timeout(Token, State) ->- case maps:get(connecting_token, State, undefined) of- Token ->- Pid = maps:get(connecting_pid, State, undefined),- gateway_nats_pool_conn:maybe_kill_connect_pid(Pid),- logger:warning("Gateway NATS RPC connect attempt timed out, retrying"),- gateway_nats_rpc_handler:do_connect(- State#{- connecting_pid => undefined,- connecting_token => undefined,- connecting_timer => undefined- }- );- _ ->- State+handle_connect_timeout(Token, #{connecting_token := Token, conn := Conn} = State) when+ Conn =/= undefined+->+ logger:warning("Gateway NATS RPC connection did not become ready, reconnecting"),+ reconnect_after_disconnect(conn_from_term(Conn), State);+handle_connect_timeout(Token, #{connecting_token := Token} = State) ->+ Pid = maps:get(connecting_pid, State, undefined),+ gateway_nats_pool_conn:maybe_kill_connect_pid(Pid),+ logger:warning("Gateway NATS RPC connect attempt timed out, retrying"),+ gateway_nats_rpc_handler:do_connect(+ State#{+ connecting_pid => undefined,+ connecting_token => undefined,+ connecting_timer => undefined+ }+ );+handle_connect_timeout(_Token, State) ->+ State.++-spec ensure_subscriptions(map()) -> map().+ensure_subscriptions(#{ready := true, conn := Conn} = State) when Conn =/= undefined ->+ {RpcState, RpcDone} = gateway_nats_rpc_handler:do_subscribe(State),+ {ExtraState, ExtraDone} = subscribe_extra_subjects(Conn, RpcState),+ publish_status(subscribe_outcome(RpcDone andalso ExtraDone, ExtraState));+ensure_subscriptions(State) ->+ publish_status(State).++-spec subscribe_outcome(boolean(), map()) -> map().+subscribe_outcome(true, State) ->+ State#{subscribe_failures => 0};+subscribe_outcome(false, #{subscribe_timer := TimerRef} = State) when is_reference(TimerRef) ->+ State;+subscribe_outcome(false, State) ->+ Failures = maps:get(subscribe_failures, State, 0) + 1,+ Delay = gateway_nats_pool_conn:subscribe_retry_delay(Failures),+ logger:warning("Gateway NATS RPC will retry its subscriptions", #{+ attempt => Failures, delay_ms => Delay+ }),+ State#{+ subscribe_failures => Failures,+ subscribe_timer => erlang:send_after(Delay, self(), resubscribe)+ }.++-spec publish_status(map()) -> map().+publish_status(State) ->+ Status = subscription_status(State),+ case persistent_term:get(?STATUS_KEY, undefined) of+ Status -> ok;+ _ -> persistent_term:put(?STATUS_KEY, Status)+ end,+ State.++-spec subscription_status(map()) -> not_required | ready | missing.+subscription_status(#{rpc_enabled := false}) ->+ not_required;+subscription_status(State) ->+ Role = fluxer_gateway_sup:current_role(),+ case gateway_nats_rpc_handler:rpc_subjects_for_role(Role) of+ [] -> not_required;+ Subjects -> subjects_status(Subjects, State)end.--spec subscribe_extra_subject(nats:conn(), binary(), binary(), map()) ->+-spec subjects_status([binary()], map()) -> ready | missing.+subjects_status(Subjects, #{ready := true, rpc_subs := Subs}) ->+ Subscribed = lists:all(fun(Subject) -> maps:is_key(Subject, Subs) end, Subjects),+ case Subscribed of+ true -> ready;+ false -> missing+ end;+subjects_status(_Subjects, _State) ->+ missing.++-spec subscribe_extra_subject(binary(), binary(), map()) ->{{ok, term()} | {error, term()}, map()}.-subscribe_extra_subject(Conn, Subject, QueueGroup, #{extra_subscriptions := Extra} = State) ->+subscribe_extra_subject(Subject, QueueGroup, #{extra_subscriptions := Extra} = State) ->Key = {Subject, QueueGroup},case maps:get(Key, Extra, undefined) of#{sid := Sid} when Sid =/= undefined ->{{ok, Sid}, State};_ ->- subscribe_new_extra_subject(Conn, Subject, QueueGroup, Key, Extra, State)+ Wanted = #{subject => Subject, queue_group => QueueGroup, sid => undefined},+ subscribe_wanted_extra_subject(Key, State#{+ extra_subscriptions => Extra#{Key => Wanted}+ })end.--spec subscribe_new_extra_subject(nats:conn(), binary(), binary(), term(), map(), map()) ->+-spec subscribe_wanted_extra_subject(term(), map()) ->{{ok, term()} | {error, term()}, map()}.-subscribe_new_extra_subject(Conn, Subject, QueueGroup, Key, Extra, State) ->- case nats:sub(Conn, Subject, subscription_opts(QueueGroup)) of- {ok, Sid} ->- NewExtra = Extra#{- Key => #{subject => Subject, queue_group => QueueGroup, sid => Sid}- },- {{ok, Sid}, State#{extra_subscriptions => NewExtra}};- {error, Reason} ->- {{error, Reason}, State}- end.---spec subscribe_extra_subjects(nats:conn(), map()) -> map().+subscribe_wanted_extra_subject(Key, #{ready := true, conn := Conn} = State) when+ Conn =/= undefined+->+ NewState = ensure_subscriptions(State),+ case maps:get(Key, maps:get(extra_subscriptions, NewState)) of+ #{sid := Sid} when Sid =/= undefined -> {{ok, Sid}, NewState};+ _ -> {{error, subscribe_failed}, NewState}+ end;+subscribe_wanted_extra_subject(_Key, State) ->+ {{error, not_connected}, State}.++-spec subscribe_extra_subjects(nats:conn(), map()) -> {map(), boolean()}.subscribe_extra_subjects(Conn, #{extra_subscriptions := Extra} = State) ->NewExtra = maps:fold(fun(Key, Sub, Acc) ->@@ -296,7 +386,8 @@ subscribe_extra_subjects(Conn, #{extra_subscriptions := Extra} = State) ->Extra,Extra),- State#{extra_subscriptions => NewExtra}.+ Done = lists:all(fun(#{sid := Sid}) -> Sid =/= undefined end, maps:values(NewExtra)),+ {State#{extra_subscriptions => NewExtra}, Done}.-spec subscribe_recorded_extra_subject(nats:conn(), term(), map(), map()) -> map().subscribe_recorded_extra_subject(_Conn, _Key, #{sid := Sid}, Acc) when Sid =/= undefined ->@@ -304,12 +395,12 @@ subscribe_recorded_extra_subject(_Conn, _Key, #{sid := Sid}, Acc) when Sid =/= usubscribe_recorded_extra_subject(Conn, Key, #{subject := Subject, queue_group := QueueGroup} = Sub, Acc) ->- case nats:sub(Conn, Subject, subscription_opts(QueueGroup)) of+ case gateway_nats_pool_conn:guarded_sub(Conn, Subject, subscription_opts(QueueGroup)) of{ok, Sid} ->- notify_resubscribed(Subject),- Acc#{Key => Sub#{sid => Sid}};+ notify_if_resubscribed(maps:get(subscribed_before, Sub, false), Subject),+ Acc#{Key => Sub#{sid => Sid, subscribed_before => true}};{error, Reason} ->- logger:error("Gateway NATS RPC failed to resubscribe extra subject", #{+ logger:error("Gateway NATS RPC failed to subscribe extra subject", #{subject => Subject, reason => Reason}),Acc#{Key => Sub#{sid => undefined}}@@ -351,6 +442,12 @@ subject_owner(<<"config.gateway.rollout">>) -> gateway_rollout_config;subject_owner(<<"config.channel.threads">>) -> channel_threads_config;subject_owner(_Subject) -> undefined.+-spec notify_if_resubscribed(boolean(), binary()) -> ok.+notify_if_resubscribed(true, Subject) ->+ notify_resubscribed(Subject);+notify_if_resubscribed(false, _Subject) ->+ ok.+-spec notify_resubscribed(binary()) -> ok.notify_resubscribed(Subject) ->case subject_owner(Subject) of@@ -416,7 +513,7 @@ do_dispatch_rpc(ReplyTo, _Subject, _Payload, HC, MaxH, _HRefs, State) when HC >=<<"ok">> => false, <<"error">> => <<"overloaded">>})),- _ = gateway_nats_pool:pub_reply(ReplyTo, ErrResp),+ _ = spawn(fun() -> gateway_nats_pool:pub_reply(ReplyTo, ErrResp) end),State;do_dispatch_rpc(ReplyTo, Subject, Payload, HC, _MaxH, HRefs, State) ->{_Pid, MRef} = spawn_monitor(fun() ->@@ -425,23 +522,16 @@ do_dispatch_rpc(ReplyTo, Subject, Payload, HC, _MaxH, HRefs, State) ->State#{handler_count => HC + 1, handler_refs => HRefs#{MRef => true}}.-spec do_disable_rpc(map()) -> map().-do_disable_rpc(#{conn := Conn, sub := Sids} = State) when- Conn =/= undefined, is_list(Sids)-->- [gateway_nats_pool_conn:ignore_unsub(Conn, Sid) || Sid <- Sids],- ignore_pool_call(fun gateway_nats_pool:disable_rpc_subscription/0),- logger:info("Gateway NATS RPC unsubscribed", #{subscription_count => length(Sids)}),- State#{sub => undefined, rpc_enabled => false};-do_disable_rpc(#{conn := Conn, sub := Sid} = State) when- Conn =/= undefined, Sid =/= undefined+do_disable_rpc(#{conn := Conn, rpc_subs := Subs} = State) when+ Conn =/= undefined, map_size(Subs) > 0->- gateway_nats_pool_conn:ignore_unsub(Conn, Sid),+ [gateway_nats_pool_conn:ignore_unsub(Conn, Sid) || Sid <- maps:values(Subs)],ignore_pool_call(fun gateway_nats_pool:disable_rpc_subscription/0),- logger:info("Gateway NATS RPC unsubscribed"),- State#{sub => undefined, rpc_enabled => false};+ logger:info("Gateway NATS RPC unsubscribed", #{subscription_count => map_size(Subs)}),+ publish_status(State#{rpc_subs => #{}, rpc_enabled => false});do_disable_rpc(State) ->ignore_pool_call(fun gateway_nats_pool:disable_rpc_subscription/0),- State#{rpc_enabled => false}.+ publish_status(State#{rpc_subs => #{}, rpc_enabled => false}).-spec conn_from_term(term()) -> nats:conn().conn_from_term(Conn) when is_pid(Conn) ->diff --git a/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl b/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erlindex 983e1df4b..93b887be0 100644--- a/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl+++ b/fluxer_gateway/src/gateway/gateway_nats_rpc_handler.erl@@ -157,56 +157,54 @@ connect_and_notify(Host, Port, Opts, Parent) ->end,ok.--spec do_subscribe(map()) -> map().-do_subscribe(#{conn := Conn, rpc_enabled := RpcEnabled} = State) when+-spec do_subscribe(map()) -> {map(), boolean()}.+do_subscribe(#{conn := Conn, ready := true, rpc_enabled := true} = State) whenConn =/= undefined->- case RpcEnabled of- false ->- logger:info(- "Gateway NATS RPC connected but subscription disabled"- " (FLUXER_GATEWAY_NATS_RPC_ENABLED=false)"- ),- State;- _ ->- subscribe_for_role(Conn, State)- end;+ subscribe_for_role(Conn, State);+do_subscribe(#{conn := Conn, ready := true} = State) when Conn =/= undefined ->+ logger:info(+ "Gateway NATS RPC connected but subscription disabled"+ " (FLUXER_GATEWAY_NATS_RPC_ENABLED=false)"+ ),+ {State, true};do_subscribe(State) ->- State.+ {State, false}.--spec subscribe_for_role(nats:conn(), map()) -> map().+-spec subscribe_for_role(nats:conn(), map()) -> {map(), boolean()}.subscribe_for_role(Conn, State) ->Role = fluxer_gateway_sup:current_role(),- Subjects = rpc_subjects_for_role(Role),- case Subjects of+ Subs = maps:get(rpc_subs, State, #{}),+ case [Subject || Subject <- rpc_subjects_for_role(Role), not maps:is_key(Subject, Subs)] of[] ->- logger:info("Gateway NATS RPC connected but not subscribing", #{role => Role}),- State;- _ ->- Sids = subscribe_subjects(Conn, Role, Subjects),- State#{sub => Sids}+ {State, true};+ Missing ->+ NewSubs = subscribe_missing(Conn, Role, Missing, Subs),+ {+ State#{rpc_subs => NewSubs},+ map_size(NewSubs) - map_size(Subs) =:= length(Missing)+ }end.--spec subscribe_subjects(nats:conn(), atom(), [binary()]) -> [term()].-subscribe_subjects(Conn, Role, Subjects) ->- lists:filtermap(- fun(Subject) ->- subscribe_subject(Conn, Role, Subject)- end,- Subjects+-spec subscribe_missing(nats:conn(), atom(), [binary()], map()) -> map().+subscribe_missing(Conn, Role, Missing, Subs) ->+ lists:foldl(+ fun(Subject, Acc) -> subscribe_subject(Conn, Role, Subject, Acc) end,+ Subs,+ Missing).--spec subscribe_subject(nats:conn(), atom(), binary()) -> {true, term()} | false.-subscribe_subject(Conn, Role, Subject) ->- case nats:sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of+-spec subscribe_subject(nats:conn(), atom(), binary(), map()) -> map().+subscribe_subject(Conn, Role, Subject, Subs) ->+ case gateway_nats_pool_conn:guarded_sub(Conn, Subject, #{queue_group => ?QUEUE_GROUP}) of{ok, Sid} ->logger:info("Gateway NATS RPC subscribed", #{subject => Subject, role => Role}),- {true, Sid};+ Subs#{Subject => Sid};{error, Reason} ->logger:error("Gateway NATS RPC failed to subscribe", #{subject => Subject, reason => Reason}),- false+ Subsend.-spec schedule_reconnect(map()) -> map().diff --git a/fluxer_gateway/src/gateway/health_handler.erl b/fluxer_gateway/src/gateway/health_handler.erlindex 25bb59271..f464920ec 100644--- a/fluxer_gateway/src/gateway/health_handler.erl+++ b/fluxer_gateway/src/gateway/health_handler.erl@@ -41,14 +41,16 @@ readiness_response(Req) ->false ->{403, <<"FORBIDDEN">>};true ->- readiness_status(gateway_node_router:is_ready())+ readiness_status(gateway_node_router:is_ready(), gateway_nats_rpc:nats_ready())end.--spec readiness_status(boolean()) -> {200 | 503, binary()}.-readiness_status(true) ->- {200, <<"OK">>};-readiness_status(false) ->- {503, <<"DRAINING">>}.+-spec readiness_status(boolean(), boolean()) -> {200 | 503, binary()}.+readiness_status(false, _NatsReady) ->+ {503, <<"DRAINING">>};+readiness_status(true, false) ->+ {503, <<"NATS_UNAVAILABLE">>};+readiness_status(true, true) ->+ {200, <<"OK">>}.-spec drain_response(cowboy_req:req()) -> {200 | 403, binary()}.drain_response(Req) ->diff --git a/fluxer_gateway/test/gateway_nats_resubscribe_tests.erl b/fluxer_gateway/test/gateway_nats_resubscribe_tests.erlnew file mode 100644index 000000000..412dbb3f8--- /dev/null+++ b/fluxer_gateway/test/gateway_nats_resubscribe_tests.erl@@ -0,0 +1,291 @@+%% SPDX-License-Identifier: AGPL-3.0-or-later++-module(gateway_nats_resubscribe_tests).+-typing([eqwalizer]).+-ifdef(TEST).+-include_lib("eunit/include/eunit.hrl").++-define(CONFIG_KEY, {fluxer_gateway, runtime_config}).+-define(RPC_SUBJECT, <<"rpc.gateway.>">>).+-define(EXTRA_SUBJECT, <<"config.gateway.rollout">>).+-define(INFO,+ <<"INFO {\"server_id\":\"fake\",\"version\":\"2.10.0\",\"proto\":1,",+ "\"max_payload\":1048576,\"headers\":true}\r\n">>+).++subscribe_retry_delay_backs_off_and_stops_growing_test() ->+ Delays = [gateway_nats_pool_conn:subscribe_retry_delay(N) || N <- lists:seq(1, 8)],+ ?assertEqual([250, 500, 1000, 2000, 4000, 5000, 5000, 5000], Delays),+ ?assertEqual(250, gateway_nats_pool_conn:subscribe_retry_delay(0)).++closed_notice_before_connect_result_is_not_lost_test_() ->+ with_env(fun closed_notice_before_connect_result_is_not_lost/1).++closed_notice_before_connect_result_is_not_lost(Port) ->+ State0 = rpc_state(),+ Conn = failed_conn(Port),+ Attempt = dead_pid(),+ {noreply, State1} = gateway_nats_rpc:handle_info(+ {Conn, closed}, State0#{connecting_pid => Attempt}+ ),+ ?assertNot(is_process_alive(Conn)),+ {noreply, State2} = gateway_nats_rpc:handle_info(+ {nats_connect_result, Attempt, {ok, Conn}}, State1+ ),+ MRef = maps:get(monitor_ref, State2),+ Down = receive_down(MRef, Conn),+ {noreply, State3} = gateway_nats_rpc:handle_info(Down, State2),+ ?assertEqual(undefined, maps:get(conn, State3)),+ ?assertEqual(connect, receive_message(connect, 3000)).++ready_notice_before_connect_result_still_subscribes_test_() ->+ with_env(fun ready_notice_before_connect_result_still_subscribes/1).++ready_notice_before_connect_result_still_subscribes(Port) ->+ Server = start_fake_nats(Port),+ State0 = rpc_state(),+ {ok, Conn} = connect(Port),+ ready = receive_message({Conn, ready}, 5000, ready),+ Attempt = dead_pid(),+ {noreply, State1} = gateway_nats_rpc:handle_info(+ {nats_connect_result, Attempt, {ok, Conn}}, State0#{connecting_pid => Attempt}+ ),+ ?assert(maps:get(ready, State1)),+ ?assert(maps:is_key(?RPC_SUBJECT, maps:get(rpc_subs, State1))),+ ?assertEqual(ok, await_subs(?RPC_SUBJECT, 1, 5000)),+ gateway_nats_pool_conn:ignore_disconnect(Conn),+ stop_fake_nats(Server).++connection_that_never_becomes_ready_is_replaced_test_() ->+ with_env(fun connection_that_never_becomes_ready_is_replaced/1).++connection_that_never_becomes_ready_is_replaced(Port) ->+ State0 = rpc_state(),+ Conn = failed_conn(Port),+ Attempt = dead_pid(),+ Token = make_ref(),+ {noreply, State1} = gateway_nats_rpc:handle_info(+ {nats_connect_result, Attempt, {ok, Conn}},+ State0#{connecting_pid => Attempt, connecting_token => Token}+ ),+ ?assertEqual(Conn, maps:get(conn, State1)),+ ?assertNot(maps:get(ready, State1)),+ {noreply, State2} = gateway_nats_rpc:handle_info({connect_timeout, Token}, State1),+ ?assertEqual(undefined, maps:get(conn, State2)),+ ?assertNot(is_process_alive(Conn)),+ ?assertEqual(connect, receive_message(connect, 3000)).++subscribe_on_a_stuck_connection_gives_up_test_() ->+ with_env(fun subscribe_on_a_stuck_connection_gives_up/1).++subscribe_on_a_stuck_connection_gives_up(Port) ->+ Conn = failed_conn(Port),+ Result = gateway_nats_pool_conn:guarded_sub(Conn, ?RPC_SUBJECT, #{}),+ ?assertMatch({error, _}, Result),+ ?assertNot(is_process_alive(Conn)).++gateway_recovers_when_nats_starts_late_and_restarts_test_() ->+ with_env(fun gateway_recovers_when_nats_starts_late_and_restarts/1).++gateway_recovers_when_nats_starts_late_and_restarts(Port) ->+ {ok, Rpc} = gateway_nats_rpc:start_link(),+ {ok, Pool} = gateway_nats_pool:start_link(),+ timer:sleep(1500),+ ?assertEqual({error, no_connection}, gateway_nats_pool_conn:get_pool_conn()),+ ?assertNot(gateway_nats_rpc:nats_ready()),+ ?assertEqual({error, not_connected}, gateway_nats_rpc:subscribe(?EXTRA_SUBJECT, <<>>)),+ Server = start_fake_nats(Port),+ ?assertEqual(ok, await_subs(?RPC_SUBJECT, 3, 10000)),+ ?assertEqual(ok, await_subs(?EXTRA_SUBJECT, 1, 10000)),+ ?assertEqual(ok, wait_until(fun gateway_nats_rpc:nats_ready/0, 5000)),+ stop_fake_nats(Server),+ ?assertEqual(ok, wait_until(fun() -> not gateway_nats_rpc:nats_ready() end, 5000)),+ flush_subs(),+ Restarted = start_fake_nats(Port),+ ?assertEqual(ok, await_subs(?RPC_SUBJECT, 3, 10000)),+ ?assertEqual(ok, await_subs(?EXTRA_SUBJECT, 1, 10000)),+ ?assertEqual(ok, wait_until(fun gateway_nats_rpc:nats_ready/0, 5000)),+ ok = gen_server:stop(Pool),+ ok = gen_server:stop(Rpc),+ stop_fake_nats(Restarted).++with_env(Test) ->+ {setup, fun setup/0, fun cleanup/1, fun({Port, _Saved}) ->+ {timeout, 60, fun() -> run_trapping_exits(Test, Port) end}+ end}.++run_trapping_exits(Test, Port) ->+ TrapExit = process_flag(trap_exit, true),+ try+ Test(Port)+ after+ process_flag(trap_exit, TrapExit),+ flush()+ end.++setup() ->+ Saved = #{+ config => persistent_term:get(?CONFIG_KEY, undefined),+ logger => logger:get_primary_config()+ },+ ok = logger:set_primary_config(level, none),+ Port = free_port(),+ persistent_term:put(?CONFIG_KEY, #{+ nats_core_url => "nats://127.0.0.1:" ++ integer_to_list(Port),+ nats_auth_token => "",+ nats_rpc_enabled => true,+ nats_pool_size => 2,+ gateway_role => all,+ gateway_nats_rpc_max_handlers => 16+ }),+ {Port, Saved}.++cleanup({_Port, #{config := Config, logger := Logger}}) ->+ restore_config(Config),+ persistent_term:erase({gateway_nats_rpc, subscription_status}),+ ok = logger:set_primary_config(Logger).++restore_config(undefined) ->+ persistent_term:erase(?CONFIG_KEY);+restore_config(Config) ->+ persistent_term:put(?CONFIG_KEY, Config).++rpc_state() ->+ {ok, State} = gateway_nats_rpc:init([]),+ connect = receive_message(connect, 1000),+ State.++connect(Port) ->+ Server = #{scheme => <<"nats">>, host => "127.0.0.1", port => Port, family => inet},+ nats:connect(Server, #{buffer_size => 0}).++failed_conn(Port) ->+ {ok, Conn} = connect(Port),+ closed = receive_message({Conn, closed}, 5000, closed),+ flush_conn(Conn),+ ?assert(is_process_alive(Conn)),+ ?assertEqual(false, nats:is_ready(Conn)),+ Conn.++dead_pid() ->+ {Pid, MRef} = spawn_monitor(fun() -> ok end),+ receive+ {'DOWN', MRef, process, Pid, _} -> Pid+ end.++free_port() ->+ {ok, Socket} = gen_tcp:listen(0, [{ip, {127, 0, 0, 1}}]),+ {ok, Port} = inet:port(Socket),+ ok = gen_tcp:close(Socket),+ Port.++start_fake_nats(Port) ->+ Owner = self(),+ Pid = spawn(fun() -> listen(Port, Owner) end),+ listening = receive_message({fake_nats_listening, Pid}, 5000, listening),+ Pid.++stop_fake_nats(Pid) ->+ MRef = erlang:monitor(process, Pid),+ exit(Pid, kill),+ receive+ {'DOWN', MRef, process, Pid, _} -> ok+ end.++listen(Port, Owner) ->+ Opts = [binary, {packet, line}, {active, false}, {reuseaddr, true}, {ip, {127, 0, 0, 1}}],+ {ok, Listen} = gen_tcp:listen(Port, Opts),+ Owner ! {fake_nats_listening, self()},+ accept_loop(Listen, Owner).++accept_loop(Listen, Owner) ->+ {ok, Socket} = gen_tcp:accept(Listen),+ Handler = spawn_link(fun() -> serve(Owner) end),+ ok = gen_tcp:controlling_process(Socket, Handler),+ Handler ! {socket, Socket},+ accept_loop(Listen, Owner).++serve(Owner) ->+ receive+ {socket, Socket} ->+ ok = gen_tcp:send(Socket, ?INFO),+ serve_loop(Socket, Owner)+ end.++serve_loop(Socket, Owner) ->+ case gen_tcp:recv(Socket, 0) of+ {ok, Line} ->+ handle_line(Socket, Owner, Line),+ serve_loop(Socket, Owner);+ {error, _Reason} ->+ ok+ end.++handle_line(Socket, _Owner, <<"PING", _/binary>>) ->+ gen_tcp:send(Socket, <<"PONG\r\n">>);+handle_line(_Socket, Owner, <<"SUB ", Rest/binary>>) ->+ [Subject | _] = binary:split(Rest, <<" ">>),+ Owner ! {fake_nats_sub, Subject},+ ok;+handle_line(_Socket, _Owner, _Line) ->+ ok.++await_subs(_Subject, 0, _Timeout) ->+ ok;+await_subs(Subject, Count, Timeout) ->+ receive+ {fake_nats_sub, Subject} -> await_subs(Subject, Count - 1, Timeout)+ after Timeout ->+ {missing, Subject, Count}+ end.++wait_until(Check, Timeout) when Timeout =< 0 ->+ {timeout, Check()};+wait_until(Check, Timeout) ->+ case Check() of+ true ->+ ok;+ false ->+ timer:sleep(50),+ wait_until(Check, Timeout - 50)+ end.++receive_down(MRef, Conn) ->+ receive+ {'DOWN', MRef, process, Conn, _} = Down -> Down+ after 3000 ->+ error(no_down_message)+ end.++receive_message(Message, Timeout) ->+ receive_message(Message, Timeout, Message).++receive_message(Message, Timeout, Result) ->+ receive+ Message -> Result+ after Timeout ->+ {timeout, Message}+ end.++flush_conn(Conn) ->+ receive+ {Conn, _} -> flush_conn(Conn)+ after 100 ->+ ok+ end.++flush_subs() ->+ receive+ {fake_nats_sub, _Subject} -> flush_subs()+ after 0 ->+ ok+ end.++flush() ->+ receive+ _ -> flush()+ after 0 ->+ ok+ end.++-endif.diff --git a/fluxer_rexctl/internal/stack/services_test.go b/fluxer_rexctl/internal/stack/services_test.goindex 37cf9fdc0..d7ee8c6d3 100644--- a/fluxer_rexctl/internal/stack/services_test.go+++ b/fluxer_rexctl/internal/stack/services_test.go@@ -6,6 +6,8 @@ import ("slices""strings""testing"++ "go.yaml.in/yaml/v4")func TestPortsDefaults(t *testing.T) {@@ -61,3 +63,37 @@ func TestComposeVoicePortsShareOneVar(t *testing.T) {t.Error("docker-compose.yml should keep the livekit signal port 7880 fixed and internal")}}++func TestNatsClientsWaitForHealthyNats(t *testing.T) {+ data, err := ReadFile(Embedded(), "docker-compose.yml")+ if err != nil {+ t.Fatal(err)+ }+ var compose struct {+ Services map[string]struct {+ DependsOn map[string]struct {+ Condition string `yaml:"condition"`+ } `yaml:"depends_on"`+ Healthcheck struct {+ Test []string `yaml:"test"`+ } `yaml:"healthcheck"`+ } `yaml:"services"`+ }+ if err := yaml.Unmarshal(data, &compose); err != nil {+ t.Fatal(err)+ }+ for _, name := range []string{"api", "worker", "gateway", "push", "media-proxy", "snowflakes", "users", "gifs", "messages", "unfurl"} {+ service, ok := compose.Services[name]+ if !ok {+ t.Errorf("docker-compose.yml has no %s service", name)+ continue+ }+ if got := service.DependsOn["nats"].Condition; got != "service_healthy" {+ t.Errorf("%s depends on nats with condition %q, want service_healthy", name, got)+ }+ }+ check := strings.Join(compose.Services["gateway"].Healthcheck.Test, " ")+ if !strings.Contains(check, "GET /_health/ready ") {+ t.Errorf("gateway healthcheck %q should ask /_health/ready, which fails while the gateway's NATS subscriptions are missing", check)+ }+}