A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
refactor(gateway): move pin peer calls out, drop inline tests
Hampus03fc2ad028c2
diff --git a/fluxer_gateway/src/gateway/gateway_guild_pin_keeper.erl b/fluxer_gateway/src/gateway/gateway_guild_pin_keeper.erlindex 31aef62f6..a2afa1441 100644--- a/fluxer_gateway/src/gateway/gateway_guild_pin_keeper.erl+++ b/fluxer_gateway/src/gateway/gateway_guild_pin_keeper.erl@@ -16,9 +16,7 @@]).-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).--define(ROUTER, gateway_node_router).-define(PEER_NAME_PREFIX, "fluxer_gateway@").--define(LOADED_BEAM_NAME, "/tmp/fluxer-guild-pin/gateway_node_router.beam").-define(RPC_TIMEOUT_MS, 2000).-define(PATCH_TIMEOUT_MS, 10000).-define(ARM_PIN_TIMEOUT_MS, 3000).@@ -27,24 +25,19 @@-define(REPATCH_DELAY_MS, 100).-define(RECONNECT_INTERVAL_MS, 20).-define(RECONNECT_WINDOW_MS, 600000).--define(STOP_TIMEOUT_MS, 5000).-define(PREWARM_TIMEOUT_MS, 55000).-define(RELEASE_TIMEOUT_MS, 150000).-define(RELOAD_TIMEOUT_MS, 60000).-define(MAX_SOURCE_MAILBOX, 50).--define(RELEASE_ROUNDS, 3).--define(RELEASE_RETRY_MS, 200).-type guild_id() :: pos_integer().--type beam() :: #{binary := binary(), md5 := binary()}.--type outcome() :: ok | {error, term()}.-type worker() :: {patch | unpin | reconnect, node()} | sweep.-type peer() :: #{status := patched | down | {failed, term()}, role := atom(), at => integer()}.-type state() :: #{guild_ids := [guild_id()],- beam := beam() | {error, term()},+ beam := gateway_guild_pin_remote:beam() | {error, term()},base_md5s := [binary()],armed := boolean(),ever_armed := boolean(),@@ -87,16 +80,6 @@ apply_boot_pins() ->application:set_env(fluxer_gateway, guild_pinned_only_nodes, [node()])end.--spec pin_local([guild_id()]) -> ok.-pin_local(GuildIds) ->- Pins = as_pins(application:get_env(fluxer_gateway, guild_owner_pins, #{})),- application:set_env(fluxer_gateway, guild_owner_pins, pin_guilds(Pins, GuildIds, node())).---spec unpin_local([guild_id()]) -> ok.-unpin_local(GuildIds) ->- Pins = as_pins(application:get_env(fluxer_gateway, guild_owner_pins, #{})),- application:set_env(fluxer_gateway, guild_owner_pins, unpin_guilds(Pins, GuildIds, node())).--spec stage() -> {ok, map()} | {error, term()}.stage() ->gen_server:call(?MODULE, stage, ?CALL_TIMEOUT_MS).@@ -120,17 +103,18 @@ status() ->-spec init([]) -> {ok, state()}.init([]) ->ok = net_kernel:monitor_nodes(true),- Armed = application:get_env(fluxer_gateway, guild_pin_keeper_armed, false) =:= true,- EverArmed =- application:get_env(fluxer_gateway, guild_pin_keeper_ever_armed, false) =:= true,+ Armed = env_flag(guild_pin_keeper_armed),+ EverArmed = env_flag(guild_pin_keeper_ever_armed),ok = maybe_resume_armed(Armed),{ok, #{guild_ids => pinned_guild_ids(),- beam => read_beam(+ beam => gateway_guild_pin_remote:read_beam(fluxer_gateway_env:get(guild_pin_keeper_beam),fluxer_gateway_env:get(guild_pin_keeper_beam_md5)),- base_md5s => parse_md5_list(fluxer_gateway_env:get(guild_pin_keeper_base_md5s)),+ base_md5s => gateway_guild_pin_remote:parse_md5_list(+ fluxer_gateway_env:get(guild_pin_keeper_base_md5s)+ ),armed => Armed,ever_armed => Armed orelse EverArmed,peers => #{},@@ -142,6 +126,13 @@ init([]) ->drifted => []}}.+-spec env_flag(atom()) -> boolean().+env_flag(Key) ->+ case application:get_env(fluxer_gateway, Key, false) of+ true -> true;+ _ -> false+ end.+-spec maybe_resume_armed(boolean()) -> ok.maybe_resume_armed(false) ->ok;@@ -180,13 +171,13 @@ handle_info({repatch, Node}, State) when is_atom(Node) ->handle_info(sweep, State) ->{noreply, start_sweep(State#{sweep_timer := undefined})};handle_info({worker_result, Ref, Result}, #{workers := Workers} = State) when- is_map_key(Ref, Workers)+ is_reference(Ref), is_map_key(Ref, Workers)->erlang:demonitor(Ref, [flush]),{Worker, _WorkerPid} = maps:get(Ref, Workers),{noreply, handle_worker_result(Worker, Result, drop_worker(Ref, State))};handle_info({'DOWN', Ref, process, _Pid, Reason}, #{workers := Workers} = State) when- is_map_key(Ref, Workers)+ is_reference(Ref), is_map_key(Ref, Workers)->{Worker, _WorkerPid} = maps:get(Ref, Workers),Result = {unknown, {error, {worker_down, Reason}}},@@ -208,8 +199,10 @@ do_stage(#{beam := {error, Reason}}) ->{error, {beam_unavailable, Reason}};do_stage(#{beam := Beam, base_md5s := BaseMd5s}) ->Peers = gateway_peers(nodes()),- Results = run_parallel(- fun(Node) -> ensure_router(Node, Beam, BaseMd5s) end, Peers, ?PATCH_TIMEOUT_MS+ Results = gateway_guild_pin_remote:run_parallel(+ fun(Node) -> gateway_guild_pin_remote:ensure_router(Node, Beam, BaseMd5s) end,+ Peers,+ ?PATCH_TIMEOUT_MS),case [{Node, Result} || {Node, Result} <- Results, Result =/= ok] of[] -> {ok, #{staged => length(Results)}};@@ -220,7 +213,7 @@ do_stage(#{beam := Beam, base_md5s := BaseMd5s}) ->do_prewarm(#{guild_ids := []}) ->{error, no_pinned_guilds};do_prewarm(#{guild_ids := GuildIds}) ->- ok = pin_local(GuildIds),+ ok = gateway_guild_pin_remote:pin_local(GuildIds),Started = maps:from_list([{G, guild_manager:start_or_lookup(G, ?PREWARM_TIMEOUT_MS)}|| G <- GuildIds@@ -247,7 +240,7 @@ do_arm(#{guild_ids := GuildIds} = State) ->-spec reload_and_arm(state()) -> {{ok, map()} | {error, term()}, state()}.reload_and_arm(#{guild_ids := GuildIds} = State) ->- ok = pin_local(GuildIds),+ ok = gateway_guild_pin_remote:pin_local(GuildIds),Refreshes = [{GuildId, refresh_local_copy(GuildId)} || GuildId <- GuildIds],case [Refresh || {_GuildId, Result} = Refresh <- Refreshes, not refreshed(Result)] of[] -> arm_if_calm(State);@@ -262,21 +255,13 @@ refreshed(_Result) -> false.-spec refresh_local_copy(guild_id()) -> ok | adopted | {error, term()}.refresh_local_copy(GuildId) ->case- local_copy(GuildId) =/= undefined andalso routed_here(GuildId, gateway_peers(nodes()))+ local_copy(GuildId) =/= undefined andalso+ gateway_guild_pin_remote:routed_here(GuildId, gateway_peers(nodes()))oftrue -> adopted;false -> reload_local_copy(GuildId)end.--spec routed_here(guild_id(), [node()]) -> boolean().-routed_here(_GuildId, []) ->- true;-routed_here(GuildId, Peers) ->- Answers = erpc:multicall(- Peers, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS- ),- lists:all(fun(Answer) -> Answer =:= {ok, {ok, node()}} end, Answers).--spec arm_if_calm(state()) -> {{ok, map()} | {error, term()}, state()}.arm_if_calm(#{guild_ids := GuildIds} = State) ->case source_busy(GuildIds) of@@ -289,56 +274,36 @@ source_busy(GuildIds) ->Max = application:get_env(fluxer_gateway, guild_pin_keeper_max_source_mailbox, ?MAX_SOURCE_MAILBOX),- case busy_copies(GuildIds, gateway_peers(nodes()), Max) of+ case gateway_guild_pin_remote:busy_copies(GuildIds, gateway_peers(nodes()), Max) of[] -> ok;Busy -> {error, #{source_busy => Busy, max_mailbox => Max, next => retry_when_calm}}end.--spec busy_copies([guild_id()], [node()], non_neg_integer()) -> [{node(), term()}].-busy_copies(GuildIds, Nodes, Max) ->- [- {Node, Len}- || GuildId <- GuildIds,- Key <- [process_registry:build_process_key(guild, GuildId)],- Node <- stray_nodes(Key, Nodes),- Len <- [copy_mailbox(Node, Key)],- not (is_integer(Len) andalso Len =< Max)- ].---spec copy_mailbox(node(), process_registry:process_key()) -> term().-copy_mailbox(Node, Key) ->- case rpc:call(Node, ets, lookup, [process_registry_table, Key], ?RPC_TIMEOUT_MS) of- [{_, Pid}] when is_pid(Pid) ->- case- rpc:call(Node, erlang, process_info, [Pid, message_queue_len], ?RPC_TIMEOUT_MS)- of- {message_queue_len, Len} -> Len;- undefined -> 0;- Other -> {unreadable, Other}- end;- _ ->- 0- end.--spec reload_local_copy(guild_id()) -> ok | {error, term()}.reload_local_copy(GuildId) ->case {local_copy(GuildId), guild_handoff_freeze:shard_pid(GuildId, node())} of{undefined, _} ->{error, not_prewarmed};{_Pid, {ok, Shard}} ->- Request = {reload_guild, GuildId},- case shard_utils:safe_gen_call_remote(Shard, Request, ?RELOAD_TIMEOUT_MS) of- ok -> ok;- Other -> {error, Other}- end;+ reload_on_shard(Shard, GuildId);{_Pid, {error, _} = Error} ->Errorend.+-spec reload_on_shard(pid(), guild_id()) -> ok | {error, term()}.+reload_on_shard(Shard, GuildId) ->+ Request = {reload_guild, GuildId},+ case shard_utils:safe_gen_call_remote(Shard, Request, ?RELOAD_TIMEOUT_MS) of+ ok -> ok;+ Other -> {error, Other}+ end.+-spec arm_peers([node()], state()) -> {{ok, map()} | {error, term()}, state()}.-arm_peers(Peers, #{beam := Beam, base_md5s := BaseMd5s} = State) ->- Ready = run_parallel(- fun(Node) -> ensure_router(Node, Beam, BaseMd5s) end, Peers, ?PATCH_TIMEOUT_MS+arm_peers(Peers, #{beam := #{md5 := _} = Beam, base_md5s := BaseMd5s} = State) ->+ Ready = gateway_guild_pin_remote:run_parallel(+ fun(Node) -> gateway_guild_pin_remote:ensure_router(Node, Beam, BaseMd5s) end,+ Peers,+ ?PATCH_TIMEOUT_MS),case [{Node, Result} || {Node, Result} <- Ready, Result =/= ok] of[] -> pin_peers(Peers, State);@@ -346,11 +311,15 @@ arm_peers(Peers, #{beam := Beam, base_md5s := BaseMd5s} = State) ->end.-spec pin_peers([node()], state()) -> {{ok, map()} | {error, term()}, state()}.-pin_peers(Peers, #{beam := Beam, base_md5s := BaseMd5s, guild_ids := GuildIds} = State) ->+pin_peers(+ Peers, #{beam := #{md5 := _} = Beam, base_md5s := BaseMd5s, guild_ids := GuildIds} = State+) ->ok = application:set_env(fluxer_gateway, guild_pin_keeper_ever_armed, true),State0 = State#{ever_armed := true},- Results = run_parallel(- fun(Node) -> patch_peer(Node, Beam, BaseMd5s, GuildIds) end, Peers, ?ARM_PIN_TIMEOUT_MS+ Results = gateway_guild_pin_remote:run_parallel(+ fun(Node) -> gateway_guild_pin_remote:patch_peer(Node, Beam, BaseMd5s, GuildIds) end,+ Peers,+ ?ARM_PIN_TIMEOUT_MS),case [{Node, Result} || {Node, Result} <- Results, not is_patched(Result)] of[] ->@@ -367,9 +336,9 @@ do_disarm(#{guild_ids := GuildIds} = State) ->ok = application:unset_env(fluxer_gateway, guild_pin_keeper_armed),State1 = settle_workers(cancel_sweep(State#{armed := false})),Peers = gateway_peers(nodes()),- case release_peers(Peers, GuildIds) of+ case gateway_guild_pin_remote:release_peers(Peers, GuildIds) of[] ->- ok = unpin_local(GuildIds),+ ok = gateway_guild_pin_remote:unpin_local(GuildIds),Stopped = [{GuildId, stop_local_copy(GuildId)} || GuildId <- GuildIds],Reply = #{unpinned => length(Peers), pinned_after => [], stopped_local => Stopped},{{ok, Reply}, State1};@@ -378,52 +347,6 @@ do_disarm(#{guild_ids := GuildIds} = State) ->{{error, Reply}, State1}end.--spec release_peers([node()], [guild_id()]) -> [{node(), term()}].-release_peers(Peers, GuildIds) ->- release_peers(Peers, GuildIds, ?RELEASE_ROUNDS).---spec release_peers([node()], [guild_id()], pos_integer()) -> [{node(), term()}].-release_peers([], _GuildIds, _Left) ->- [];-release_peers(Peers, GuildIds, Left) ->- _ = run_parallel(fun(Node) -> unpin_peer(Node, GuildIds) end, Peers, ?PATCH_TIMEOUT_MS),- Checked = run_parallel(- fun(Node) -> routes_here(Node, GuildIds) end, Peers, ?PATCH_TIMEOUT_MS- ),- case [{Node, Answer} || {Node, Answer} <- Checked, Answer =/= released] of- [] ->- [];- Still when Left =:= 1 ->- Still;- Still ->- timer:sleep(?RELEASE_RETRY_MS),- release_peers([Node || {Node, _Answer} <- Still], GuildIds, Left - 1)- end.---spec routes_here(node(), [guild_id()]) -> released | {pinned | unverified, term()}.-routes_here(Node, GuildIds) ->- Answers = [- rpc:call(Node, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS)- || GuildId <- GuildIds- ],- case [Answer || Answer <- Answers, not released_answer(Answer)] of- [] -> released;- Wrong -> release_status(Wrong)- end.---spec released_answer(term()) -> boolean().-released_answer({ok, Owner}) -> Owner =/= node();-released_answer({error, {no_active_nodes, _}}) -> true;-released_answer({badrpc, nodedown}) -> true;-released_answer(_Answer) -> false.---spec release_status([term()]) -> {pinned | unverified, term()}.-release_status(Wrong) ->- case lists:member({ok, node()}, Wrong) of- true -> {pinned, Wrong};- false -> {unverified, Wrong}- end.--spec settle_workers(state()) -> state().settle_workers(#{workers := Workers} = State) ->Deadline = erlang:monotonic_time(millisecond) + ?PATCH_TIMEOUT_MS,@@ -455,7 +378,7 @@ await_worker(Ref, Deadline, State) ->stop_local_copy(GuildId) ->case local_copy(GuildId) ofundefined -> not_running;- _Pid -> stop_stray(GuildId, node())+ _Pid -> gateway_guild_pin_remote:stop_stray(GuildId, node())end.-spec handle_nodeup(node(), state()) -> state().@@ -464,14 +387,20 @@ handle_nodeup(Node, #{armed := true} = State) ->true -> start_patch(Node, State);false -> Stateend;-handle_nodeup(Node, #{ever_armed := true, guild_ids := GuildIds} = State) ->+handle_nodeup(Node, #{ever_armed := true} = State) ->case is_gateway_peer(Node) of- true -> start_worker({unpin, Node}, fun() -> unpin_peer(Node, GuildIds) end, State);+ true -> start_unpin(Node, State);false -> Stateend;handle_nodeup(_Node, State) ->State.+-spec start_unpin(node(), state()) -> state().+start_unpin(Node, #{guild_ids := GuildIds} = State) ->+ start_worker(+ {unpin, Node}, fun() -> gateway_guild_pin_remote:unpin_peer(Node, GuildIds) end, State+ ).+-spec handle_nodedown(node(), state()) -> state().handle_nodedown(Node, #{armed := true, peers := Peers} = State) ->case maps:get(Node, Peers, undefined) of@@ -492,15 +421,11 @@ maybe_reconnect(_Node, _Role, State) ->State.-spec start_patch(node(), state()) -> state().+start_patch(_Node, #{beam := {error, _Reason}} = State) ->+ State;start_patch(Node, #{beam := Beam, base_md5s := BaseMd5s, guild_ids := GuildIds} = State) ->- case Beam of- #{} ->- start_worker(- {patch, Node}, fun() -> patch_peer(Node, Beam, BaseMd5s, GuildIds) end, State- );- {error, _Reason} ->- State- end.+ Patch = fun() -> gateway_guild_pin_remote:patch_peer(Node, Beam, BaseMd5s, GuildIds) end,+ start_worker({patch, Node}, Patch, State).-spec handle_worker_result(worker(), term(), state()) -> state().handle_worker_result({patch, Node}, {Role, ok}, State) when is_atom(Role) ->@@ -515,10 +440,14 @@ handle_worker_result(sweep, {ok, #{stopped := Stopped, unpinned := Unpinned}}, S#{peers := Peers} = State,Drifted = [Node- || Node <- Unpinned, maps:get(status, maps:get(Node, Peers, #{}), none) =:= patched+ || Node <- eqwalizer:dynamic_cast(Unpinned),+ maps:get(status, maps:get(Node, Peers, #{}), none) =:= patched],lists:foreach(fun(Node) -> self() ! {repatch, Node} end, Drifted),- schedule_sweep(?SWEEP_INTERVAL_MS, record_strays(Stopped, State#{drifted := Drifted}));+ schedule_sweep(+ ?SWEEP_INTERVAL_MS,+ record_strays(eqwalizer:dynamic_cast(Stopped), State#{drifted := Drifted})+ );handle_worker_result(sweep, _Result, State) ->schedule_sweep(?SWEEP_INTERVAL_MS, State);handle_worker_result(_Worker, _Result, State) ->@@ -557,64 +486,16 @@ start_sweep(#{armed := false} = State) ->State;start_sweep(#{workers := Workers, blocked := Blocked} = State) ->case has_worker(sweep, Workers) of- true ->- State;- false ->- GuildIds = maps:get(guild_ids, State),- Nodes = gateway_peers(nodes()) -- maps:keys(Blocked),- start_worker(sweep, fun() -> {ok, sweep(GuildIds, Nodes)} end, State)- end.---spec sweep([guild_id()], [node()]) -> #{stopped := [map()], unpinned := [node()]}.-sweep(GuildIds, Nodes) ->- #{- unpinned => lists:usort(- lists:flatmap(fun(G) -> unpinned_nodes(G, Nodes) end, GuildIds)- ),- stopped => lists:flatmap(fun(GuildId) -> sweep_guild(GuildId, Nodes) end, GuildIds)- }.---spec unpinned_nodes(guild_id(), [node()]) -> [node()].-unpinned_nodes(_GuildId, []) ->- [];-unpinned_nodes(GuildId, Nodes) ->- Expected = {ok, node()},- Answers = erpc:multicall(- Nodes, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS- ),- [Node || {Node, Answer} <- lists:zip(Nodes, Answers), Answer =/= {ok, Expected}].---spec sweep_guild(guild_id(), [node()]) -> [map()].-sweep_guild(GuildId, Nodes) ->- Key = process_registry:build_process_key(guild, GuildId),- case process_registry:registry_whereis(Key) of- undefined ->- [];- _LocalPid ->- [- #{guild_id => GuildId, node => Node, result => stop_stray(GuildId, Node)}- || Node <- stray_nodes(Key, Nodes)- ]+ true -> State;+ false -> start_sweep_worker(maps:get(guild_ids, State), maps:keys(Blocked), State)end.--spec stray_nodes(process_registry:process_key(), [node()]) -> [node()].-stray_nodes(_Key, []) ->- [];-stray_nodes(Key, Nodes) ->- Results = erpc:multicall(- Nodes, ets, lookup, [process_registry_table, Key], ?RPC_TIMEOUT_MS- ),- [Node || {Node, {ok, [{_, Pid}]}} <- lists:zip(Nodes, Results), is_pid(Pid)].---spec stop_stray(guild_id(), node()) -> term().-stop_stray(GuildId, Node) ->- case guild_handoff_freeze:shard_pid(GuildId, Node) of- {ok, ShardPid} ->- Request = {stop_guild, GuildId, {shutdown, handoff}},- shard_utils:safe_gen_call_remote(ShardPid, Request, ?STOP_TIMEOUT_MS);- {error, _Reason} = Error ->- Error- end.+-spec start_sweep_worker([guild_id()], [node()], state()) -> state().+start_sweep_worker(GuildIds, Blocked, State) ->+ Nodes = gateway_peers(nodes()) -- Blocked,+ start_worker(+ sweep, fun() -> {ok, gateway_guild_pin_remote:sweep(GuildIds, Nodes)} end, State+ ).-spec record_strays([map()], state()) -> state().record_strays([], State) ->@@ -626,146 +507,20 @@ record_strays(Stopped, #{strays_stopped := Count} = State) ->last_stray := #{at => erlang:system_time(millisecond), stopped => Stopped}}.--spec patch_peer(node(), beam(), [binary()], [guild_id()]) -> {atom(), outcome()}.-patch_peer(Node, Beam, BaseMd5s, GuildIds) ->- Role = peer_role(Node),- Outcome =- maybe- ok ?= ensure_router(Node, Beam, BaseMd5s),- ok ?= pin_peer(Node, GuildIds),- verify_pins(Node, GuildIds)- end,- {Role, Outcome}.---spec ensure_router(node(), beam(), [binary()]) -> ok | {error, term()}.-ensure_router(Node, #{binary := Bin, md5 := PinnedMd5}, BaseMd5s) ->- case remote_router_md5(Node) of- {ok, PinnedMd5} ->- ok;- {ok, Md5} ->- case lists:member(Md5, BaseMd5s) of- true -> load_router(Node, Bin, PinnedMd5);- false -> pin_aware_router(Node, Md5)- end;- {error, _Reason} = Error ->- Error- end.---spec pin_aware_router(node(), binary()) -> ok | {error, term()}.-pin_aware_router(Node, Md5) ->- Args = [?ROUTER, pinned_guild_owner, 1],- case rpc:call(Node, erlang, function_exported, Args, ?RPC_TIMEOUT_MS) of- true -> ok;- _ -> {error, {unexpected_router_md5, binary:encode_hex(Md5)}}- end.---spec remote_router_md5(node()) -> {ok, binary()} | {error, term()}.-remote_router_md5(Node) ->- case rpc:call(Node, code, ensure_loaded, [?ROUTER], ?RPC_TIMEOUT_MS) of- {module, ?ROUTER} ->- case rpc:call(Node, erlang, get_module_info, [?ROUTER, md5], ?RPC_TIMEOUT_MS) of- Md5 when is_binary(Md5) -> {ok, Md5};- Other -> {error, {router_md5_unavailable, Other}}- end;- Other ->- {error, {router_unavailable, Other}}- end.---spec load_router(node(), binary(), binary()) -> ok | {error, term()}.-load_router(Node, Bin, PinnedMd5) ->- case rpc:call(Node, erlang, check_old_code, [?ROUTER], ?RPC_TIMEOUT_MS) of- false ->- Args = [?ROUTER, ?LOADED_BEAM_NAME, Bin],- case rpc:call(Node, code, load_binary, Args, ?RPC_TIMEOUT_MS) of- {module, ?ROUTER} -> confirm_router(Node, PinnedMd5);- Other -> {error, {router_load_failed, Other}}- end;- Other ->- {error, {router_old_code, Other}}- end.---spec confirm_router(node(), binary()) -> ok | {error, term()}.-confirm_router(Node, PinnedMd5) ->- case remote_router_md5(Node) of- {ok, PinnedMd5} ->- _ = rpc:call(Node, code, soft_purge, [?ROUTER], ?RPC_TIMEOUT_MS),- ok;- Other ->- {error, {router_md5_after_load, Other}}- end.---spec pin_peer(node(), [guild_id()]) -> ok | {error, term()}.-pin_peer(Node, GuildIds) ->- update_remote_pins(Node, fun(Pins) -> pin_guilds(Pins, GuildIds, node()) end).---spec unpin_peer(node(), [guild_id()]) -> ok | {error, term()}.-unpin_peer(Node, GuildIds) ->- update_remote_pins(Node, fun(Pins) -> unpin_guilds(Pins, GuildIds, node()) end).---spec update_remote_pins(node(), fun((#{guild_id() => node()}) -> #{guild_id() => node()})) ->- ok | {error, term()}.-update_remote_pins(Node, Update) ->- case- rpc:call(- Node, application, get_env, [fluxer_gateway, guild_owner_pins, #{}], ?RPC_TIMEOUT_MS- )- of- Pins when is_map(Pins) -> write_remote_pins(Node, as_pins(Pins), Update(as_pins(Pins)));- Other -> {error, {pins_unreadable, Other}}- end.---spec write_remote_pins(node(), #{guild_id() => node()}, #{guild_id() => node()}) ->- ok | {error, term()}.-write_remote_pins(_Node, Same, Same) ->- ok;-write_remote_pins(Node, _Old, New) when map_size(New) =:= 0 ->- remote_env(Node, unset_env, [fluxer_gateway, guild_owner_pins]);-write_remote_pins(Node, _Old, New) ->- remote_env(Node, set_env, [fluxer_gateway, guild_owner_pins, New]).---spec remote_env(node(), set_env | unset_env, [term()]) -> ok | {error, term()}.-remote_env(Node, Function, Args) ->- case rpc:call(Node, application, Function, Args, ?RPC_TIMEOUT_MS) of- ok -> ok;- Other -> {error, {Function, Other}}- end.---spec verify_pins(node(), [guild_id()]) -> ok | {error, term()}.-verify_pins(Node, GuildIds) ->- Expected = {ok, node()},- Answers = [- {GuildId,- rpc:call(Node, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS)}- || GuildId <- GuildIds- ],- case [Answer || {_GuildId, Got} = Answer <- Answers, Got =/= Expected] of- [] -> ok;- Wrong -> {error, {pin_not_effective, Wrong}}- end.---spec peer_role(node()) -> atom().-peer_role(Node) ->- case rpc:call(Node, fluxer_gateway_sup, current_role, [], ?RPC_TIMEOUT_MS) of- Role when is_atom(Role) -> Role;- _Other -> unknown- end.--spec reconnect_loop(node(), integer()) -> connected | gave_up.reconnect_loop(Node, Deadline) ->case net_kernel:connect_node(Node) of- true ->- connected;- _ ->- case erlang:monotonic_time(millisecond) >= Deadline of- true ->- gave_up;- false ->- receive- after ?RECONNECT_INTERVAL_MS -> reconnect_loop(Node, Deadline)- end- end+ true -> connected;+ _ -> retry_connect(erlang:monotonic_time(millisecond) >= Deadline, Node, Deadline)end.+-spec retry_connect(boolean(), node(), integer()) -> connected | gave_up.+retry_connect(true, _Node, _Deadline) ->+ gave_up;+retry_connect(false, Node, Deadline) ->+ timer:sleep(?RECONNECT_INTERVAL_MS),+ reconnect_loop(Node, Deadline).+-spec kill_worker(reference(), state()) -> state().kill_worker(Ref, #{workers := Workers} = State) ->{_Worker, Pid} = maps:get(Ref, Workers),@@ -779,14 +534,24 @@ start_worker(Worker, Fun, #{workers := Workers} = State) ->true ->State;false ->- Parent = self(),- {Pid, Ref} = spawn_monitor(fun() ->- receive- {run, MonRef} -> Parent ! {worker_result, MonRef, Fun()}- end- end),- Pid ! {run, Ref},- State#{workers := Workers#{Ref => {Worker, Pid}}}+ spawn_worker(Worker, Fun, State)+ end.++-spec spawn_worker(worker(), fun(() -> term()), state()) -> state().+spawn_worker(Worker, Fun, #{workers := Workers} = State) ->+ Parent = self(),+ {Pid, Ref} = spawn_monitor(fun() -> run_worker(Parent, Fun) end),+ Pid ! {run, Ref},+ State#{workers := Workers#{Ref => {Worker, Pid}}}.++-spec run_worker(pid(), fun(() -> term())) -> ok.+run_worker(Parent, Fun) ->+ receive+ {run, MonRef} ->+ Parent ! {worker_result, MonRef, Fun()},+ ok+ after infinity ->+ okend.-spec has_worker(worker(), #{reference() => {worker(), pid()}}) -> boolean().@@ -797,28 +562,6 @@ has_worker(Worker, Workers) ->drop_worker(Ref, #{workers := Workers} = State) ->State#{workers := maps:remove(Ref, Workers)}.--spec run_parallel(fun((node()) -> term()), [node()], pos_integer()) -> [{node(), term()}].-run_parallel(Fun, Nodes, TimeoutMs) ->- Parent = self(),- Ref = make_ref(),- Pids = [{spawn(fun() -> Parent ! {Ref, Node, Fun(Node)} end), Node} || Node <- Nodes],- Deadline = erlang:monotonic_time(millisecond) + TimeoutMs,- Results = collect_parallel(Ref, Nodes, #{}, Deadline),- [exit(Pid, kill) || {Pid, Node} <- Pids, not maps:is_key(Node, Results)],- [{Node, maps:get(Node, Results, {error, timeout})} || Node <- Nodes].---spec collect_parallel(reference(), [node()], #{node() => term()}, integer()) ->- #{node() => term()}.-collect_parallel(_Ref, Nodes, Results, _Deadline) when map_size(Results) =:= length(Nodes) ->- Results;-collect_parallel(Ref, Nodes, Results, Deadline) ->- Wait = max(0, Deadline - erlang:monotonic_time(millisecond)),- receive- {Ref, Node, Result} -> collect_parallel(Ref, Nodes, Results#{Node => Result}, Deadline)- after Wait ->- Results- end.--spec record_patches([{node(), term()}], state()) -> state().record_patches(Results, #{peers := Peers} = State) ->NewPeers = lists:foldl(@@ -879,50 +622,11 @@ status_map(State) ->local_copy(GuildId) ->process_registry:registry_whereis(process_registry:build_process_key(guild, GuildId)).--spec beam_status(beam() | {error, term()}) -> binary() | {error, term()}.+-spec beam_status(gateway_guild_pin_remote:beam() | {error, term()}) ->+ binary() | {error, term()}.beam_status(#{md5 := Md5}) -> binary:encode_hex(Md5);beam_status({error, _Reason} = Error) -> Error.--spec read_beam(term(), term()) -> beam() | {error, term()}.-read_beam(Path, ExpectedHex) when is_list(Path), is_binary(ExpectedHex) ->- case file:read_file(Path) of- {ok, Bin} -> check_beam(Bin, decode_md5(ExpectedHex));- {error, Reason} -> {error, {read_failed, Path, Reason}}- end;-read_beam(_Path, _ExpectedHex) ->- {error, not_configured}.---spec check_beam(binary(), {ok, binary()} | error) -> beam() | {error, term()}.-check_beam(_Bin, error) ->- {error, invalid_expected_md5};-check_beam(Bin, {ok, Expected}) ->- case beam_lib:md5(Bin) of- {ok, {?ROUTER, Expected}} -> #{binary => Bin, md5 => Expected};- {ok, {Module, Md5}} -> {error, {beam_mismatch, Module, binary:encode_hex(Md5)}};- {error, beam_lib, Reason} -> {error, {invalid_beam, Reason}}- end.---spec parse_md5_list(term()) -> [binary()].-parse_md5_list(Value) when is_binary(Value) ->- [- Md5- || Token <- string:lexemes(binary_to_list(Value), ", "), {ok, Md5} <- [decode_md5(Token)]- ];-parse_md5_list(_Value) ->- [].---spec decode_md5(string() | binary()) -> {ok, binary()} | error.-decode_md5(Hex) when is_list(Hex) ->- decode_md5(list_to_binary(Hex));-decode_md5(Hex) when byte_size(Hex) =:= 32 ->- try binary:decode_hex(Hex) of- Md5 -> {ok, Md5}- catch- error:badarg -> error- end;-decode_md5(_Hex) ->- error.--spec pinned_guild_ids() -> [guild_id()].pinned_guild_ids() ->case fluxer_gateway_env:get(pinned_guild_ids) of@@ -941,203 +645,3 @@ is_gateway_peer(Node) ->-spec is_patched(term()) -> boolean().is_patched({_Role, ok}) -> true;is_patched(_) -> false.---spec as_pins(term()) -> #{guild_id() => node()}.-as_pins(Pins) when is_map(Pins) ->- maps:filter(fun(Key, Value) -> is_integer(Key) andalso is_atom(Value) end, Pins);-as_pins(_Pins) ->- #{}.---spec pin_guilds(#{guild_id() => node()}, [guild_id()], node()) -> #{guild_id() => node()}.-pin_guilds(Pins, GuildIds, Owner) ->- maps:merge(Pins, maps:from_list([{GuildId, Owner} || GuildId <- GuildIds])).---spec unpin_guilds(#{guild_id() => node()}, [guild_id()], node()) -> #{guild_id() => node()}.-unpin_guilds(Pins, GuildIds, Owner) ->- maps:filter(- fun(GuildId, Node) -> not (Node =:= Owner andalso lists:member(GuildId, GuildIds)) end,- Pins- ).---ifdef(TEST).--include_lib("eunit/include/eunit.hrl").---define(PG, 1100000000000000001).--unpin_only_removes_this_nodes_pins_test() ->- Pins = #{?PG => node(), 42 => 'pin_other@h', 7 => node()},- ?assertEqual(#{42 => 'pin_other@h', 7 => node()}, unpin_guilds(Pins, [?PG], node())),- ?assertEqual(Pins, unpin_guilds(Pins, [42], node())).--pin_overrides_a_stale_owner_test() ->- Pins = #{?PG => 'fluxer_gateway_dn_dead@h', 42 => 'g@h'},- ?assertEqual(#{?PG => node(), 42 => 'g@h'}, pin_guilds(Pins, [?PG], node())).--only_release_named_nodes_are_peers_test() ->- ?assertEqual(- ['[email protected]'],- gateway_peers(['[email protected]', '[email protected]', 'probe@h'])- ).--beam_must_match_the_expected_md5_test() ->- {?ROUTER, Bin, _Path} = code:get_object_code(?ROUTER),- {ok, {?ROUTER, Md5}} = beam_lib:md5(Bin),- ?assertMatch(#{md5 := Md5}, check_beam(Bin, decode_md5(binary:encode_hex(Md5)))),- ?assertMatch({error, {beam_mismatch, ?ROUTER, _}}, check_beam(Bin, {ok, <<0:128>>})),- ?assertEqual({error, invalid_expected_md5}, check_beam(Bin, decode_md5(<<"D5E4">>))),- ?assertEqual({error, not_configured}, read_beam(undefined, undefined)).--base_md5_list_parses_hex_and_skips_garbage_test() ->- ?assertEqual(- [binary:decode_hex(<<"5D7C0BB7B8DA42FC92C6B3BDC7584CD2">>)],- parse_md5_list(<<"5D7C0BB7B8DA42FC92C6B3BDC7584CD2, nope">>)- ).--boot_pins_are_off_without_pinned_guilds_test() ->- persistent_term:put({fluxer_gateway, runtime_config}, #{}),- try- ok = apply_boot_pins(),- ?assertEqual(undefined, application:get_env(fluxer_gateway, guild_owner_pins)),- ?assertEqual(undefined, application:get_env(fluxer_gateway, guild_pinned_only_nodes)),- ?assertEqual([], child_specs())- after- persistent_term:erase({fluxer_gateway, runtime_config})- end.--local_pins_follow_arm_and_disarm_test() ->- application:set_env(fluxer_gateway, guild_owner_pins, #{42 => 'pin_other@h'}),- try- ok = pin_local([?PG]),- ?assertEqual(- {ok, #{?PG => node(), 42 => 'pin_other@h'}},- application:get_env(fluxer_gateway, guild_owner_pins)- ),- ok = unpin_local([?PG]),- ?assertEqual(- {ok, #{42 => 'pin_other@h'}}, application:get_env(fluxer_gateway, guild_owner_pins)- )- after- application:unset_env(fluxer_gateway, guild_owner_pins)- end.--boot_pins_leave_this_node_owning_nothing_until_prewarm_test() ->- persistent_term:put({fluxer_gateway, runtime_config}, #{- pinned_guild_ids => [?PG], gateway_role => guilds- }),- try- ok = apply_boot_pins(),- ?assertEqual(undefined, application:get_env(fluxer_gateway, guild_owner_pins)),- ?assertEqual(- {ok, [node()]}, application:get_env(fluxer_gateway, guild_pinned_only_nodes)- ),- ?assertEqual(- {error, {no_active_nodes, guilds}}, ?ROUTER:owner_node_result(?PG, guilds)- ),- ok = pin_local([?PG]),- ?assertEqual({ok, node()}, ?ROUTER:owner_node_result(?PG, guilds)),- ?assertMatch([#{id := ?MODULE}], child_specs())- after- application:unset_env(fluxer_gateway, guild_owner_pins),- application:unset_env(fluxer_gateway, guild_pinned_only_nodes),- persistent_term:erase({fluxer_gateway, runtime_config})- end.--sweep_leaves_remote_copies_alone_without_a_local_copy_test() ->- ok = process_registry:init(),- ?assertEqual([], sweep_guild(?PG, ['[email protected]'])),- ?assertEqual(#{stopped => [], unpinned => []}, sweep([?PG], [])).--release_unpins_and_verifies_the_route_test() ->- persistent_term:put({gateway_cluster_membership, members}, [node(), 'g1@h', 'g2@h']),- persistent_term:put({gateway_cluster_membership, members_by_role}, #{- guilds => [node(), 'g1@h', 'g2@h']- }),- application:set_env(fluxer_gateway, guild_pinned_only_nodes, [node()]),- application:set_env(fluxer_gateway, guild_owner_pins, #{?PG => node(), 42 => 'g1@h'}),- try- ?assertMatch({pinned, [{ok, _}]}, routes_here(node(), [?PG])),- ?assertEqual([], release_peers([node(), '[email protected]'], [?PG])),- ?assertEqual(- {ok, #{42 => 'g1@h'}}, application:get_env(fluxer_gateway, guild_owner_pins)- ),- ?assertEqual(released, routes_here(node(), [?PG]))- after- application:unset_env(fluxer_gateway, guild_owner_pins),- application:unset_env(fluxer_gateway, guild_pinned_only_nodes),- persistent_term:erase({gateway_cluster_membership, members}),- persistent_term:erase({gateway_cluster_membership, members_by_role})- end.--arm_refuses_without_a_prewarmed_copy_test() ->- ok = process_registry:init(),- State = #{guild_ids => [?PG], beam => #{binary => <<>>, md5 => <<0:128>>}},- try- ?assertEqual(- {{error, #{reload_failed => [{?PG, {error, not_prewarmed}}]}}, State}, do_arm(State)- ),- ?assertEqual(- undefined, application:get_env(fluxer_gateway, guild_pin_keeper_ever_armed)- )- after- application:unset_env(fluxer_gateway, guild_owner_pins)- end.--arm_waits_for_a_calm_source_copy_test() ->- ok = process_registry:init(),- {ok, Pid} = gen_event:start(),- Key = process_registry:build_process_key(guild, ?PG),- true = ets:insert(process_registry_table, {Key, Pid}),- try- ?assertEqual([], busy_copies([?PG], [node()], 100)),- true = erlang:suspend_process(Pid),- [Pid ! {backlog, N} || N <- lists:seq(1, 101)],- ?assertEqual([{node(), 101}], busy_copies([?PG], [node()], 100)),- ?assertEqual([], busy_copies([?PG], [node()], 101))- after- ets:delete(process_registry_table, Key),- exit(Pid, kill)- end.--arm_pins_no_peer_until_every_peer_is_ready_test() ->- Down = '[email protected]',- State = #{guild_ids => [?PG], beam => #{binary => <<>>, md5 => <<0:128>>}, base_md5s => []},- ?assertMatch(- {{error, #{unready := [{Down, {error, _}}], next := retry}}, State},- arm_peers([Down], State)- ),- ?assertEqual(undefined, application:get_env(fluxer_gateway, guild_pin_keeper_ever_armed)).--arm_adopts_a_copy_that_already_owns_the_guild_test() ->- ok = process_registry:init(),- Key = process_registry:build_process_key(guild, ?PG),- true = ets:insert(process_registry_table, {Key, self()}),- try- ?assertEqual(adopted, refresh_local_copy(?PG))- after- ets:delete(process_registry_table, Key)- end.--router_with_pins_needs_no_load_test() ->- Beam = #{binary => <<>>, md5 => <<0:128>>},- ?assertEqual(ok, ensure_router(node(), Beam, [])).--permanent_patch_failure_blocks_the_peer_until_patched_test() ->- Node = '[email protected]',- State = #{armed => true, peers => #{}, blocked => #{}},- Failed = handle_worker_result(- {patch, Node}, {guilds, {error, {unexpected_router_md5, <<"AB">>}}}, State- ),- ?assertEqual(#{Node => {unexpected_router_md5, <<"AB">>}}, maps:get(blocked, Failed)),- Patched = handle_worker_result({patch, Node}, {guilds, ok}, Failed),- ?assertEqual(#{}, maps:get(blocked, Patched)),- receive- sweep -> ok- after 0 -> ok- end.--permanent_patch_errors_are_not_retried_test() ->- ?assert(permanent_patch_error({unexpected_router_md5, <<"AB">>})),- ?assert(permanent_patch_error({router_old_code, true})),- ?assertNot(permanent_patch_error({router_unavailable, {badrpc, nodedown}})).---endif.diff --git a/fluxer_gateway/src/gateway/gateway_guild_pin_remote.erl b/fluxer_gateway/src/gateway/gateway_guild_pin_remote.erlnew file mode 100644index 000000000..f7860742a--- /dev/null+++ b/fluxer_gateway/src/gateway/gateway_guild_pin_remote.erl@@ -0,0 +1,410 @@+%% SPDX-License-Identifier: AGPL-3.0-or-later++-module(gateway_guild_pin_remote).+-typing([eqwalizer]).++-export([+ busy_copies/3,+ sweep/2,+ stop_stray/2,+ patch_peer/4,+ ensure_router/3,+ unpin_peer/2,+ routes_here/2,+ read_beam/2,+ parse_md5_list/1,+ as_pins/1,+ pin_guilds/3,+ unpin_guilds/3,+ pin_local/1,+ unpin_local/1,+ routed_here/2,+ release_peers/2,+ run_parallel/3+]).++-define(ROUTER, gateway_node_router).+-define(LOADED_BEAM_NAME, "/tmp/fluxer-guild-pin/gateway_node_router.beam").+-define(RPC_TIMEOUT_MS, 2000).+-define(STOP_TIMEOUT_MS, 5000).+-define(PATCH_TIMEOUT_MS, 10000).+-define(RELEASE_ROUNDS, 3).+-define(RELEASE_RETRY_MS, 200).++-type guild_id() :: pos_integer().+-type beam() :: #{binary := binary(), md5 := binary()}.+-type outcome() :: ok | {error, term()}.++-export_type([guild_id/0, beam/0, outcome/0]).++-spec busy_copies([guild_id()], [node()], non_neg_integer()) -> [{node(), term()}].+busy_copies(GuildIds, Nodes, Max) ->+ [+ {Node, Len}+ || GuildId <- GuildIds,+ Key <- [process_registry:build_process_key(guild, GuildId)],+ Node <- stray_nodes(Key, Nodes),+ Len <- [copy_mailbox(Node, Key)],+ not (is_integer(Len) andalso Len =< Max)+ ].++-spec copy_mailbox(node(), process_registry:process_key()) -> term().+copy_mailbox(Node, Key) ->+ case rpc:call(Node, ets, lookup, [process_registry_table, Key], ?RPC_TIMEOUT_MS) of+ [{_, Pid}] when is_pid(Pid) -> mailbox_len(Node, Pid);+ _ -> 0+ end.++-spec mailbox_len(node(), pid()) -> term().+mailbox_len(Node, Pid) ->+ case rpc:call(Node, erlang, process_info, [Pid, message_queue_len], ?RPC_TIMEOUT_MS) of+ {message_queue_len, Len} -> Len;+ undefined -> 0;+ Other -> {unreadable, Other}+ end.++-spec sweep([guild_id()], [node()]) -> #{stopped := [map()], unpinned := [node()]}.+sweep(GuildIds, Nodes) ->+ #{+ unpinned => lists:usort(+ lists:flatmap(fun(G) -> unpinned_nodes(G, Nodes) end, GuildIds)+ ),+ stopped => lists:flatmap(fun(GuildId) -> sweep_guild(GuildId, Nodes) end, GuildIds)+ }.++-spec unpinned_nodes(guild_id(), [node()]) -> [node()].+unpinned_nodes(_GuildId, []) ->+ [];+unpinned_nodes(GuildId, Nodes) ->+ Expected = {ok, node()},+ Answers = erpc:multicall(+ Nodes, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS+ ),+ [Node || {Node, Answer} <- lists:zip(Nodes, Answers), Answer =/= {ok, Expected}].++-spec sweep_guild(guild_id(), [node()]) -> [map()].+sweep_guild(GuildId, Nodes) ->+ Key = process_registry:build_process_key(guild, GuildId),+ case process_registry:registry_whereis(Key) of+ undefined ->+ [];+ _LocalPid ->+ [+ #{guild_id => GuildId, node => Node, result => stop_stray(GuildId, Node)}+ || Node <- stray_nodes(Key, Nodes)+ ]+ end.++-spec stray_nodes(process_registry:process_key(), [node()]) -> [node()].+stray_nodes(_Key, []) ->+ [];+stray_nodes(Key, Nodes) ->+ Results = erpc:multicall(+ Nodes, ets, lookup, [process_registry_table, Key], ?RPC_TIMEOUT_MS+ ),+ [Node || {Node, {ok, [{_, Pid}]}} <- lists:zip(Nodes, Results), is_pid(Pid)].++-spec stop_stray(guild_id(), node()) -> term().+stop_stray(GuildId, Node) ->+ case guild_handoff_freeze:shard_pid(GuildId, Node) of+ {ok, ShardPid} ->+ Request = {stop_guild, GuildId, {shutdown, handoff}},+ shard_utils:safe_gen_call_remote(ShardPid, Request, ?STOP_TIMEOUT_MS);+ {error, _Reason} = Error ->+ Error+ end.++-spec patch_peer(node(), beam(), [binary()], [guild_id()]) -> {atom(), outcome()}.+patch_peer(Node, Beam, BaseMd5s, GuildIds) ->+ Role = peer_role(Node),+ Outcome =+ maybe+ ok ?= ensure_router(Node, Beam, BaseMd5s),+ ok ?= pin_peer(Node, GuildIds),+ verify_pins(Node, GuildIds)+ end,+ {Role, Outcome}.++-spec ensure_router(node(), beam(), [binary()]) -> ok | {error, term()}.+ensure_router(Node, #{binary := Bin, md5 := PinnedMd5}, BaseMd5s) ->+ case remote_router_md5(Node) of+ {ok, PinnedMd5} ->+ ok;+ {ok, Md5} ->+ replace_router(lists:member(Md5, BaseMd5s), Node, Bin, PinnedMd5, Md5);+ {error, _Reason} = Error ->+ Error+ end.++-spec replace_router(boolean(), node(), binary(), binary(), binary()) -> ok | {error, term()}.+replace_router(true, Node, Bin, PinnedMd5, _Md5) ->+ load_router(Node, Bin, PinnedMd5);+replace_router(false, Node, _Bin, _PinnedMd5, Md5) ->+ pin_aware_router(Node, Md5).++-spec pin_aware_router(node(), binary()) -> ok | {error, term()}.+pin_aware_router(Node, Md5) ->+ Args = [?ROUTER, pinned_guild_owner, 1],+ case rpc:call(Node, erlang, function_exported, Args, ?RPC_TIMEOUT_MS) of+ true -> ok;+ _ -> {error, {unexpected_router_md5, binary:encode_hex(Md5)}}+ end.++-spec remote_router_md5(node()) -> {ok, binary()} | {error, term()}.+remote_router_md5(Node) ->+ case rpc:call(Node, code, ensure_loaded, [?ROUTER], ?RPC_TIMEOUT_MS) of+ {module, ?ROUTER} -> loaded_router_md5(Node);+ Other -> {error, {router_unavailable, Other}}+ end.++-spec loaded_router_md5(node()) -> {ok, binary()} | {error, term()}.+loaded_router_md5(Node) ->+ case rpc:call(Node, erlang, get_module_info, [?ROUTER, md5], ?RPC_TIMEOUT_MS) of+ Md5 when is_binary(Md5) -> {ok, Md5};+ Other -> {error, {router_md5_unavailable, Other}}+ end.++-spec load_router(node(), binary(), binary()) -> ok | {error, term()}.+load_router(Node, Bin, PinnedMd5) ->+ case rpc:call(Node, erlang, check_old_code, [?ROUTER], ?RPC_TIMEOUT_MS) of+ false -> load_router_binary(Node, Bin, PinnedMd5);+ Other -> {error, {router_old_code, Other}}+ end.++-spec load_router_binary(node(), binary(), binary()) -> ok | {error, term()}.+load_router_binary(Node, Bin, PinnedMd5) ->+ Args = [?ROUTER, ?LOADED_BEAM_NAME, Bin],+ case rpc:call(Node, code, load_binary, Args, ?RPC_TIMEOUT_MS) of+ {module, ?ROUTER} -> confirm_router(Node, PinnedMd5);+ Other -> {error, {router_load_failed, Other}}+ end.++-spec confirm_router(node(), binary()) -> ok | {error, term()}.+confirm_router(Node, PinnedMd5) ->+ case remote_router_md5(Node) of+ {ok, PinnedMd5} ->+ _ = rpc:call(Node, code, soft_purge, [?ROUTER], ?RPC_TIMEOUT_MS),+ ok;+ Other ->+ {error, {router_md5_after_load, Other}}+ end.++-spec pin_peer(node(), [guild_id()]) -> ok | {error, term()}.+pin_peer(Node, GuildIds) ->+ update_remote_pins(Node, fun(Pins) -> pin_guilds(Pins, GuildIds, node()) end).++-spec unpin_peer(node(), [guild_id()]) -> ok | {error, term()}.+unpin_peer(Node, GuildIds) ->+ update_remote_pins(Node, fun(Pins) -> unpin_guilds(Pins, GuildIds, node()) end).++-spec update_remote_pins(node(), fun((#{guild_id() => node()}) -> #{guild_id() => node()})) ->+ ok | {error, term()}.+update_remote_pins(Node, Update) ->+ case+ rpc:call(+ Node, application, get_env, [fluxer_gateway, guild_owner_pins, #{}], ?RPC_TIMEOUT_MS+ )+ of+ Pins when is_map(Pins) -> write_remote_pins(Node, as_pins(Pins), Update(as_pins(Pins)));+ Other -> {error, {pins_unreadable, Other}}+ end.++-spec write_remote_pins(node(), #{guild_id() => node()}, #{guild_id() => node()}) ->+ ok | {error, term()}.+write_remote_pins(_Node, Same, Same) ->+ ok;+write_remote_pins(Node, _Old, New) when map_size(New) =:= 0 ->+ remote_env(Node, unset_env, [fluxer_gateway, guild_owner_pins]);+write_remote_pins(Node, _Old, New) ->+ remote_env(Node, set_env, [fluxer_gateway, guild_owner_pins, New]).++-spec remote_env(node(), set_env | unset_env, [term()]) -> ok | {error, term()}.+remote_env(Node, Function, Args) ->+ case rpc:call(Node, application, Function, Args, ?RPC_TIMEOUT_MS) of+ ok -> ok;+ Other -> {error, {Function, Other}}+ end.++-spec verify_pins(node(), [guild_id()]) -> ok | {error, term()}.+verify_pins(Node, GuildIds) ->+ Expected = {ok, node()},+ Answers = [+ {GuildId,+ rpc:call(Node, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS)}+ || GuildId <- GuildIds+ ],+ case [Answer || {_GuildId, Got} = Answer <- Answers, Got =/= Expected] of+ [] -> ok;+ Wrong -> {error, {pin_not_effective, Wrong}}+ end.++-spec peer_role(node()) -> atom().+peer_role(Node) ->+ case rpc:call(Node, fluxer_gateway_sup, current_role, [], ?RPC_TIMEOUT_MS) of+ Role when is_atom(Role) -> Role;+ _Other -> unknown+ end.++-spec routes_here(node(), [guild_id()]) -> released | {pinned | unverified, term()}.+routes_here(Node, GuildIds) ->+ Answers = [+ rpc:call(Node, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS)+ || GuildId <- GuildIds+ ],+ case [Answer || Answer <- Answers, not released_answer(Answer)] of+ [] -> released;+ Wrong -> release_status(Wrong)+ end.++-spec released_answer(term()) -> boolean().+released_answer({ok, Owner}) -> Owner =/= node();+released_answer({error, {no_active_nodes, _}}) -> true;+released_answer({badrpc, nodedown}) -> true;+released_answer(_Answer) -> false.++-spec release_status([term()]) -> {pinned | unverified, term()}.+release_status(Wrong) ->+ case lists:member({ok, node()}, Wrong) of+ true -> {pinned, Wrong};+ false -> {unverified, Wrong}+ end.++-spec read_beam(term(), term()) -> beam() | {error, term()}.+read_beam(Path, ExpectedHex) when is_list(Path), is_binary(ExpectedHex) ->+ case file:read_file(eqwalizer:dynamic_cast(Path)) of+ {ok, Bin} -> check_beam(Bin, decode_md5(ExpectedHex));+ {error, Reason} -> {error, {read_failed, Path, Reason}}+ end;+read_beam(_Path, _ExpectedHex) ->+ {error, not_configured}.++-spec check_beam(binary(), {ok, binary()} | error) -> beam() | {error, term()}.+check_beam(_Bin, error) ->+ {error, invalid_expected_md5};+check_beam(Bin, {ok, Expected}) ->+ case beam_lib:md5(Bin) of+ {ok, {?ROUTER, Expected}} -> #{binary => Bin, md5 => Expected};+ {ok, {Module, Md5}} -> {error, {beam_mismatch, Module, binary:encode_hex(Md5)}};+ {error, beam_lib, Reason} -> {error, {invalid_beam, Reason}}+ end.++-spec parse_md5_list(term()) -> [binary()].+parse_md5_list(Value) when is_binary(Value) ->+ [+ Md5+ || Token <- string:lexemes(binary_to_list(Value), ", "), {ok, Md5} <- [decode_md5(Token)]+ ];+parse_md5_list(_Value) ->+ [].++-spec decode_md5(string() | binary()) -> {ok, binary()} | error.+decode_md5(Hex) when is_list(Hex) ->+ decode_md5(list_to_binary(Hex));+decode_md5(Hex) when byte_size(Hex) =:= 32 ->+ try binary:decode_hex(Hex) of+ Md5 -> {ok, Md5}+ catch+ error:badarg -> error+ end;+decode_md5(_Hex) ->+ error.++-spec as_pins(term()) -> #{guild_id() => node()}.+as_pins(Pins) when is_map(Pins) ->+ eqwalizer:dynamic_cast(+ maps:filter(fun(Key, Value) -> is_integer(Key) andalso is_atom(Value) end, Pins)+ );+as_pins(_Pins) ->+ #{}.++-spec pin_guilds(#{guild_id() => node()}, [guild_id()], node()) -> #{guild_id() => node()}.+pin_guilds(Pins, GuildIds, Owner) ->+ maps:merge(Pins, maps:from_list([{GuildId, Owner} || GuildId <- GuildIds])).++-spec unpin_guilds(#{guild_id() => node()}, [guild_id()], node()) -> #{guild_id() => node()}.+unpin_guilds(Pins, GuildIds, Owner) ->+ maps:filter(+ fun(GuildId, Node) -> not (Node =:= Owner andalso lists:member(GuildId, GuildIds)) end,+ Pins+ ).++-spec pin_local([guild_id()]) -> ok.+pin_local(GuildIds) ->+ Pins = gateway_guild_pin_remote:as_pins(+ application:get_env(fluxer_gateway, guild_owner_pins, #{})+ ),+ application:set_env(+ fluxer_gateway,+ guild_owner_pins,+ gateway_guild_pin_remote:pin_guilds(Pins, GuildIds, node())+ ).++-spec unpin_local([guild_id()]) -> ok.+unpin_local(GuildIds) ->+ Pins = gateway_guild_pin_remote:as_pins(+ application:get_env(fluxer_gateway, guild_owner_pins, #{})+ ),+ application:set_env(+ fluxer_gateway,+ guild_owner_pins,+ gateway_guild_pin_remote:unpin_guilds(Pins, GuildIds, node())+ ).++-spec routed_here(guild_id(), [node()]) -> boolean().+routed_here(_GuildId, []) ->+ true;+routed_here(GuildId, Peers) ->+ Answers = erpc:multicall(+ Peers, ?ROUTER, owner_node_result, [GuildId, guilds], ?RPC_TIMEOUT_MS+ ),+ lists:all(fun(Answer) -> Answer =:= {ok, {ok, node()}} end, Answers).++-spec release_peers([node()], [guild_id()]) -> [{node(), term()}].+release_peers(Peers, GuildIds) ->+ release_peers(Peers, GuildIds, ?RELEASE_ROUNDS).++-spec release_peers([node()], [guild_id()], pos_integer()) -> [{node(), term()}].+release_peers([], _GuildIds, _Left) ->+ [];+release_peers(Peers, GuildIds, Left) ->+ _ = run_parallel(+ fun(Node) -> gateway_guild_pin_remote:unpin_peer(Node, GuildIds) end,+ Peers,+ ?PATCH_TIMEOUT_MS+ ),+ Checked = run_parallel(+ fun(Node) -> gateway_guild_pin_remote:routes_here(Node, GuildIds) end,+ Peers,+ ?PATCH_TIMEOUT_MS+ ),+ case [{Node, Answer} || {Node, Answer} <- Checked, Answer =/= released] of+ [] ->+ [];+ Still when Left =:= 1 ->+ Still;+ Still ->+ timer:sleep(?RELEASE_RETRY_MS),+ release_peers([Node || {Node, _Answer} <- Still], GuildIds, Left - 1)+ end.++-spec run_parallel(fun((node()) -> term()), [node()], pos_integer()) -> [{node(), term()}].+run_parallel(Fun, Nodes, TimeoutMs) ->+ Parent = self(),+ Ref = make_ref(),+ Pids = [{spawn(fun() -> Parent ! {Ref, Node, Fun(Node)} end), Node} || Node <- Nodes],+ Deadline = erlang:monotonic_time(millisecond) + TimeoutMs,+ Results = collect_parallel(Ref, Nodes, #{}, Deadline),+ [exit(Pid, kill) || {Pid, Node} <- Pids, not maps:is_key(Node, Results)],+ [{Node, maps:get(Node, Results, {error, timeout})} || Node <- Nodes].++-spec collect_parallel(reference(), [node()], #{node() => term()}, integer()) ->+ #{node() => term()}.+collect_parallel(_Ref, Nodes, Results, _Deadline) when map_size(Results) =:= length(Nodes) ->+ Results;+collect_parallel(Ref, Nodes, Results, Deadline) ->+ Wait = max(0, Deadline - erlang:monotonic_time(millisecond)),+ receive+ {Ref, Node, Result} -> collect_parallel(Ref, Nodes, Results#{Node => Result}, Deadline)+ after Wait ->+ Results+ end.