Skip to content

Agent Mesh

update relays

Antonio Ojeac2886b17fe4d
diff --git a/cmd/sam-node/connection_monitor.go b/cmd/sam-node/connection_monitor.go
index a44ca8a0..2cfe00a7 100644
--- a/cmd/sam-node/connection_monitor.go
+++ b/cmd/sam-node/connection_monitor.go
@@ -14,6 +14,7 @@ type hubConnectionManager interface {
LoadHubURL() (string, error)
SaveHubConfig(pubKey []byte, addrs []string) error
+ UpdateRelays(addrs []multiaddr.Multiaddr)
}
// checkHubConnection monitors the connection to the hub and attempts to recover it if disconnected.
@@ -46,6 +47,7 @@ func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable b
for _, addr := range p2pAddrs {
if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {
logger.Infof("[Monitor] Successfully reconnected to Hub via P2P.")
+ mgr.UpdateRelays([]multiaddr.Multiaddr{addr})
return false, true
}
}
@@ -81,6 +83,7 @@ func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable b
for _, addr := range newHubAddrs {
if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {
logger.Infof("[Monitor] Successfully reconnected to Hub via HTTP fallback.")
+ mgr.UpdateRelays(newHubAddrs)
return false, true
}
}
diff --git a/cmd/sam-node/connection_monitor_test.go b/cmd/sam-node/connection_monitor_test.go
index 45232270..75eaf622 100644
--- a/cmd/sam-node/connection_monitor_test.go
+++ b/cmd/sam-node/connection_monitor_test.go
@@ -55,6 +55,8 @@ func (m *mockHubConnectionManager) SaveHubConfig(pubKey []byte, addrs []string)
return m.saveConfigErr
}
+func (m *mockHubConnectionManager) UpdateRelays(addrs []multiaddr.Multiaddr) {}
+
func TestCheckHubConnection(t *testing.T) {
cases := []struct {
name string
diff --git a/cmd/sam-node/gate_test.go b/cmd/sam-node/gate_test.go
index fde70fd1..5e9e2341 100644
--- a/cmd/sam-node/gate_test.go
+++ b/cmd/sam-node/gate_test.go
@@ -111,8 +111,20 @@ func startBareNode(t *testing.T, ctx context.Context) (*SamNode, func()) {
if err != nil {
t.Fatal(err)
}
- node, err := NewSamNode(ctx, priv, nil, nil, store, "test-mesh", "1s",
- []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)
+ node, err := NewSamNode(ctx, SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: nil,
+ Store: store,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatal(err)
}
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index fe18a9e9..a53d0f8b 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -54,24 +54,26 @@ var (
hubAddr string
listenAddrs []string
- jwtFlag string
- jwtPathFlag string
- clientIDFlag string
- clientSecretFlag string
- oidcIssuerFlag string
- deviceAuthURLFlag string
- audienceFlag string
- hubPublicKeyFlag string
- bindAddrFlag string
- meshFlag string
- discoveryIntervalFlag string
- enableRelayFlag bool
- logLevelFlag string
- configFile string
- keyGracePeriodFlag time.Duration
- dataDirFlag string
- allowLoopbackFlag bool
- headlessFlag bool
+ jwtFlag string
+ jwtPathFlag string
+ clientIDFlag string
+ clientSecretFlag string
+ oidcIssuerFlag string
+ deviceAuthURLFlag string
+ audienceFlag string
+ hubPublicKeyFlag string
+ bindAddrFlag string
+ meshFlag string
+ discoveryIntervalFlag string
+ monitorBootstrapFlag time.Duration
+ monitorCheckIntervalFlag time.Duration
+ enableRelayFlag bool
+ logLevelFlag string
+ configFile string
+ keyGracePeriodFlag time.Duration
+ dataDirFlag string
+ allowLoopbackFlag bool
+ headlessFlag bool
apiTokenFlag string
tlsCertFlag string
@@ -177,7 +179,21 @@ func main() {
logger.Fatal("Hub public key not found in store and not provided. Cannot verify peers.")
}
priv := getOrGenerateKey(store)
- node, err = NewSamNode(context.Background(), priv, hubPubKey, hubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)
+ node, err = NewSamNode(context.Background(), SamNodeConfig{
+ PrivKey: priv,
+ HubPubKey: hubPubKey,
+ HubAddrs: hubAddrs,
+ Store: store,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: listenAddrs,
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ MonitorBootstrap: monitorBootstrapFlag,
+ MonitorInterval: monitorCheckIntervalFlag,
+ })
if err != nil {
logger.Fatalf("Failed to start mesh node: %v", err)
}
@@ -216,7 +232,20 @@ func main() {
priv := getOrGenerateKey(store)
enrollCtx, enrollCancel := context.WithCancel(context.Background())
- node, err = NewSamNode(enrollCtx, priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)
+ node, err = NewSamNode(enrollCtx, SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: initHubAddrs,
+ Store: store,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: listenAddrs,
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ MonitorBootstrap: monitorBootstrapFlag,
+ MonitorInterval: monitorCheckIntervalFlag,
+ })
if err != nil {
enrollCancel()
logger.Fatalf("Failed to initialize node for enrollment: %v", err)
@@ -243,7 +272,21 @@ func main() {
hubPubKey = storedPubKey
logger.Debugf("listenAddrs: %v, allowLoopback: %v", listenAddrs, allowLoopbackFlag)
- node, err = NewSamNode(context.Background(), priv, hubPubKey, newHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)
+ node, err = NewSamNode(context.Background(), SamNodeConfig{
+ PrivKey: priv,
+ HubPubKey: hubPubKey,
+ HubAddrs: newHubAddrs,
+ Store: store,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: listenAddrs,
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ MonitorBootstrap: monitorBootstrapFlag,
+ MonitorInterval: monitorCheckIntervalFlag,
+ })
if err != nil {
logger.Fatalf("Failed to start mesh node after enrollment: %v", err)
}
@@ -374,7 +417,20 @@ func main() {
}
priv := getOrGenerateKey(store)
- node, err := NewSamNode(context.Background(), priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"}, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)
+ node, err := NewSamNode(context.Background(), SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: initHubAddrs,
+ Store: store,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"},
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
logger.Fatalf("Failed to initialize node for enrollment: %v", err)
}
@@ -401,6 +457,8 @@ func main() {
runCmd.Flags().StringVar(&bindAddrFlag, "bind-addr", "127.0.0.1:8080", "Local TCP address for the HTTP server (MCP and Sidecar API)")
runCmd.Flags().StringVar(&meshFlag, "mesh", DefaultMeshName, "Mesh federation name")
runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", DefaultDiscoveryInterval, "Polling interval for DHT discovery")
+ runCmd.Flags().DurationVar(&monitorBootstrapFlag, "monitor-bootstrap", 2*time.Minute, "Initial wait before monitoring hub connection")
+ runCmd.Flags().DurationVar(&monitorCheckIntervalFlag, "monitor-interval", 1*time.Minute, "Interval for checking hub connection")
runCmd.Flags().BoolVar(&enableRelayFlag, "enable-relay", false, "Allow this node to serve as a relay for others")
runCmd.Flags().StringVar(&logLevelFlag, "log-level", "info", "Log level (debug, info, warn, error)")
runCmd.Flags().DurationVar(&keyGracePeriodFlag, "key-grace-period", 24*time.Hour, "Key grace period for old keys (e.g. 24h)")
@@ -550,6 +608,8 @@ func (n *SamNode) Enroll(ctx context.Context, jwt string) error {
n.keysMu.Unlock()
// Connect and Auth to hub after enrollment to join the mesh
+ var lastAuthErr error
+ var authed bool
for _, addrStr := range enrollResp.HubAddresses {
addr, err := multiaddr.NewMultiaddr(addrStr)
if err != nil {
@@ -558,11 +618,17 @@ func (n *SamNode) Enroll(ctx context.Context, jwt string) error {
}
if err := n.ConnectAndAuthWithHub(ctx, addr); err != nil {
logger.Warnf("Failed to connect and auth with hub after enrollment: %v", err)
+ lastAuthErr = err
} else {
+ authed = true
break
}
}
+ if !authed {
+ return fmt.Errorf("failed to connect and authenticate with any hub after HTTP enrollment (last error: %v)", lastAuthErr)
+ }
+
fmt.Println("Successfully enrolled via HTTP and stored identity and hub config.")
return nil
}
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 2595bcaf..d0d2e3ae 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -114,23 +114,56 @@ type SamNode struct {
AllowLoopback bool
authSuccess chan struct{}
authOnce sync.Once
+ currentRelays []peer.AddrInfo
+}
+
+// UpdateRelays updates the current relays used by AutoRelay.
+func (n *SamNode) UpdateRelays(addrs []multiaddr.Multiaddr) {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ var newRelays []peer.AddrInfo
+ for _, addr := range addrs {
+ if addrInfo, err := peer.AddrInfoFromP2pAddr(addr); err == nil && addrInfo.ID != "" {
+ newRelays = append(newRelays, *addrInfo)
+ n.Host.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL)
+ }
+ }
+ n.currentRelays = newRelays
+ logger.Infof("[Relay] Updated current relays for AutoRelay: %v", newRelays)
+}
+
+// SamNodeConfig holds all configuration options for a SamNode.
+type SamNodeConfig struct {
+ PrivKey crypto.PrivKey
+ HubPubKey ed25519.PublicKey
+ HubAddrs []multiaddr.Multiaddr
+ Store *Store
+ MeshID string
+ DiscoveryInterval string
+ ListenAddrs []string
+ EnableRelay bool
+ NodeConfig *NodeConfigComplete
+ KeyGracePeriod time.Duration
+ AllowLoopback bool
+ MonitorBootstrap time.Duration
+ MonitorInterval time.Duration
}
// NewSamNode creates a new Agent instance secured with the 4-layer pipeline.
-func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.PublicKey, hubAddrs []multiaddr.Multiaddr, store *Store, meshID string, discoveryInterval string, listenAddrs []string, enableRelay bool, nodeConfig *NodeConfigComplete, keyGracePeriod time.Duration, allowLoopback bool) (*SamNode, error) {
+func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {
var trustedKeys []TrustedKey
- if len(hubPubKey) > 0 {
- trustedKeys = []TrustedKey{{Key: hubPubKey, ReceivedAt: time.Now()}}
+ if len(cfg.HubPubKey) > 0 {
+ trustedKeys = []TrustedKey{{Key: cfg.HubPubKey, ReceivedAt: time.Now()}}
}
node := &SamNode{
- Store: store,
+ Store: cfg.Store,
trustedKeys: trustedKeys,
peerLastEventTime: make(map[string]int64),
receivedMsgs: make(map[string][]string),
topics: make(map[string]*pubsub.Topic),
- LocalPolicy: nodeConfig,
- AllowLoopback: allowLoopback,
+ LocalPolicy: cfg.NodeConfig,
+ AllowLoopback: cfg.AllowLoopback,
authSuccess: make(chan struct{}),
}
@@ -154,7 +187,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
// Convert Hub multiaddrs to peer.AddrInfo to use as static relays
var staticRelays []peer.AddrInfo
- for _, addr := range hubAddrs {
+ for _, addr := range cfg.HubAddrs {
if addrInfo, err := peer.AddrInfoFromP2pAddr(addr); err == nil && addrInfo.ID != "" {
staticRelays = append(staticRelays, *addrInfo)
if node.HubPeerID == "" {
@@ -175,11 +208,11 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
// Layer 1: Establish FIPS-compliant Transports & NAT Services
opts := []libp2p.Option{
- libp2p.Identity(privKey),
+ libp2p.Identity(cfg.PrivKey),
libp2p.DefaultTransports,
libp2p.Security(libp2ptls.ID, libp2ptls.New),
libp2p.ConnectionGater(gater),
- libp2p.ListenAddrStrings(listenAddrs...),
+ libp2p.ListenAddrStrings(cfg.ListenAddrs...),
libp2p.EnableNATService(),
libp2p.EnableAutoNATv2(),
libp2p.ForceReachabilityPrivate(),
@@ -187,7 +220,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
libp2p.EnableHolePunching(),
libp2p.ConnectionManager(cm),
libp2p.AddrsFactory(func(addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {
- if allowLoopback {
+ if cfg.AllowLoopback {
return addrs
}
var filtered []multiaddr.Multiaddr
@@ -202,10 +235,15 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
// If we have a Hub, configure it as our static fallback relay for NAT hole-punching
if len(staticRelays) > 0 {
+ node.currentRelays = staticRelays
opts = append(opts, libp2p.EnableAutoRelayWithPeerSource(
func(ctx context.Context, numPeers int) <-chan peer.AddrInfo {
logger.Infof("[Relay] AutoRelay called PeerSource for %d peers", numPeers)
- c := make(chan peer.AddrInfo, len(staticRelays))
+ node.mu.Lock()
+ currentRelays := node.currentRelays
+ node.mu.Unlock()
+
+ c := make(chan peer.AddrInfo, len(currentRelays))
go func() {
defer close(c)
select {
@@ -213,7 +251,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
logger.Infof("[Relay] PeerSource context done")
case <-node.authSuccess:
logger.Infof("[Relay] Yielding static relays to AutoRelay")
- for _, r := range staticRelays {
+ for _, r := range currentRelays {
c <- r
}
}
@@ -226,7 +264,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
}
// If the user explicitly opts in, allow this node to serve as a relay for others
- if enableRelay {
+ if cfg.EnableRelay {
logger.Infof("[Relay] Enabling Relay Service")
}
@@ -241,7 +279,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
h.Peerstore().AddAddrs(pi.ID, pi.Addrs, peerstore.PermanentAddrTTL)
}
- if enableRelay {
+ if cfg.EnableRelay {
logger.Infof("[Relay] Enabling Relay Service with ACL")
_, err = relay.New(h, relay.WithACL(&nodeRelayACL{node: node}))
if err != nil {
@@ -265,7 +303,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
var authenticated bool
var fatalAuthErr error
- for _, addr := range hubAddrs {
+ for _, addr := range cfg.HubAddrs {
if err := node.ConnectAndAuthWithHub(ctx, addr); err != nil {
logger.Warnf("[AuthN] Failed to bootstrap and auth with hub %s: %v", addr, err)
if errors.Is(err, ErrFatalAuth) {
@@ -277,7 +315,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
}
}
- if len(hubAddrs) > 0 && !authenticated {
+ if len(cfg.HubAddrs) > 0 && !authenticated {
if fatalAuthErr != nil {
return nil, fmt.Errorf("fatal auth failure: %w", fatalAuthErr)
}
@@ -314,14 +352,14 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
// Listen for Network Evictions/Revocations from the Hub
go node.listenForHubEvents(ctx)
- interval, err := time.ParseDuration(discoveryInterval)
+ interval, err := time.ParseDuration(cfg.DiscoveryInterval)
if err != nil {
- logger.Warnf("[Discovery] Invalid discovery interval '%s', using default %s: %v", discoveryInterval, DefaultDiscoveryInterval, err)
+ logger.Warnf("[Discovery] Invalid discovery interval '%s', using default %s: %v", cfg.DiscoveryInterval, DefaultDiscoveryInterval, err)
interval, _ = time.ParseDuration(DefaultDiscoveryInterval)
}
// Start DHT Discovery
- go node.startDiscovery(ctx, meshID, interval)
+ go node.startDiscovery(ctx, cfg.MeshID, interval)
// Layer 3: Open the Lobby Door (Auth Protocol is bypassed by Layer 4)
node.Host.SetStreamHandler(api.AuthProtocolID, node.HandleAuthHandshake)
@@ -330,7 +368,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
node.Host.SetStreamHandler(api.MCPProtocolID, node.WithBiscuitAuth(node.HandleMCPStream))
// Start key pruning
- node.startKeyPruning(ctx, keyGracePeriod)
+ node.startKeyPruning(ctx, cfg.KeyGracePeriod)
// Start Ingress HTTP Server
if err := node.StartIngressServer(ctx); err != nil {
@@ -338,7 +376,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
}
// Start connection monitor
- node.startConnectionMonitor(ctx, 2*time.Minute, 1*time.Minute, 3)
+ node.startConnectionMonitor(ctx, cfg.MonitorBootstrap, cfg.MonitorInterval, 3)
return node, nil
}
diff --git a/cmd/sam-node/node_test.go b/cmd/sam-node/node_test.go
index e7a3a16d..15aa9011 100644
--- a/cmd/sam-node/node_test.go
+++ b/cmd/sam-node/node_test.go
@@ -98,7 +98,20 @@ func TestConnectionMonitor_CrashesAfterFailures(t *testing.T) {
if os.Getenv("BE_CRASHER_MONITOR") == "1" {
priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)
store, _ := NewStore(t.TempDir())
- node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)
+ node, err := NewSamNode(context.Background(), SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: nil,
+ Store: store,
+ MeshID: "test",
+ DiscoveryInterval: "10s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: nil,
+ KeyGracePeriod: 0,
+ AllowLoopback: false,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
os.Exit(0) // Ignore NewSamNode errors for this crasher
}
diff --git a/cmd/sam-node/proxy_test.go b/cmd/sam-node/proxy_test.go
index 0f61af06..659c3055 100644
--- a/cmd/sam-node/proxy_test.go
+++ b/cmd/sam-node/proxy_test.go
@@ -53,7 +53,20 @@ func TestDatapathIntegration(t *testing.T) {
// Pre-populate stores with dummy keys to avoid enrollment failure if required
// For this test, we assume we can run without full enrollment if we bypass AuthHandler
- nodeA, err := NewSamNode(ctx, privA, nil, nil, storeA, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)
+ nodeA, err := NewSamNode(ctx, SamNodeConfig{
+ PrivKey: privA,
+ HubAddrs: nil,
+ Store: storeA,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatal(err)
}
@@ -63,7 +76,20 @@ func TestDatapathIntegration(t *testing.T) {
}
}()
- nodeB, err := NewSamNode(ctx, privB, nil, nil, storeB, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)
+ nodeB, err := NewSamNode(ctx, SamNodeConfig{
+ PrivKey: privB,
+ HubAddrs: nil,
+ Store: storeB,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatal(err)
}
@@ -212,13 +238,39 @@ func TestStdioDatapathIntegration(t *testing.T) {
}
defer func() { _ = storeB.Close() }()
- nodeA, err := NewSamNode(ctx, privA, nil, nil, storeA, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)
+ nodeA, err := NewSamNode(ctx, SamNodeConfig{
+ PrivKey: privA,
+ HubAddrs: nil,
+ Store: storeA,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatal(err)
}
defer func() { _ = nodeA.Host.Close() }()
- nodeB, err := NewSamNode(ctx, privB, nil, nil, storeB, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)
+ nodeB, err := NewSamNode(ctx, SamNodeConfig{
+ PrivKey: privB,
+ HubAddrs: nil,
+ Store: storeB,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatal(err)
}
diff --git a/cmd/sam-node/sidecar_test.go b/cmd/sam-node/sidecar_test.go
index 382c310f..5c7160a1 100644
--- a/cmd/sam-node/sidecar_test.go
+++ b/cmd/sam-node/sidecar_test.go
@@ -501,8 +501,20 @@ func TestRegisterService_PopulatesAggregatedTools(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- node, err := NewSamNode(ctx, priv, nil, nil, store, "test-mesh", "1s",
- []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)
+ node, err := NewSamNode(ctx, SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: nil,
+ Store: store,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatal(err)
}
@@ -553,8 +565,20 @@ func TestUnregisterService_DetachesAggregation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- node, err := NewSamNode(ctx, priv, nil, nil, store, "test-mesh", "1s",
- []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)
+ node, err := NewSamNode(ctx, SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: nil,
+ Store: store,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "1s",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: &NodeConfigComplete{},
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatal(err)
}
diff --git a/cmd/sam-node/static_service_test.go b/cmd/sam-node/static_service_test.go
index 32b4aeae..28ad55b3 100644
--- a/cmd/sam-node/static_service_test.go
+++ b/cmd/sam-node/static_service_test.go
@@ -14,7 +14,9 @@ import (
"github.com/libp2p/go-libp2p"
dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-msgio"
"github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/multiformats/go-multiaddr"
"google.golang.org/protobuf/proto"
@@ -32,6 +34,16 @@ func startMockLibp2pHub(t *testing.T) (peer.ID, string) {
t.Fatalf("failed to create DHT on mock hub: %v", err)
}
+ // Add dummy auth handler
+ h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {
+ defer func() { _ = s.Close() }()
+ // Just send a success response to make the client happy
+ resp := &api.AuthResponse{Success: true}
+ data, _ := proto.Marshal(resp)
+ writer := msgio.NewVarintWriter(s)
+ _ = writer.WriteMsg(data)
+ })
+
// Start HTTP server for enrollment
mux := http.NewServeMux()
mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {
@@ -94,7 +106,20 @@ func TestStaticServiceRegistrationRequiresConnection(t *testing.T) {
}
// 4. Create Node with empty hub addrs initially
- node, err := NewSamNode(context.Background(), priv, nil, []multiaddr.Multiaddr{}, store, "test-mesh", "100ms", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 24*time.Hour, true)
+ node, err := NewSamNode(context.Background(), SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: []multiaddr.Multiaddr{},
+ Store: store,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "100ms",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: nil,
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatalf("failed to create node: %v", err)
}
@@ -167,7 +192,20 @@ func TestStaticServiceRegistrationCommandFailure(t *testing.T) {
}
// 4. Create Node with empty hub addrs initially
- node, err := NewSamNode(context.Background(), priv, nil, []multiaddr.Multiaddr{}, store, "test-mesh", "100ms", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 24*time.Hour, true)
+ node, err := NewSamNode(context.Background(), SamNodeConfig{
+ PrivKey: priv,
+ HubAddrs: []multiaddr.Multiaddr{},
+ Store: store,
+ MeshID: "test-mesh",
+ DiscoveryInterval: "100ms",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ EnableRelay: false,
+ NodeConfig: nil,
+ KeyGracePeriod: 24 * time.Hour,
+ AllowLoopback: true,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ })
if err != nil {
t.Fatalf("failed to create node: %v", err)
}
diff --git a/tests/integration/failover_test.go b/tests/integration/failover_test.go
new file mode 100644
index 00000000..1a7128ef
--- /dev/null
+++ b/tests/integration/failover_test.go
@@ -0,0 +1,233 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration_test
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/http/httputil"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+)
+
+func TestFailoverUpdatesRelay(t *testing.T) {
+ hubBin := buildBinary(t, "./cmd/sam-hub")
+ nodeBin := buildBinary(t, "./cmd/sam-node")
+
+ tmpDir := t.TempDir()
+
+ // Create a mock policy file
+ policyFile := filepath.Join(tmpDir, "policies.yaml")
+ policyContent := `version: "v1alpha1"
+bindings: []
+roles: {}
+`
+ if err := os.WriteFile(policyFile, []byte(policyContent), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ httpPortA := getFreePort(t)
+ p2pPortA := getFreePort(t)
+ httpPortB := getFreePort(t)
+ p2pPortB := getFreePort(t)
+
+ // Mock OIDC
+ oidcURL, jwtTokenStr := startMockOIDC(t)
+ jwtPath := filepath.Join(tmpDir, "jwt.txt")
+ if err := os.WriteFile(jwtPath, []byte(jwtTokenStr), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ // Dynamic Load Balancer for HTTP discovery
+ var lbMu sync.Mutex
+ activeHubPort := httpPortA
+
+ lb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ lbMu.Lock()
+ targetPort := activeHubPort
+ lbMu.Unlock()
+
+ target, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", targetPort))
+ proxy := httputil.NewSingleHostReverseProxy(target)
+ proxy.ServeHTTP(w, r)
+ }))
+ defer lb.Close()
+
+ // Start Hub A
+ cmdA := exec.Command(hubBin,
+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", httpPortA),
+ "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p2pPortA),
+ "--policy-file", policyFile,
+ "--keys-db", filepath.Join(tmpDir, "keysA.db"),
+ "--allow-loopback",
+ "--issuer", oidcURL,
+ "--insecure-skip-tls-verify",
+ )
+ var stdoutA, stderrA safeBuffer
+ cmdA.Stdout = &stdoutA
+ cmdA.Stderr = &stderrA
+ if err := cmdA.Start(); err != nil {
+ t.Fatalf("failed to start Hub A: %v", err)
+ }
+
+ // Fetch Hub A's Peer ID
+ peerIDA := fetchPeerID(t, httpPortA)
+ t.Logf("Hub A PeerID: %s", peerIDA) // Copy keysA.db to keysB.db to avoid bbolt file lock conflicts
+ keysA := filepath.Join(tmpDir, "keysA.db")
+ keysB := filepath.Join(tmpDir, "keysB.db")
+ data, err := os.ReadFile(keysA)
+ if err != nil {
+ t.Fatalf("Failed to read keysA.db: %v", err)
+ }
+ if err := os.WriteFile(keysB, data, 0600); err != nil {
+ t.Fatalf("Failed to write keysB.db: %v", err)
+ }
+
+ // Start Hub B
+ cmdB := exec.Command(hubBin,
+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", httpPortB),
+ "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p2pPortB),
+ "--policy-file", policyFile,
+ "--keys-db", keysB,
+ "--allow-loopback",
+ "--issuer", oidcURL,
+ "--insecure-skip-tls-verify",
+ )
+ var stdoutB, stderrB safeBuffer
+ cmdB.Stdout = &stdoutB
+ cmdB.Stderr = &stderrB
+ if err := cmdB.Start(); err != nil {
+ t.Fatalf("failed to start Hub B: %v", err)
+ }
+ defer func() { _ = cmdB.Process.Kill() }()
+
+ peerIDB := fetchPeerID(t, httpPortB)
+ t.Logf("Hub B PeerID: %s", peerIDB)
+
+ // Start Node B
+ nodeHome := filepath.Join(tmpDir, "node-home")
+ env := append(os.Environ(),
+ "HOME="+nodeHome,
+ "XDG_CONFIG_HOME="+filepath.Join(nodeHome, ".config"),
+ )
+ cmdNode := exec.Command(nodeBin, "run",
+ "--hub", lb.URL,
+ "--listen", "/ip4/127.0.0.1/tcp/0",
+ "--jwt-path", jwtPath,
+ "--bind-addr", "127.0.0.1:0",
+ "--api-token", "dummy-token",
+ "--allow-loopback",
+ "--monitor-bootstrap", "1s",
+ "--monitor-interval", "1s",
+ )
+ cmdNode.Dir = repoRoot(t)
+ cmdNode.Env = env
+ var stdoutNode, stderrNode safeBuffer
+ cmdNode.Stdout = &stdoutNode
+ cmdNode.Stderr = &stderrNode
+
+ if err := cmdNode.Start(); err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = cmdNode.Process.Kill() }()
+
+ // Wait for Node B to get a reservation on Hub A
+ var nodePeerID string
+ var out string
+ for i := 0; i < 100; i++ {
+ out = stdoutNode.String() + stderrNode.String()
+ if strings.Contains(out, "PeerID:") {
+ idx := strings.Index(out, "PeerID:")
+ parts := strings.Split(strings.TrimSpace(out[idx+len("PeerID:"):]), "\n")
+ if len(parts) > 0 {
+ nodePeerID = strings.TrimSpace(parts[0])
+ }
+ }
+ if strings.Contains(out, "Yielding static relays to AutoRelay") {
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ if nodePeerID == "" {
+ t.Fatalf("Node B failed to get PeerID in time.\nOutput:\n%s", out)
+ }
+ t.Logf("Node B started. Node PeerID: %s", nodePeerID)
+
+ // Now FAILOVER: Switch LB to Hub B and KILL Hub A
+ t.Logf("Initiating failover to Hub B...")
+ lbMu.Lock()
+ activeHubPort = httpPortB
+ lbMu.Unlock()
+ _ = cmdA.Process.Kill()
+
+ // Wait for Node B's AutoRelay to get updated
+ for i := 0; i < 150; i++ {
+ out = stdoutNode.String() + stderrNode.String()
+ if strings.Contains(out, "Successfully reconnected to Hub via HTTP fallback.") {
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ if !strings.Contains(out, "Successfully reconnected to Hub via HTTP fallback.") {
+ t.Fatalf("Node B failed to detect failover and reconnect.\nOutput:\n%s", out)
+ }
+ t.Log("Node B successfully reconnected to Hub B!")
+
+ // Final verification: Ensure we can actually reach Node B via the Hub B relay
+ clientHost, err := libp2p.New(libp2p.NoListenAddrs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer clientHost.Close() //nolint:errcheck
+
+ relayAddrStr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s/p2p-circuit/p2p/%s", p2pPortB, peerIDB, nodePeerID)
+ relayAddr, err := multiaddr.NewMultiaddr(relayAddrStr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ addrInfo, err := peer.AddrInfoFromP2pAddr(relayAddr)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+ defer cancel()
+
+ var connectErr error
+ for i := 0; i < 15; i++ {
+ connectErr = clientHost.Connect(ctx, *addrInfo)
+ if connectErr == nil {
+ break
+ }
+ time.Sleep(1 * time.Second)
+ }
+
+ if connectErr != nil {
+ t.Fatalf("Failed to connect to Node B via Hub B relay: %v\nOutput: %s", connectErr, stdoutNode.String()+stderrNode.String())
+ }
+ t.Log("Successfully connected to Node B via Hub B relay circuit!")
+}