Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
update relays
Antonio Ojeac2886b17fe4d
diff --git a/cmd/sam-node/connection_monitor.go b/cmd/sam-node/connection_monitor.goindex a44ca8a0..2cfe00a7 100644--- a/cmd/sam-node/connection_monitor.go+++ b/cmd/sam-node/connection_monitor.go@@ -14,6 +14,7 @@ type hubConnectionManager interface {LoadHubURL() (string, error)SaveHubConfig(pubKey []byte, addrs []string) error+ UpdateRelays(addrs []multiaddr.Multiaddr)}// checkHubConnection monitors the connection to the hub and attempts to recover it if disconnected.@@ -46,6 +47,7 @@ func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable bfor _, addr := range p2pAddrs {if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {logger.Infof("[Monitor] Successfully reconnected to Hub via P2P.")+ mgr.UpdateRelays([]multiaddr.Multiaddr{addr})return false, true}}@@ -81,6 +83,7 @@ func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable bfor _, addr := range newHubAddrs {if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {logger.Infof("[Monitor] Successfully reconnected to Hub via HTTP fallback.")+ mgr.UpdateRelays(newHubAddrs)return false, true}}diff --git a/cmd/sam-node/connection_monitor_test.go b/cmd/sam-node/connection_monitor_test.goindex 45232270..75eaf622 100644--- a/cmd/sam-node/connection_monitor_test.go+++ b/cmd/sam-node/connection_monitor_test.go@@ -55,6 +55,8 @@ func (m *mockHubConnectionManager) SaveHubConfig(pubKey []byte, addrs []string)return m.saveConfigErr}+func (m *mockHubConnectionManager) UpdateRelays(addrs []multiaddr.Multiaddr) {}+func TestCheckHubConnection(t *testing.T) {cases := []struct {name stringdiff --git a/cmd/sam-node/gate_test.go b/cmd/sam-node/gate_test.goindex fde70fd1..5e9e2341 100644--- a/cmd/sam-node/gate_test.go+++ b/cmd/sam-node/gate_test.go@@ -111,8 +111,20 @@ func startBareNode(t *testing.T, ctx context.Context) (*SamNode, func()) {if err != nil {t.Fatal(err)}- node, err := NewSamNode(ctx, priv, nil, nil, store, "test-mesh", "1s",- []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)+ node, err := NewSamNode(ctx, SamNodeConfig{+ PrivKey: priv,+ HubAddrs: nil,+ Store: store,+ MeshID: "test-mesh",+ DiscoveryInterval: "1s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: &NodeConfigComplete{},+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatal(err)}diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.goindex fe18a9e9..a53d0f8b 100644--- a/cmd/sam-node/main.go+++ b/cmd/sam-node/main.go@@ -54,24 +54,26 @@ var (hubAddr stringlistenAddrs []string- jwtFlag string- jwtPathFlag string- clientIDFlag string- clientSecretFlag string- oidcIssuerFlag string- deviceAuthURLFlag string- audienceFlag string- hubPublicKeyFlag string- bindAddrFlag string- meshFlag string- discoveryIntervalFlag string- enableRelayFlag bool- logLevelFlag string- configFile string- keyGracePeriodFlag time.Duration- dataDirFlag string- allowLoopbackFlag bool- headlessFlag bool+ jwtFlag string+ jwtPathFlag string+ clientIDFlag string+ clientSecretFlag string+ oidcIssuerFlag string+ deviceAuthURLFlag string+ audienceFlag string+ hubPublicKeyFlag string+ bindAddrFlag string+ meshFlag string+ discoveryIntervalFlag string+ monitorBootstrapFlag time.Duration+ monitorCheckIntervalFlag time.Duration+ enableRelayFlag bool+ logLevelFlag string+ configFile string+ keyGracePeriodFlag time.Duration+ dataDirFlag string+ allowLoopbackFlag bool+ headlessFlag boolapiTokenFlag stringtlsCertFlag string@@ -177,7 +179,21 @@ func main() {logger.Fatal("Hub public key not found in store and not provided. Cannot verify peers.")}priv := getOrGenerateKey(store)- node, err = NewSamNode(context.Background(), priv, hubPubKey, hubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)+ node, err = NewSamNode(context.Background(), SamNodeConfig{+ PrivKey: priv,+ HubPubKey: hubPubKey,+ HubAddrs: hubAddrs,+ Store: store,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: listenAddrs,+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ MonitorBootstrap: monitorBootstrapFlag,+ MonitorInterval: monitorCheckIntervalFlag,+ })if err != nil {logger.Fatalf("Failed to start mesh node: %v", err)}@@ -216,7 +232,20 @@ func main() {priv := getOrGenerateKey(store)enrollCtx, enrollCancel := context.WithCancel(context.Background())- node, err = NewSamNode(enrollCtx, priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)+ node, err = NewSamNode(enrollCtx, SamNodeConfig{+ PrivKey: priv,+ HubAddrs: initHubAddrs,+ Store: store,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: listenAddrs,+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ MonitorBootstrap: monitorBootstrapFlag,+ MonitorInterval: monitorCheckIntervalFlag,+ })if err != nil {enrollCancel()logger.Fatalf("Failed to initialize node for enrollment: %v", err)@@ -243,7 +272,21 @@ func main() {hubPubKey = storedPubKeylogger.Debugf("listenAddrs: %v, allowLoopback: %v", listenAddrs, allowLoopbackFlag)- node, err = NewSamNode(context.Background(), priv, hubPubKey, newHubAddrs, store, meshFlag, discoveryIntervalFlag, listenAddrs, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)+ node, err = NewSamNode(context.Background(), SamNodeConfig{+ PrivKey: priv,+ HubPubKey: hubPubKey,+ HubAddrs: newHubAddrs,+ Store: store,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: listenAddrs,+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ MonitorBootstrap: monitorBootstrapFlag,+ MonitorInterval: monitorCheckIntervalFlag,+ })if err != nil {logger.Fatalf("Failed to start mesh node after enrollment: %v", err)}@@ -374,7 +417,20 @@ func main() {}priv := getOrGenerateKey(store)- node, err := NewSamNode(context.Background(), priv, nil, initHubAddrs, store, meshFlag, discoveryIntervalFlag, []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"}, enableRelayFlag, nodeConfig, keyGracePeriodFlag, allowLoopbackFlag)+ node, err := NewSamNode(context.Background(), SamNodeConfig{+ PrivKey: priv,+ HubAddrs: initHubAddrs,+ Store: store,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"},+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {logger.Fatalf("Failed to initialize node for enrollment: %v", err)}@@ -401,6 +457,8 @@ func main() {runCmd.Flags().StringVar(&bindAddrFlag, "bind-addr", "127.0.0.1:8080", "Local TCP address for the HTTP server (MCP and Sidecar API)")runCmd.Flags().StringVar(&meshFlag, "mesh", DefaultMeshName, "Mesh federation name")runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", DefaultDiscoveryInterval, "Polling interval for DHT discovery")+ runCmd.Flags().DurationVar(&monitorBootstrapFlag, "monitor-bootstrap", 2*time.Minute, "Initial wait before monitoring hub connection")+ runCmd.Flags().DurationVar(&monitorCheckIntervalFlag, "monitor-interval", 1*time.Minute, "Interval for checking hub connection")runCmd.Flags().BoolVar(&enableRelayFlag, "enable-relay", false, "Allow this node to serve as a relay for others")runCmd.Flags().StringVar(&logLevelFlag, "log-level", "info", "Log level (debug, info, warn, error)")runCmd.Flags().DurationVar(&keyGracePeriodFlag, "key-grace-period", 24*time.Hour, "Key grace period for old keys (e.g. 24h)")@@ -550,6 +608,8 @@ func (n *SamNode) Enroll(ctx context.Context, jwt string) error {n.keysMu.Unlock()// Connect and Auth to hub after enrollment to join the mesh+ var lastAuthErr error+ var authed boolfor _, addrStr := range enrollResp.HubAddresses {addr, err := multiaddr.NewMultiaddr(addrStr)if err != nil {@@ -558,11 +618,17 @@ func (n *SamNode) Enroll(ctx context.Context, jwt string) error {}if err := n.ConnectAndAuthWithHub(ctx, addr); err != nil {logger.Warnf("Failed to connect and auth with hub after enrollment: %v", err)+ lastAuthErr = err} else {+ authed = truebreak}}+ if !authed {+ return fmt.Errorf("failed to connect and authenticate with any hub after HTTP enrollment (last error: %v)", lastAuthErr)+ }+fmt.Println("Successfully enrolled via HTTP and stored identity and hub config.")return nil}diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.goindex 2595bcaf..d0d2e3ae 100644--- a/cmd/sam-node/node.go+++ b/cmd/sam-node/node.go@@ -114,23 +114,56 @@ type SamNode struct {AllowLoopback boolauthSuccess chan struct{}authOnce sync.Once+ currentRelays []peer.AddrInfo+}++// UpdateRelays updates the current relays used by AutoRelay.+func (n *SamNode) UpdateRelays(addrs []multiaddr.Multiaddr) {+ n.mu.Lock()+ defer n.mu.Unlock()+ var newRelays []peer.AddrInfo+ for _, addr := range addrs {+ if addrInfo, err := peer.AddrInfoFromP2pAddr(addr); err == nil && addrInfo.ID != "" {+ newRelays = append(newRelays, *addrInfo)+ n.Host.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL)+ }+ }+ n.currentRelays = newRelays+ logger.Infof("[Relay] Updated current relays for AutoRelay: %v", newRelays)+}++// SamNodeConfig holds all configuration options for a SamNode.+type SamNodeConfig struct {+ PrivKey crypto.PrivKey+ HubPubKey ed25519.PublicKey+ HubAddrs []multiaddr.Multiaddr+ Store *Store+ MeshID string+ DiscoveryInterval string+ ListenAddrs []string+ EnableRelay bool+ NodeConfig *NodeConfigComplete+ KeyGracePeriod time.Duration+ AllowLoopback bool+ MonitorBootstrap time.Duration+ MonitorInterval time.Duration}// NewSamNode creates a new Agent instance secured with the 4-layer pipeline.-func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.PublicKey, hubAddrs []multiaddr.Multiaddr, store *Store, meshID string, discoveryInterval string, listenAddrs []string, enableRelay bool, nodeConfig *NodeConfigComplete, keyGracePeriod time.Duration, allowLoopback bool) (*SamNode, error) {+func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {var trustedKeys []TrustedKey- if len(hubPubKey) > 0 {- trustedKeys = []TrustedKey{{Key: hubPubKey, ReceivedAt: time.Now()}}+ if len(cfg.HubPubKey) > 0 {+ trustedKeys = []TrustedKey{{Key: cfg.HubPubKey, ReceivedAt: time.Now()}}}node := &SamNode{- Store: store,+ Store: cfg.Store,trustedKeys: trustedKeys,peerLastEventTime: make(map[string]int64),receivedMsgs: make(map[string][]string),topics: make(map[string]*pubsub.Topic),- LocalPolicy: nodeConfig,- AllowLoopback: allowLoopback,+ LocalPolicy: cfg.NodeConfig,+ AllowLoopback: cfg.AllowLoopback,authSuccess: make(chan struct{}),}@@ -154,7 +187,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P// Convert Hub multiaddrs to peer.AddrInfo to use as static relaysvar staticRelays []peer.AddrInfo- for _, addr := range hubAddrs {+ for _, addr := range cfg.HubAddrs {if addrInfo, err := peer.AddrInfoFromP2pAddr(addr); err == nil && addrInfo.ID != "" {staticRelays = append(staticRelays, *addrInfo)if node.HubPeerID == "" {@@ -175,11 +208,11 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P// Layer 1: Establish FIPS-compliant Transports & NAT Servicesopts := []libp2p.Option{- libp2p.Identity(privKey),+ libp2p.Identity(cfg.PrivKey),libp2p.DefaultTransports,libp2p.Security(libp2ptls.ID, libp2ptls.New),libp2p.ConnectionGater(gater),- libp2p.ListenAddrStrings(listenAddrs...),+ libp2p.ListenAddrStrings(cfg.ListenAddrs...),libp2p.EnableNATService(),libp2p.EnableAutoNATv2(),libp2p.ForceReachabilityPrivate(),@@ -187,7 +220,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.Plibp2p.EnableHolePunching(),libp2p.ConnectionManager(cm),libp2p.AddrsFactory(func(addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {- if allowLoopback {+ if cfg.AllowLoopback {return addrs}var filtered []multiaddr.Multiaddr@@ -202,10 +235,15 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P// If we have a Hub, configure it as our static fallback relay for NAT hole-punchingif len(staticRelays) > 0 {+ node.currentRelays = staticRelaysopts = append(opts, libp2p.EnableAutoRelayWithPeerSource(func(ctx context.Context, numPeers int) <-chan peer.AddrInfo {logger.Infof("[Relay] AutoRelay called PeerSource for %d peers", numPeers)- c := make(chan peer.AddrInfo, len(staticRelays))+ node.mu.Lock()+ currentRelays := node.currentRelays+ node.mu.Unlock()++ c := make(chan peer.AddrInfo, len(currentRelays))go func() {defer close(c)select {@@ -213,7 +251,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.Plogger.Infof("[Relay] PeerSource context done")case <-node.authSuccess:logger.Infof("[Relay] Yielding static relays to AutoRelay")- for _, r := range staticRelays {+ for _, r := range currentRelays {c <- r}}@@ -226,7 +264,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P}// If the user explicitly opts in, allow this node to serve as a relay for others- if enableRelay {+ if cfg.EnableRelay {logger.Infof("[Relay] Enabling Relay Service")}@@ -241,7 +279,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.Ph.Peerstore().AddAddrs(pi.ID, pi.Addrs, peerstore.PermanentAddrTTL)}- if enableRelay {+ if cfg.EnableRelay {logger.Infof("[Relay] Enabling Relay Service with ACL")_, err = relay.New(h, relay.WithACL(&nodeRelayACL{node: node}))if err != nil {@@ -265,7 +303,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.Pvar authenticated boolvar fatalAuthErr error- for _, addr := range hubAddrs {+ for _, addr := range cfg.HubAddrs {if err := node.ConnectAndAuthWithHub(ctx, addr); err != nil {logger.Warnf("[AuthN] Failed to bootstrap and auth with hub %s: %v", addr, err)if errors.Is(err, ErrFatalAuth) {@@ -277,7 +315,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P}}- if len(hubAddrs) > 0 && !authenticated {+ if len(cfg.HubAddrs) > 0 && !authenticated {if fatalAuthErr != nil {return nil, fmt.Errorf("fatal auth failure: %w", fatalAuthErr)}@@ -314,14 +352,14 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P// Listen for Network Evictions/Revocations from the Hubgo node.listenForHubEvents(ctx)- interval, err := time.ParseDuration(discoveryInterval)+ interval, err := time.ParseDuration(cfg.DiscoveryInterval)if err != nil {- logger.Warnf("[Discovery] Invalid discovery interval '%s', using default %s: %v", discoveryInterval, DefaultDiscoveryInterval, err)+ logger.Warnf("[Discovery] Invalid discovery interval '%s', using default %s: %v", cfg.DiscoveryInterval, DefaultDiscoveryInterval, err)interval, _ = time.ParseDuration(DefaultDiscoveryInterval)}// Start DHT Discovery- go node.startDiscovery(ctx, meshID, interval)+ go node.startDiscovery(ctx, cfg.MeshID, interval)// Layer 3: Open the Lobby Door (Auth Protocol is bypassed by Layer 4)node.Host.SetStreamHandler(api.AuthProtocolID, node.HandleAuthHandshake)@@ -330,7 +368,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.Pnode.Host.SetStreamHandler(api.MCPProtocolID, node.WithBiscuitAuth(node.HandleMCPStream))// Start key pruning- node.startKeyPruning(ctx, keyGracePeriod)+ node.startKeyPruning(ctx, cfg.KeyGracePeriod)// Start Ingress HTTP Serverif err := node.StartIngressServer(ctx); err != nil {@@ -338,7 +376,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P}// Start connection monitor- node.startConnectionMonitor(ctx, 2*time.Minute, 1*time.Minute, 3)+ node.startConnectionMonitor(ctx, cfg.MonitorBootstrap, cfg.MonitorInterval, 3)return node, nil}diff --git a/cmd/sam-node/node_test.go b/cmd/sam-node/node_test.goindex e7a3a16d..15aa9011 100644--- a/cmd/sam-node/node_test.go+++ b/cmd/sam-node/node_test.go@@ -98,7 +98,20 @@ func TestConnectionMonitor_CrashesAfterFailures(t *testing.T) {if os.Getenv("BE_CRASHER_MONITOR") == "1" {priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)store, _ := NewStore(t.TempDir())- node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)+ node, err := NewSamNode(context.Background(), SamNodeConfig{+ PrivKey: priv,+ HubAddrs: nil,+ Store: store,+ MeshID: "test",+ DiscoveryInterval: "10s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: nil,+ KeyGracePeriod: 0,+ AllowLoopback: false,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {os.Exit(0) // Ignore NewSamNode errors for this crasher}diff --git a/cmd/sam-node/proxy_test.go b/cmd/sam-node/proxy_test.goindex 0f61af06..659c3055 100644--- a/cmd/sam-node/proxy_test.go+++ b/cmd/sam-node/proxy_test.go@@ -53,7 +53,20 @@ func TestDatapathIntegration(t *testing.T) {// Pre-populate stores with dummy keys to avoid enrollment failure if required// For this test, we assume we can run without full enrollment if we bypass AuthHandler- nodeA, err := NewSamNode(ctx, privA, nil, nil, storeA, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)+ nodeA, err := NewSamNode(ctx, SamNodeConfig{+ PrivKey: privA,+ HubAddrs: nil,+ Store: storeA,+ MeshID: "test-mesh",+ DiscoveryInterval: "1s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: &NodeConfigComplete{},+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatal(err)}@@ -63,7 +76,20 @@ func TestDatapathIntegration(t *testing.T) {}}()- nodeB, err := NewSamNode(ctx, privB, nil, nil, storeB, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)+ nodeB, err := NewSamNode(ctx, SamNodeConfig{+ PrivKey: privB,+ HubAddrs: nil,+ Store: storeB,+ MeshID: "test-mesh",+ DiscoveryInterval: "1s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: &NodeConfigComplete{},+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatal(err)}@@ -212,13 +238,39 @@ func TestStdioDatapathIntegration(t *testing.T) {}defer func() { _ = storeB.Close() }()- nodeA, err := NewSamNode(ctx, privA, nil, nil, storeA, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)+ nodeA, err := NewSamNode(ctx, SamNodeConfig{+ PrivKey: privA,+ HubAddrs: nil,+ Store: storeA,+ MeshID: "test-mesh",+ DiscoveryInterval: "1s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: &NodeConfigComplete{},+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatal(err)}defer func() { _ = nodeA.Host.Close() }()- nodeB, err := NewSamNode(ctx, privB, nil, nil, storeB, "test-mesh", "1s", []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)+ nodeB, err := NewSamNode(ctx, SamNodeConfig{+ PrivKey: privB,+ HubAddrs: nil,+ Store: storeB,+ MeshID: "test-mesh",+ DiscoveryInterval: "1s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: &NodeConfigComplete{},+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatal(err)}diff --git a/cmd/sam-node/sidecar_test.go b/cmd/sam-node/sidecar_test.goindex 382c310f..5c7160a1 100644--- a/cmd/sam-node/sidecar_test.go+++ b/cmd/sam-node/sidecar_test.go@@ -501,8 +501,20 @@ func TestRegisterService_PopulatesAggregatedTools(t *testing.T) {if err != nil {t.Fatal(err)}- node, err := NewSamNode(ctx, priv, nil, nil, store, "test-mesh", "1s",- []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)+ node, err := NewSamNode(ctx, SamNodeConfig{+ PrivKey: priv,+ HubAddrs: nil,+ Store: store,+ MeshID: "test-mesh",+ DiscoveryInterval: "1s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: &NodeConfigComplete{},+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatal(err)}@@ -553,8 +565,20 @@ func TestUnregisterService_DetachesAggregation(t *testing.T) {if err != nil {t.Fatal(err)}- node, err := NewSamNode(ctx, priv, nil, nil, store, "test-mesh", "1s",- []string{"/ip4/127.0.0.1/tcp/0"}, false, &NodeConfigComplete{}, 24*time.Hour, true)+ node, err := NewSamNode(ctx, SamNodeConfig{+ PrivKey: priv,+ HubAddrs: nil,+ Store: store,+ MeshID: "test-mesh",+ DiscoveryInterval: "1s",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: &NodeConfigComplete{},+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatal(err)}diff --git a/cmd/sam-node/static_service_test.go b/cmd/sam-node/static_service_test.goindex 32b4aeae..28ad55b3 100644--- a/cmd/sam-node/static_service_test.go+++ b/cmd/sam-node/static_service_test.go@@ -14,7 +14,9 @@ import ("github.com/libp2p/go-libp2p"dht "github.com/libp2p/go-libp2p-kad-dht""github.com/libp2p/go-libp2p/core/crypto"+ "github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-msgio""github.com/modelcontextprotocol/go-sdk/mcp""github.com/multiformats/go-multiaddr""google.golang.org/protobuf/proto"@@ -32,6 +34,16 @@ func startMockLibp2pHub(t *testing.T) (peer.ID, string) {t.Fatalf("failed to create DHT on mock hub: %v", err)}+ // Add dummy auth handler+ h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {+ defer func() { _ = s.Close() }()+ // Just send a success response to make the client happy+ resp := &api.AuthResponse{Success: true}+ data, _ := proto.Marshal(resp)+ writer := msgio.NewVarintWriter(s)+ _ = writer.WriteMsg(data)+ })+// Start HTTP server for enrollmentmux := http.NewServeMux()mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {@@ -94,7 +106,20 @@ func TestStaticServiceRegistrationRequiresConnection(t *testing.T) {}// 4. Create Node with empty hub addrs initially- node, err := NewSamNode(context.Background(), priv, nil, []multiaddr.Multiaddr{}, store, "test-mesh", "100ms", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 24*time.Hour, true)+ node, err := NewSamNode(context.Background(), SamNodeConfig{+ PrivKey: priv,+ HubAddrs: []multiaddr.Multiaddr{},+ Store: store,+ MeshID: "test-mesh",+ DiscoveryInterval: "100ms",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: nil,+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatalf("failed to create node: %v", err)}@@ -167,7 +192,20 @@ func TestStaticServiceRegistrationCommandFailure(t *testing.T) {}// 4. Create Node with empty hub addrs initially- node, err := NewSamNode(context.Background(), priv, nil, []multiaddr.Multiaddr{}, store, "test-mesh", "100ms", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 24*time.Hour, true)+ node, err := NewSamNode(context.Background(), SamNodeConfig{+ PrivKey: priv,+ HubAddrs: []multiaddr.Multiaddr{},+ Store: store,+ MeshID: "test-mesh",+ DiscoveryInterval: "100ms",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ EnableRelay: false,+ NodeConfig: nil,+ KeyGracePeriod: 24 * time.Hour,+ AllowLoopback: true,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ })if err != nil {t.Fatalf("failed to create node: %v", err)}diff --git a/tests/integration/failover_test.go b/tests/integration/failover_test.gonew file mode 100644index 00000000..1a7128ef--- /dev/null+++ b/tests/integration/failover_test.go@@ -0,0 +1,233 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package integration_test++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "net/http/httputil"+ "net/url"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/libp2p/go-libp2p"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/multiformats/go-multiaddr"+)++func TestFailoverUpdatesRelay(t *testing.T) {+ hubBin := buildBinary(t, "./cmd/sam-hub")+ nodeBin := buildBinary(t, "./cmd/sam-node")++ tmpDir := t.TempDir()++ // Create a mock policy file+ policyFile := filepath.Join(tmpDir, "policies.yaml")+ policyContent := `version: "v1alpha1"+bindings: []+roles: {}+`+ if err := os.WriteFile(policyFile, []byte(policyContent), 0644); err != nil {+ t.Fatal(err)+ }++ httpPortA := getFreePort(t)+ p2pPortA := getFreePort(t)+ httpPortB := getFreePort(t)+ p2pPortB := getFreePort(t)++ // Mock OIDC+ oidcURL, jwtTokenStr := startMockOIDC(t)+ jwtPath := filepath.Join(tmpDir, "jwt.txt")+ if err := os.WriteFile(jwtPath, []byte(jwtTokenStr), 0644); err != nil {+ t.Fatal(err)+ }++ // Dynamic Load Balancer for HTTP discovery+ var lbMu sync.Mutex+ activeHubPort := httpPortA++ lb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ lbMu.Lock()+ targetPort := activeHubPort+ lbMu.Unlock()++ target, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", targetPort))+ proxy := httputil.NewSingleHostReverseProxy(target)+ proxy.ServeHTTP(w, r)+ }))+ defer lb.Close()++ // Start Hub A+ cmdA := exec.Command(hubBin,+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", httpPortA),+ "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p2pPortA),+ "--policy-file", policyFile,+ "--keys-db", filepath.Join(tmpDir, "keysA.db"),+ "--allow-loopback",+ "--issuer", oidcURL,+ "--insecure-skip-tls-verify",+ )+ var stdoutA, stderrA safeBuffer+ cmdA.Stdout = &stdoutA+ cmdA.Stderr = &stderrA+ if err := cmdA.Start(); err != nil {+ t.Fatalf("failed to start Hub A: %v", err)+ }++ // Fetch Hub A's Peer ID+ peerIDA := fetchPeerID(t, httpPortA)+ t.Logf("Hub A PeerID: %s", peerIDA) // Copy keysA.db to keysB.db to avoid bbolt file lock conflicts+ keysA := filepath.Join(tmpDir, "keysA.db")+ keysB := filepath.Join(tmpDir, "keysB.db")+ data, err := os.ReadFile(keysA)+ if err != nil {+ t.Fatalf("Failed to read keysA.db: %v", err)+ }+ if err := os.WriteFile(keysB, data, 0600); err != nil {+ t.Fatalf("Failed to write keysB.db: %v", err)+ }++ // Start Hub B+ cmdB := exec.Command(hubBin,+ "--bind-address", fmt.Sprintf("127.0.0.1:%d", httpPortB),+ "--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", p2pPortB),+ "--policy-file", policyFile,+ "--keys-db", keysB,+ "--allow-loopback",+ "--issuer", oidcURL,+ "--insecure-skip-tls-verify",+ )+ var stdoutB, stderrB safeBuffer+ cmdB.Stdout = &stdoutB+ cmdB.Stderr = &stderrB+ if err := cmdB.Start(); err != nil {+ t.Fatalf("failed to start Hub B: %v", err)+ }+ defer func() { _ = cmdB.Process.Kill() }()++ peerIDB := fetchPeerID(t, httpPortB)+ t.Logf("Hub B PeerID: %s", peerIDB)++ // Start Node B+ nodeHome := filepath.Join(tmpDir, "node-home")+ env := append(os.Environ(),+ "HOME="+nodeHome,+ "XDG_CONFIG_HOME="+filepath.Join(nodeHome, ".config"),+ )+ cmdNode := exec.Command(nodeBin, "run",+ "--hub", lb.URL,+ "--listen", "/ip4/127.0.0.1/tcp/0",+ "--jwt-path", jwtPath,+ "--bind-addr", "127.0.0.1:0",+ "--api-token", "dummy-token",+ "--allow-loopback",+ "--monitor-bootstrap", "1s",+ "--monitor-interval", "1s",+ )+ cmdNode.Dir = repoRoot(t)+ cmdNode.Env = env+ var stdoutNode, stderrNode safeBuffer+ cmdNode.Stdout = &stdoutNode+ cmdNode.Stderr = &stderrNode++ if err := cmdNode.Start(); err != nil {+ t.Fatal(err)+ }+ defer func() { _ = cmdNode.Process.Kill() }()++ // Wait for Node B to get a reservation on Hub A+ var nodePeerID string+ var out string+ for i := 0; i < 100; i++ {+ out = stdoutNode.String() + stderrNode.String()+ if strings.Contains(out, "PeerID:") {+ idx := strings.Index(out, "PeerID:")+ parts := strings.Split(strings.TrimSpace(out[idx+len("PeerID:"):]), "\n")+ if len(parts) > 0 {+ nodePeerID = strings.TrimSpace(parts[0])+ }+ }+ if strings.Contains(out, "Yielding static relays to AutoRelay") {+ break+ }+ time.Sleep(100 * time.Millisecond)+ }+ if nodePeerID == "" {+ t.Fatalf("Node B failed to get PeerID in time.\nOutput:\n%s", out)+ }+ t.Logf("Node B started. Node PeerID: %s", nodePeerID)++ // Now FAILOVER: Switch LB to Hub B and KILL Hub A+ t.Logf("Initiating failover to Hub B...")+ lbMu.Lock()+ activeHubPort = httpPortB+ lbMu.Unlock()+ _ = cmdA.Process.Kill()++ // Wait for Node B's AutoRelay to get updated+ for i := 0; i < 150; i++ {+ out = stdoutNode.String() + stderrNode.String()+ if strings.Contains(out, "Successfully reconnected to Hub via HTTP fallback.") {+ break+ }+ time.Sleep(100 * time.Millisecond)+ }+ if !strings.Contains(out, "Successfully reconnected to Hub via HTTP fallback.") {+ t.Fatalf("Node B failed to detect failover and reconnect.\nOutput:\n%s", out)+ }+ t.Log("Node B successfully reconnected to Hub B!")++ // Final verification: Ensure we can actually reach Node B via the Hub B relay+ clientHost, err := libp2p.New(libp2p.NoListenAddrs)+ if err != nil {+ t.Fatal(err)+ }+ defer clientHost.Close() //nolint:errcheck++ relayAddrStr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s/p2p-circuit/p2p/%s", p2pPortB, peerIDB, nodePeerID)+ relayAddr, err := multiaddr.NewMultiaddr(relayAddrStr)+ if err != nil {+ t.Fatal(err)+ }+ addrInfo, err := peer.AddrInfoFromP2pAddr(relayAddr)+ if err != nil {+ t.Fatal(err)+ }++ ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)+ defer cancel()++ var connectErr error+ for i := 0; i < 15; i++ {+ connectErr = clientHost.Connect(ctx, *addrInfo)+ if connectErr == nil {+ break+ }+ time.Sleep(1 * time.Second)+ }++ if connectErr != nil {+ t.Fatalf("Failed to connect to Node B via Hub B relay: %v\nOutput: %s", connectErr, stdoutNode.String()+stderrNode.String())+ }+ t.Log("Successfully connected to Node B via Hub B relay circuit!")+}