Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
Fix node logic
Antonio Ojeabe0b507f94da
diff --git a/cmd/sam-node/connection_monitor.go b/cmd/sam-node/connection_monitor.gonew file mode 100644index 00000000..0f940651--- /dev/null+++ b/cmd/sam-node/connection_monitor.go@@ -0,0 +1,80 @@+package main++import (+ "context"++ "github.com/google/sam/api"+ "github.com/multiformats/go-multiaddr"+)++type hubConnectionManager interface {+ IsConnected() bool+ HubPeerIDString() string+ LoadHubConfig() ([]byte, []string, error)+ ConnectAndAuthWithHub(ctx context.Context, addr multiaddr.Multiaddr) error+ LoadHubURL() (string, error)+ DiscoverHubInfo(ctx context.Context, url string) (*api.HubInfoResponse, error)+ SaveHubConfig(pubKey []byte, addrs []string) error+}++// checkHubConnection monitors the connection to the hub and attempts to recover it if disconnected.+// It returns two booleans indicating the current state of the connection:+// - stable: true if the connection was already established and is healthy.+// - reconnected: true if the connection was lost but successfully recovered during this check.+//+// The recovery process follows these steps:+// 1. Connection Check: If already connected, return (stable=true, reconnected=false).+// 2. P2P Retry: If disconnected, attempt to reconnect using the known P2P multiaddresses stored locally.+// 3. HTTP Fallback: If P2P retries fail, fall back to HTTP discovery using the stored Hub URL to fetch+// the new Hub addresses and Peer IDs, update the local config, and attempt to reconnect.+// 4. Total Failure: If all reconnection attempts fail, return (stable=false, reconnected=false).+func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable bool, reconnected bool) {+ if mgr.IsConnected() {+ return true, false+ }++ logger.Warnf("[Monitor] Disconnected from Hub %s. Attempting to reconnect...", mgr.HubPeerIDString())++ var p2pAddrs []multiaddr.Multiaddr+ if _, storedAddrs, err := mgr.LoadHubConfig(); err == nil {+ for _, addrStr := range storedAddrs {+ if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {+ p2pAddrs = append(p2pAddrs, ma)+ }+ }+ }++ for _, addr := range p2pAddrs {+ if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {+ logger.Infof("[Monitor] Successfully reconnected to Hub via P2P.")+ return false, true+ }+ }++ hubURL, err := mgr.LoadHubURL()+ if err == nil && hubURL != "" {+ logger.Infof("[Monitor] Reconnect P2P failed. Discovering hub info from %s...", hubURL)+ info, err := mgr.DiscoverHubInfo(ctx, hubURL)+ if err == nil && len(info.HubAddresses) > 0 {+ var newHubAddrs []multiaddr.Multiaddr+ for _, addrStr := range info.HubAddresses {+ if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {+ newHubAddrs = append(newHubAddrs, ma)+ }+ }+ if len(newHubAddrs) > 0 {+ if pubKeyBytes, _, err := mgr.LoadHubConfig(); err == nil {+ _ = mgr.SaveHubConfig(pubKeyBytes, info.HubAddresses)+ }+ for _, addr := range newHubAddrs {+ if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {+ logger.Infof("[Monitor] Successfully reconnected to Hub via HTTP fallback.")+ return false, true+ }+ }+ }+ }+ }++ return false, false+}diff --git a/cmd/sam-node/connection_monitor_test.go b/cmd/sam-node/connection_monitor_test.gonew file mode 100644index 00000000..45232270--- /dev/null+++ b/cmd/sam-node/connection_monitor_test.go@@ -0,0 +1,140 @@+package main++import (+ "context"+ "errors"+ "testing"++ "github.com/google/sam/api"+ "github.com/multiformats/go-multiaddr"+)++type mockHubConnectionManager struct {+ connected bool+ peerID string+ hubConfigErr error+ storedAddrs []string+ hubURLErr error+ hubURL string+ discoverErr error+ discoverResp *api.HubInfoResponse+ connectP2PErr error+ connectHTTPReq bool+ connectHTTPErr error+ saveConfigErr error+}++func (m *mockHubConnectionManager) IsConnected() bool {+ return m.connected+}++func (m *mockHubConnectionManager) HubPeerIDString() string {+ return m.peerID+}++func (m *mockHubConnectionManager) LoadHubConfig() ([]byte, []string, error) {+ return nil, m.storedAddrs, m.hubConfigErr+}++func (m *mockHubConnectionManager) ConnectAndAuthWithHub(ctx context.Context, addr multiaddr.Multiaddr) error {+ if m.connectHTTPReq {+ return m.connectHTTPErr+ }+ return m.connectP2PErr+}++func (m *mockHubConnectionManager) LoadHubURL() (string, error) {+ return m.hubURL, m.hubURLErr+}++func (m *mockHubConnectionManager) DiscoverHubInfo(ctx context.Context, url string) (*api.HubInfoResponse, error) {+ return m.discoverResp, m.discoverErr+}++func (m *mockHubConnectionManager) SaveHubConfig(pubKey []byte, addrs []string) error {+ return m.saveConfigErr+}++func TestCheckHubConnection(t *testing.T) {+ cases := []struct {+ name string+ mgr *mockHubConnectionManager+ wantStable bool+ wantReconn bool+ }{+ {+ name: "already connected",+ mgr: &mockHubConnectionManager{+ connected: true,+ },+ wantStable: true,+ wantReconn: false,+ },+ {+ name: "disconnected, reconnects via P2P",+ mgr: &mockHubConnectionManager{+ connected: false,+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},+ connectP2PErr: nil,+ },+ wantStable: false,+ wantReconn: true,+ },+ {+ name: "disconnected, P2P fails, reconnects via HTTP",+ mgr: &mockHubConnectionManager{+ connected: false,+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},+ connectP2PErr: errors.New("p2p failed"),+ hubURL: "https://hub.example.com",+ discoverResp: &api.HubInfoResponse{+ HubAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},+ },+ connectHTTPReq: true,+ connectHTTPErr: nil,+ },+ wantStable: false,+ wantReconn: true,+ },+ {+ name: "disconnected, all fail",+ mgr: &mockHubConnectionManager{+ connected: false,+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},+ connectP2PErr: errors.New("p2p failed"),+ hubURL: "https://hub.example.com",+ discoverResp: &api.HubInfoResponse{+ HubAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},+ },+ connectHTTPReq: true,+ connectHTTPErr: errors.New("http fallback connect failed"),+ },+ wantStable: false,+ wantReconn: false,+ },+ {+ name: "disconnected, HTTP discovery fails",+ mgr: &mockHubConnectionManager{+ connected: false,+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},+ connectP2PErr: errors.New("p2p failed"),+ hubURL: "https://hub.example.com",+ discoverErr: errors.New("discovery failed"),+ },+ wantStable: false,+ wantReconn: false,+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ stable, reconnected := checkHubConnection(context.Background(), tc.mgr)+ if stable != tc.wantStable {+ t.Errorf("expected stable %v, got %v", tc.wantStable, stable)+ }+ if reconnected != tc.wantReconn {+ t.Errorf("expected reconnected %v, got %v", tc.wantReconn, reconnected)+ }+ })+ }+}diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.goindex 86366940..42d67636 100644--- a/cmd/sam-node/node.go+++ b/cmd/sam-node/node.go@@ -385,6 +385,26 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.Preturn node, nil}+func (n *SamNode) IsConnected() bool {+ return n.HubPeerID != "" && n.Host.Network().Connectedness(n.HubPeerID) == network.Connected+}++func (n *SamNode) HubPeerIDString() string {+ return n.HubPeerID.String()+}++func (n *SamNode) LoadHubConfig() ([]byte, []string, error) {+ return n.Store.LoadHubConfig()+}++func (n *SamNode) LoadHubURL() (string, error) {+ return n.Store.LoadHubURL()+}++func (n *SamNode) SaveHubConfig(pubKey []byte, addrs []string) error {+ return n.Store.SaveHubConfig(pubKey, addrs)+}+func (n *SamNode) startConnectionMonitor(ctx context.Context, bootstrapDuration, checkInterval time.Duration, maxFailures int) {go func() {// Wait for initial bootstrap to complete@@ -403,18 +423,23 @@ func (n *SamNode) startConnectionMonitor(ctx context.Context, bootstrapDuration,case <-ctx.Done():returncase <-ticker.C:- if len(n.Host.Network().Peers()) == 0 {- consecutiveFailures++- logger.Warnf("Not connected to the mesh (0 peers). Consecutive failures: %d/%d", consecutiveFailures, maxFailures)- if consecutiveFailures >= maxFailures {- logger.Fatalf("Not connected to the mesh (0 peers) for %d consecutive checks. Exiting to avoid network partition.", maxFailures)- }- } else {+ stable, reconnected := checkHubConnection(ctx, n)++ if stable {if consecutiveFailures > 0 {- logger.Infof("Reconnected to the mesh. Resetting failure count.")+ logger.Infof("[Monitor] Connection to hub is stable. Resetting failure count.")+ consecutiveFailures = 0}+ continue+ }++ if reconnected {consecutiveFailures = 0+ continue}++ consecutiveFailures+++ logger.Errorf("[Monitor] Failed to reconnect to the hub. Consecutive failures: %d", consecutiveFailures)}}}()diff --git a/cmd/sam-node/node_test.go b/cmd/sam-node/node_test.goindex e7a3a16d..cbac0c6a 100644--- a/cmd/sam-node/node_test.go+++ b/cmd/sam-node/node_test.go@@ -94,27 +94,20 @@ func TestStartRenewalLoop_ExpiredAndFails(t *testing.T) {t.Fatalf("process ran with err %v, want exit status 1 (fatal crash)", err)}-func TestConnectionMonitor_CrashesAfterFailures(t *testing.T) {- if os.Getenv("BE_CRASHER_MONITOR") == "1" {- priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)- store, _ := NewStore(t.TempDir())- node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)- if err != nil {- os.Exit(0) // Ignore NewSamNode errors for this crasher- }-- // Use very short durations- node.startConnectionMonitor(context.Background(), 10*time.Millisecond, 10*time.Millisecond, 3)- time.Sleep(1 * time.Second)- os.Exit(0) // should not be reached- return+func TestConnectionMonitor_RetriesWithoutCrashing(t *testing.T) {+ priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ store, _ := NewStore(t.TempDir())+ node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)+ if err != nil {+ t.Fatalf("Failed to create node: %v", err)}- cmd := exec.Command(os.Args[0], "-test.run=TestConnectionMonitor_CrashesAfterFailures")- cmd.Env = append(os.Environ(), "BE_CRASHER_MONITOR=1")- err := cmd.Run()- if e, ok := err.(*exec.ExitError); ok && !e.Success() {- return // Successful fatal exit- }- t.Fatalf("process ran with err %v, want exit status 1 (fatal crash)", err)+ // Use very short durations+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ node.startConnectionMonitor(ctx, 10*time.Millisecond, 10*time.Millisecond, 3)+ time.Sleep(100 * time.Millisecond) // Wait long enough for multiple failures++ // If it didn't crash, the test passes}