Skip to content

Agent Mesh

Fix node logic

Antonio Ojeabe0b507f94da
diff --git a/cmd/sam-node/connection_monitor.go b/cmd/sam-node/connection_monitor.go
new file mode 100644
index 00000000..0f940651
--- /dev/null
+++ b/cmd/sam-node/connection_monitor.go
@@ -0,0 +1,80 @@
+package main
+
+import (
+ "context"
+
+ "github.com/google/sam/api"
+ "github.com/multiformats/go-multiaddr"
+)
+
+type hubConnectionManager interface {
+ IsConnected() bool
+ HubPeerIDString() string
+ LoadHubConfig() ([]byte, []string, error)
+ ConnectAndAuthWithHub(ctx context.Context, addr multiaddr.Multiaddr) error
+ LoadHubURL() (string, error)
+ DiscoverHubInfo(ctx context.Context, url string) (*api.HubInfoResponse, error)
+ SaveHubConfig(pubKey []byte, addrs []string) error
+}
+
+// checkHubConnection monitors the connection to the hub and attempts to recover it if disconnected.
+// It returns two booleans indicating the current state of the connection:
+// - stable: true if the connection was already established and is healthy.
+// - reconnected: true if the connection was lost but successfully recovered during this check.
+//
+// The recovery process follows these steps:
+// 1. Connection Check: If already connected, return (stable=true, reconnected=false).
+// 2. P2P Retry: If disconnected, attempt to reconnect using the known P2P multiaddresses stored locally.
+// 3. HTTP Fallback: If P2P retries fail, fall back to HTTP discovery using the stored Hub URL to fetch
+// the new Hub addresses and Peer IDs, update the local config, and attempt to reconnect.
+// 4. Total Failure: If all reconnection attempts fail, return (stable=false, reconnected=false).
+func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable bool, reconnected bool) {
+ if mgr.IsConnected() {
+ return true, false
+ }
+
+ logger.Warnf("[Monitor] Disconnected from Hub %s. Attempting to reconnect...", mgr.HubPeerIDString())
+
+ var p2pAddrs []multiaddr.Multiaddr
+ if _, storedAddrs, err := mgr.LoadHubConfig(); err == nil {
+ for _, addrStr := range storedAddrs {
+ if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {
+ p2pAddrs = append(p2pAddrs, ma)
+ }
+ }
+ }
+
+ for _, addr := range p2pAddrs {
+ if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {
+ logger.Infof("[Monitor] Successfully reconnected to Hub via P2P.")
+ return false, true
+ }
+ }
+
+ hubURL, err := mgr.LoadHubURL()
+ if err == nil && hubURL != "" {
+ logger.Infof("[Monitor] Reconnect P2P failed. Discovering hub info from %s...", hubURL)
+ info, err := mgr.DiscoverHubInfo(ctx, hubURL)
+ if err == nil && len(info.HubAddresses) > 0 {
+ var newHubAddrs []multiaddr.Multiaddr
+ for _, addrStr := range info.HubAddresses {
+ if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {
+ newHubAddrs = append(newHubAddrs, ma)
+ }
+ }
+ if len(newHubAddrs) > 0 {
+ if pubKeyBytes, _, err := mgr.LoadHubConfig(); err == nil {
+ _ = mgr.SaveHubConfig(pubKeyBytes, info.HubAddresses)
+ }
+ for _, addr := range newHubAddrs {
+ if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {
+ logger.Infof("[Monitor] Successfully reconnected to Hub via HTTP fallback.")
+ return false, true
+ }
+ }
+ }
+ }
+ }
+
+ return false, false
+}
diff --git a/cmd/sam-node/connection_monitor_test.go b/cmd/sam-node/connection_monitor_test.go
new file mode 100644
index 00000000..45232270
--- /dev/null
+++ b/cmd/sam-node/connection_monitor_test.go
@@ -0,0 +1,140 @@
+package main
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/google/sam/api"
+ "github.com/multiformats/go-multiaddr"
+)
+
+type mockHubConnectionManager struct {
+ connected bool
+ peerID string
+ hubConfigErr error
+ storedAddrs []string
+ hubURLErr error
+ hubURL string
+ discoverErr error
+ discoverResp *api.HubInfoResponse
+ connectP2PErr error
+ connectHTTPReq bool
+ connectHTTPErr error
+ saveConfigErr error
+}
+
+func (m *mockHubConnectionManager) IsConnected() bool {
+ return m.connected
+}
+
+func (m *mockHubConnectionManager) HubPeerIDString() string {
+ return m.peerID
+}
+
+func (m *mockHubConnectionManager) LoadHubConfig() ([]byte, []string, error) {
+ return nil, m.storedAddrs, m.hubConfigErr
+}
+
+func (m *mockHubConnectionManager) ConnectAndAuthWithHub(ctx context.Context, addr multiaddr.Multiaddr) error {
+ if m.connectHTTPReq {
+ return m.connectHTTPErr
+ }
+ return m.connectP2PErr
+}
+
+func (m *mockHubConnectionManager) LoadHubURL() (string, error) {
+ return m.hubURL, m.hubURLErr
+}
+
+func (m *mockHubConnectionManager) DiscoverHubInfo(ctx context.Context, url string) (*api.HubInfoResponse, error) {
+ return m.discoverResp, m.discoverErr
+}
+
+func (m *mockHubConnectionManager) SaveHubConfig(pubKey []byte, addrs []string) error {
+ return m.saveConfigErr
+}
+
+func TestCheckHubConnection(t *testing.T) {
+ cases := []struct {
+ name string
+ mgr *mockHubConnectionManager
+ wantStable bool
+ wantReconn bool
+ }{
+ {
+ name: "already connected",
+ mgr: &mockHubConnectionManager{
+ connected: true,
+ },
+ wantStable: true,
+ wantReconn: false,
+ },
+ {
+ name: "disconnected, reconnects via P2P",
+ mgr: &mockHubConnectionManager{
+ connected: false,
+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
+ connectP2PErr: nil,
+ },
+ wantStable: false,
+ wantReconn: true,
+ },
+ {
+ name: "disconnected, P2P fails, reconnects via HTTP",
+ mgr: &mockHubConnectionManager{
+ connected: false,
+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
+ connectP2PErr: errors.New("p2p failed"),
+ hubURL: "https://hub.example.com",
+ discoverResp: &api.HubInfoResponse{
+ HubAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},
+ },
+ connectHTTPReq: true,
+ connectHTTPErr: nil,
+ },
+ wantStable: false,
+ wantReconn: true,
+ },
+ {
+ name: "disconnected, all fail",
+ mgr: &mockHubConnectionManager{
+ connected: false,
+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
+ connectP2PErr: errors.New("p2p failed"),
+ hubURL: "https://hub.example.com",
+ discoverResp: &api.HubInfoResponse{
+ HubAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},
+ },
+ connectHTTPReq: true,
+ connectHTTPErr: errors.New("http fallback connect failed"),
+ },
+ wantStable: false,
+ wantReconn: false,
+ },
+ {
+ name: "disconnected, HTTP discovery fails",
+ mgr: &mockHubConnectionManager{
+ connected: false,
+ storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
+ connectP2PErr: errors.New("p2p failed"),
+ hubURL: "https://hub.example.com",
+ discoverErr: errors.New("discovery failed"),
+ },
+ wantStable: false,
+ wantReconn: false,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ stable, reconnected := checkHubConnection(context.Background(), tc.mgr)
+ if stable != tc.wantStable {
+ t.Errorf("expected stable %v, got %v", tc.wantStable, stable)
+ }
+ if reconnected != tc.wantReconn {
+ t.Errorf("expected reconnected %v, got %v", tc.wantReconn, reconnected)
+ }
+ })
+ }
+}
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 86366940..42d67636 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -385,6 +385,26 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
return node, nil
}
+func (n *SamNode) IsConnected() bool {
+ return n.HubPeerID != "" && n.Host.Network().Connectedness(n.HubPeerID) == network.Connected
+}
+
+func (n *SamNode) HubPeerIDString() string {
+ return n.HubPeerID.String()
+}
+
+func (n *SamNode) LoadHubConfig() ([]byte, []string, error) {
+ return n.Store.LoadHubConfig()
+}
+
+func (n *SamNode) LoadHubURL() (string, error) {
+ return n.Store.LoadHubURL()
+}
+
+func (n *SamNode) SaveHubConfig(pubKey []byte, addrs []string) error {
+ return n.Store.SaveHubConfig(pubKey, addrs)
+}
+
func (n *SamNode) startConnectionMonitor(ctx context.Context, bootstrapDuration, checkInterval time.Duration, maxFailures int) {
go func() {
// Wait for initial bootstrap to complete
@@ -403,18 +423,23 @@ func (n *SamNode) startConnectionMonitor(ctx context.Context, bootstrapDuration,
case <-ctx.Done():
return
case <-ticker.C:
- if len(n.Host.Network().Peers()) == 0 {
- consecutiveFailures++
- logger.Warnf("Not connected to the mesh (0 peers). Consecutive failures: %d/%d", consecutiveFailures, maxFailures)
- if consecutiveFailures >= maxFailures {
- logger.Fatalf("Not connected to the mesh (0 peers) for %d consecutive checks. Exiting to avoid network partition.", maxFailures)
- }
- } else {
+ stable, reconnected := checkHubConnection(ctx, n)
+
+ if stable {
if consecutiveFailures > 0 {
- logger.Infof("Reconnected to the mesh. Resetting failure count.")
+ logger.Infof("[Monitor] Connection to hub is stable. Resetting failure count.")
+ consecutiveFailures = 0
}
+ continue
+ }
+
+ if reconnected {
consecutiveFailures = 0
+ continue
}
+
+ consecutiveFailures++
+ logger.Errorf("[Monitor] Failed to reconnect to the hub. Consecutive failures: %d", consecutiveFailures)
}
}
}()
diff --git a/cmd/sam-node/node_test.go b/cmd/sam-node/node_test.go
index e7a3a16d..cbac0c6a 100644
--- a/cmd/sam-node/node_test.go
+++ b/cmd/sam-node/node_test.go
@@ -94,27 +94,20 @@ func TestStartRenewalLoop_ExpiredAndFails(t *testing.T) {
t.Fatalf("process ran with err %v, want exit status 1 (fatal crash)", err)
}
-func TestConnectionMonitor_CrashesAfterFailures(t *testing.T) {
- if os.Getenv("BE_CRASHER_MONITOR") == "1" {
- priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)
- store, _ := NewStore(t.TempDir())
- node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)
- if err != nil {
- os.Exit(0) // Ignore NewSamNode errors for this crasher
- }
-
- // Use very short durations
- node.startConnectionMonitor(context.Background(), 10*time.Millisecond, 10*time.Millisecond, 3)
- time.Sleep(1 * time.Second)
- os.Exit(0) // should not be reached
- return
+func TestConnectionMonitor_RetriesWithoutCrashing(t *testing.T) {
+ priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ store, _ := NewStore(t.TempDir())
+ node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)
+ if err != nil {
+ t.Fatalf("Failed to create node: %v", err)
}
- cmd := exec.Command(os.Args[0], "-test.run=TestConnectionMonitor_CrashesAfterFailures")
- cmd.Env = append(os.Environ(), "BE_CRASHER_MONITOR=1")
- err := cmd.Run()
- if e, ok := err.(*exec.ExitError); ok && !e.Success() {
- return // Successful fatal exit
- }
- t.Fatalf("process ran with err %v, want exit status 1 (fatal crash)", err)
+ // Use very short durations
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ node.startConnectionMonitor(ctx, 10*time.Millisecond, 10*time.Millisecond, 3)
+ time.Sleep(100 * time.Millisecond) // Wait long enough for multiple failures
+
+ // If it didn't crash, the test passes
}