Skip to content

Agent Mesh

fix node

Antonio Ojea582d4732aabd
diff --git a/cmd/sam-node/connection_monitor.go b/cmd/sam-node/connection_monitor.go
index 0f940651..9f81ee81 100644
--- a/cmd/sam-node/connection_monitor.go
+++ b/cmd/sam-node/connection_monitor.go
@@ -23,11 +23,11 @@ type hubConnectionManager interface {
// - reconnected: true if the connection was lost but successfully recovered during this check.
//
// The recovery process follows these steps:
-// 1. Connection Check: If already connected, return (stable=true, reconnected=false).
-// 2. P2P Retry: If disconnected, attempt to reconnect using the known P2P multiaddresses stored locally.
-// 3. HTTP Fallback: If P2P retries fail, fall back to HTTP discovery using the stored Hub URL to fetch
-// the new Hub addresses and Peer IDs, update the local config, and attempt to reconnect.
-// 4. Total Failure: If all reconnection attempts fail, return (stable=false, reconnected=false).
+// 1. Connection Check: If already connected, return (stable=true, reconnected=false).
+// 2. P2P Retry: If disconnected, attempt to reconnect using the known P2P multiaddresses stored locally.
+// 3. HTTP Fallback: If P2P retries fail, fall back to HTTP discovery using the stored Hub URL to fetch
+// the new Hub addresses and Peer IDs, update the local config, and attempt to reconnect.
+// 4. Total Failure: If all reconnection attempts fail, return (stable=false, reconnected=false).
func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable bool, reconnected bool) {
if mgr.IsConnected() {
return true, false
@@ -52,27 +52,35 @@ func checkHubConnection(ctx context.Context, mgr hubConnectionManager) (stable b
}
hubURL, err := mgr.LoadHubURL()
- if err == nil && hubURL != "" {
- logger.Infof("[Monitor] Reconnect P2P failed. Discovering hub info from %s...", hubURL)
- info, err := mgr.DiscoverHubInfo(ctx, hubURL)
- if err == nil && len(info.HubAddresses) > 0 {
- var newHubAddrs []multiaddr.Multiaddr
- for _, addrStr := range info.HubAddresses {
- if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {
- newHubAddrs = append(newHubAddrs, ma)
- }
- }
- if len(newHubAddrs) > 0 {
- if pubKeyBytes, _, err := mgr.LoadHubConfig(); err == nil {
- _ = mgr.SaveHubConfig(pubKeyBytes, info.HubAddresses)
- }
- for _, addr := range newHubAddrs {
- if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {
- logger.Infof("[Monitor] Successfully reconnected to Hub via HTTP fallback.")
- return false, true
- }
- }
- }
+ if err != nil || hubURL == "" {
+ return false, false
+ }
+
+ logger.Infof("[Monitor] Reconnect P2P failed. Discovering hub info from %s...", hubURL)
+ info, err := mgr.DiscoverHubInfo(ctx, hubURL)
+ if err != nil || len(info.HubAddresses) == 0 {
+ return false, false
+ }
+
+ var newHubAddrs []multiaddr.Multiaddr
+ for _, addrStr := range info.HubAddresses {
+ if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {
+ newHubAddrs = append(newHubAddrs, ma)
+ }
+ }
+
+ if len(newHubAddrs) == 0 {
+ return false, false
+ }
+
+ if pubKeyBytes, _, err := mgr.LoadHubConfig(); err == nil {
+ _ = mgr.SaveHubConfig(pubKeyBytes, info.HubAddresses)
+ }
+
+ for _, addr := range newHubAddrs {
+ if err := mgr.ConnectAndAuthWithHub(ctx, addr); err == nil {
+ logger.Infof("[Monitor] Successfully reconnected to Hub via HTTP fallback.")
+ return false, true
}
}
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 42d67636..ba7d5e7a 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -439,7 +439,10 @@ func (n *SamNode) startConnectionMonitor(ctx context.Context, bootstrapDuration,
}
consecutiveFailures++
- logger.Errorf("[Monitor] Failed to reconnect to the hub. Consecutive failures: %d", consecutiveFailures)
+ logger.Errorf("[Monitor] Failed to reconnect to the hub. Consecutive failures: %d/%d", consecutiveFailures, maxFailures)
+ if consecutiveFailures >= maxFailures {
+ logger.Fatalf("[Monitor] Failed to reconnect to the hub for %d consecutive checks. Exiting to avoid network partition.", maxFailures)
+ }
}
}
}()
diff --git a/cmd/sam-node/node_test.go b/cmd/sam-node/node_test.go
index cbac0c6a..e7a3a16d 100644
--- a/cmd/sam-node/node_test.go
+++ b/cmd/sam-node/node_test.go
@@ -94,20 +94,27 @@ func TestStartRenewalLoop_ExpiredAndFails(t *testing.T) {
t.Fatalf("process ran with err %v, want exit status 1 (fatal crash)", err)
}
-func TestConnectionMonitor_RetriesWithoutCrashing(t *testing.T) {
- priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)
- store, _ := NewStore(t.TempDir())
- node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)
- if err != nil {
- t.Fatalf("Failed to create node: %v", err)
- }
-
- // Use very short durations
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
+func TestConnectionMonitor_CrashesAfterFailures(t *testing.T) {
+ if os.Getenv("BE_CRASHER_MONITOR") == "1" {
+ priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ store, _ := NewStore(t.TempDir())
+ node, err := NewSamNode(context.Background(), priv, nil, nil, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)
+ if err != nil {
+ os.Exit(0) // Ignore NewSamNode errors for this crasher
+ }
- node.startConnectionMonitor(ctx, 10*time.Millisecond, 10*time.Millisecond, 3)
- time.Sleep(100 * time.Millisecond) // Wait long enough for multiple failures
+ // Use very short durations
+ node.startConnectionMonitor(context.Background(), 10*time.Millisecond, 10*time.Millisecond, 3)
+ time.Sleep(1 * time.Second)
+ os.Exit(0) // should not be reached
+ return
+ }
- // If it didn't crash, the test passes
+ cmd := exec.Command(os.Args[0], "-test.run=TestConnectionMonitor_CrashesAfterFailures")
+ cmd.Env = append(os.Environ(), "BE_CRASHER_MONITOR=1")
+ err := cmd.Run()
+ if e, ok := err.(*exec.ExitError); ok && !e.Success() {
+ return // Successful fatal exit
+ }
+ t.Fatalf("process ran with err %v, want exit status 1 (fatal crash)", err)
}