Skip to content

Agent Mesh

fail hard on renewal failures

Antonio Ojea89c8cfadc25d
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 4268b8b3..7b642e09 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -311,6 +311,7 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
logger.Warnf("[AuthN] Fallback auth failed with %s: %v", addr, err)
} else {
logger.Infof("[AuthN] Fallback connection successful!")
+ authenticated = true
break
}
}
@@ -319,6 +320,10 @@ func NewSamNode(ctx context.Context, privKey crypto.PrivKey, hubPubKey ed25519.P
logger.Warnf("[AuthN] Failed to fetch updated addresses via HTTP: %v", err)
}
}
+
+ if !authenticated {
+ return nil, fmt.Errorf("failed to authenticate with any hub: all connection attempts failed")
+ }
}
// Initialize Gossipsub for Hub Events
@@ -503,7 +508,7 @@ func (n *SamNode) StartRenewalLoop(ctx context.Context, issuerURL, clientID, cli
} else if duration > 0 {
renewAfter = duration / 2
} else {
- renewAfter = 1 * time.Minute
+ renewAfter = 1 * time.Second
}
}
@@ -517,33 +522,45 @@ func (n *SamNode) StartRenewalLoop(ctx context.Context, issuerURL, clientID, cli
case <-timer.C:
fmt.Println("Renewing enrollment...")
var newJWT string
+ var fetchErr error
+
if issuerURL != "" {
tokenURL, err := n.DiscoverTokenURL(ctx, issuerURL)
if err != nil {
- fmt.Printf("Failed to discover OIDC endpoints for renewal: %v\n", err)
- continue
- }
- newJWT, err = n.FetchJWT(ctx, tokenURL, clientID, clientSecret)
- if err != nil {
- fmt.Printf("Failed to fetch JWT for renewal: %v\n", err)
- continue
+ fetchErr = fmt.Errorf("failed to discover OIDC endpoints for renewal: %w", err)
+ } else {
+ newJWT, fetchErr = n.FetchJWT(ctx, tokenURL, clientID, clientSecret)
+ if fetchErr != nil {
+ fetchErr = fmt.Errorf("failed to fetch JWT for renewal: %w", fetchErr)
+ }
}
} else if jwtPath != "" {
data, err := os.ReadFile(jwtPath)
if err != nil {
- fmt.Printf("Failed to read JWT file for renewal: %v\n", err)
- continue
+ fetchErr = fmt.Errorf("failed to read JWT file for renewal: %w", err)
+ } else {
+ newJWT = strings.TrimSpace(string(data))
}
- newJWT = strings.TrimSpace(string(data))
} else {
- fmt.Println("No credentials available for renewal.")
- continue
+ fetchErr = fmt.Errorf("no credentials available for renewal")
}
- if err := n.Enroll(ctx, newJWT); err != nil {
- fmt.Printf("Renewal enrollment failed: %v\n", err)
+ if fetchErr == nil {
+ fetchErr = n.Enroll(ctx, newJWT)
+ }
+
+ if fetchErr != nil {
+ logger.Errorf("Renewal failed: %v", fetchErr)
+
+ // Check if we are already expired and if so, die to avoid a split brain
+ exp, loadErr := n.Store.LoadIdentityExpiration()
+ if loadErr == nil && exp > 0 {
+ if time.Now().After(time.Unix(exp, 0)) {
+ logger.Fatalf("Identity expired and renewal failed. Exiting to avoid network partition.")
+ }
+ }
} else {
- fmt.Println("Enrollment renewed successfully.")
+ logger.Infof("Enrollment renewed successfully.")
}
}
}
@@ -757,7 +774,7 @@ func (n *SamNode) startDiscovery(ctx context.Context, meshID string, interval ti
if n.Host.Network().Connectedness(p.ID) != network.Connected {
logger.Infof("[Discovery] Found peer not connected via DHT: %s", p.ID)
-
+
// Log the addresses returned by DHT to confirm they include p2p-circuit paths
for _, addr := range p.Addrs {
logger.Infof("[Discovery] Peer %s advertised address: %s", p.ID, addr)
@@ -939,11 +956,11 @@ func (n *SamNode) findProvidersByCID(ctx context.Context, c cid.Cid) ([]peer.Add
for _, addr := range p.Addrs {
logger.Infof("[Discovery] Provider %s advertised address: %s", p.ID, addr)
}
-
+
if len(p.Addrs) > 0 {
n.Host.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.TempAddrTTL)
}
-
+
providers = append(providers, p)
}
return providers, nil
diff --git a/cmd/sam-node/node_test.go b/cmd/sam-node/node_test.go
index 5f0c0cd7..cd715063 100644
--- a/cmd/sam-node/node_test.go
+++ b/cmd/sam-node/node_test.go
@@ -15,16 +15,20 @@
package main
import (
+ "context"
"crypto/ed25519"
+ "os"
+ "os/exec"
+ "strings"
"testing"
"time"
"github.com/google/sam/api"
lru "github.com/hashicorp/golang-lru/v2"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/multiformats/go-multiaddr"
)
-
-
func TestHandleBannedEvent(t *testing.T) {
revokedCache, _ := lru.New[string, int64](10)
node := &SamNode{
@@ -64,3 +68,44 @@ func TestHandleKeyRotationEvent(t *testing.T) {
t.Errorf("Expected 1 trusted key, got %d", len(node.trustedKeys))
}
}
+
+func TestNewSamNode_FailsAuth(t *testing.T) {
+ priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ hubAddrs := []multiaddr.Multiaddr{multiaddr.StringCast("/ip4/127.0.0.1/tcp/9999")}
+ store, _ := NewStore(t.TempDir()) // We need a valid store
+
+ _, err := NewSamNode(context.Background(), priv, nil, hubAddrs, store, "test", "10s", []string{"/ip4/127.0.0.1/tcp/0"}, false, nil, 0, false)
+ if err == nil {
+ t.Fatal("Expected NewSamNode to fail when it cannot connect to the hub")
+ }
+ if !strings.Contains(err.Error(), "failed to authenticate with any hub") {
+ t.Fatalf("Expected 'failed to authenticate with any hub' error, got %v", err)
+ }
+}
+
+func TestStartRenewalLoop_ExpiredAndFails(t *testing.T) {
+ if os.Getenv("BE_CRASHER") == "1" {
+ store, _ := NewStore(t.TempDir())
+ // Set expiration to the past
+ _ = store.SaveIdentityExpiration(time.Now().Add(-1 * time.Hour).Unix())
+
+ node := &SamNode{
+ Store: store,
+ }
+
+ // Run the renewal loop. Since there's no JWT/Issuer provided, it fails to renew.
+ // It will see that it's expired and it failed to renew, so it will log.Fatalf
+ node.StartRenewalLoop(context.Background(), "", "", "", "")
+ time.Sleep(5 * time.Second)
+ os.Exit(0) // should not be reached
+ return
+ }
+
+ cmd := exec.Command(os.Args[0], "-test.run=TestStartRenewalLoop_ExpiredAndFails")
+ cmd.Env = append(os.Environ(), "BE_CRASHER=1")
+ err := cmd.Run()
+ if e, ok := err.(*exec.ExitError); ok && !e.Success() {
+ return // Successful fatal exit
+ }
+ t.Fatalf("process ran with err %v, want exit status 1 (fatal crash)", err)
+}