Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
test: make the rotation CUJ outlive the retiring key
Antonio Ojeacf8a528d190b
diff --git a/tests/integration/rotation_test.go b/tests/integration/rotation_test.goindex 6681c600..6cb14428 100644--- a/tests/integration/rotation_test.go+++ b/tests/integration/rotation_test.go@@ -17,6 +17,7 @@ package integration_testimport ("bytes""context"+ "crypto/ed25519""fmt""io""net/http"@@ -28,9 +29,15 @@ import ("time""github.com/google/sam/api"+ "github.com/libp2p/go-libp2p/core/peer""google.golang.org/protobuf/proto")+// TestKeyRotationIntegration is the reauthentication CUJ: the control plane+// rotates its signing key, the retiring key leaves its grace period, and+// both the router and the node must by then hold credentials signed by a+// current key. Asserting only while the retiring key is still valid would+// pass with credentials nobody refreshed, which is the failure this pins.func TestKeyRotationIntegration(t *testing.T) {cpBin := buildBinary(t, "./cmd/sam-control-plane")routerBin := buildBinary(t, "./cmd/sam-router")@@ -64,8 +71,8 @@ roles:"--admin-token-path", tokenPath(t, "test-admin-token"),"--db-dsn", filepath.Join(tmpDir, "cp-keys.db"),"--issuer", oidcURL,- "--key-rotation-interval", "4s",- "--key-grace-period", "2s",+ "--key-rotation-interval", "2s",+ "--key-grace-period", "1s","--insecure-skip-tls-verify",)cpCmd.Stdout = os.Stdout@@ -94,8 +101,8 @@ roles:"--keys-path", filepath.Join(tmpDir, "router-keys.db"),"--allow-loopback","--oidc-token", routerJWT,- "--lease-renew-interval", "2s",- "--keys-sync-interval", "1s",+ "--lease-renew-interval", "1s",+ "--keys-sync-interval", "500ms",)routerCmd.Stdout = os.StdoutrouterCmd.Stderr = os.Stderr@@ -126,6 +133,15 @@ roles:nodeLogFile, _ := os.Create(nodeLogPath)defer func() { _ = nodeLogFile.Close() }()+ // The owner socket serves /sam/identity, the node's own credential;+ // t.TempDir is too long for a socket path.+ socketDir, err := os.MkdirTemp("", "sam-rot-")+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = os.RemoveAll(socketDir) }()+ nodeSocket := filepath.Join(socketDir, "node.sock")+nodeCmd := exec.Command(nodeBin, "run","--control-plane", fmt.Sprintf("http://127.0.0.1:%d", cpPort),"--jwt", nodeJWT,@@ -133,7 +149,9 @@ roles:"--listen", "/ip4/127.0.0.1/tcp/0","--allow-loopback","--bind-addr", fmt.Sprintf("127.0.0.1:%d", nodeApiPort),+ "--socket-path", nodeSocket,"--api-token-path", tokenPath(t, "dummy-token"),+ "--control-plane-sync-interval", "500ms","--log-level", "debug",)nodeCmd.Env = nodeEnv@@ -149,18 +167,42 @@ roles:// Wait for Node to be online activelywaitForAPI(t, fmt.Sprintf("127.0.0.1:%d", nodeApiPort))+ socketClient := identityEvidenceSocketClient(nodeSocket)+ waitForIdentityEvidenceSocket(t, socketClient)- // Get initial keys- initialKeys := fetchPublicKeys(t, cpPort)+ // The key that signed the node's enrollment credential is the one whose+ // retirement matters: read the credential the way an owner application+ // would and find its signer among the control plane's live keys.+ var initial api.IdentityEvidenceResponse+ getIdentityEvidenceJSON(t, socketClient, "/sam/identity", &initial)+ nodePeer, err := peer.Decode(initial.PeerId)+ if err != nil {+ t.Fatalf("decode node PeerID from evidence: %v", err)+ }+ signer := signerOf(t, initial.Biscuit, nodePeer, fetchPublicKeys(t, cpPort))- // Wait for key rotation to happen actively- waitForKeyRotation(t, cpPort, initialKeys)+ // Once the signer has left /keys, a credential it signed verifies+ // nowhere; both components must hold re-issued credentials by then.+ retiredAt := waitForKeysRetired(t, cpPort, [][]byte{signer})// 6. Verify Node is still active and can communicate (it should renew successfully and remain online)if nodeCmd.ProcessState != nil && nodeCmd.ProcessState.Exited() {t.Fatalf("node process exited unexpectedly")}+ var current api.IdentityEvidenceResponse+ getIdentityEvidenceJSON(t, socketClient, "/sam/identity", ¤t)+ if _, err := verifyBiscuitForApplication(current.Biscuit, nodePeer, publicKeysFrom(fetchPublicKeys(t, cpPort))); err != nil {+ t.Fatalf("node credential does not verify under the current control plane keys after its signer retired: %v\n--- node.log ---\n%s", err, readLog(t, nodeLogPath))+ }+ if _, err := verifyBiscuitForApplication(current.Biscuit, nodePeer, []ed25519.PublicKey{signer}); err == nil {+ t.Fatal("node credential still verifies under the retired key: it was never refreshed")+ }++ // The router renews its lease with its credential; a renewal accepted+ // after retirement proves the router's credential was re-issued too.+ waitForLeaseRenewedAfter(t, cpPort, retiredAt)+// Double check by looking at the node's API or verifying it doesn't crashclientBin := buildBinary(t, "./cmd/mcp-client")stdout, stderr, err := runCommand(t, repoRoot(t), 5*time.Second, nil, "",@@ -233,3 +275,77 @@ func waitForKeyRotation(t *testing.T, cpPort int, initialKeys [][]byte) {}}}++// signerOf returns the one key among candidates that verifies biscuit for+// peerID.+func signerOf(t *testing.T, biscuit []byte, peerID peer.ID, candidates [][]byte) ed25519.PublicKey {+ t.Helper()+ for _, candidate := range candidates {+ key := ed25519.PublicKey(candidate)+ if _, err := verifyBiscuitForApplication(biscuit, peerID, []ed25519.PublicKey{key}); err == nil {+ return key+ }+ }+ t.Fatalf("no live control plane key verifies the node's credential")+ return nil+}++// waitForKeysRetired returns once none of retiring is served by /keys any+// more, and when that was observed.+func waitForKeysRetired(t *testing.T, cpPort int, retiring [][]byte) time.Time {+ t.Helper()+ deadline := time.Now().Add(10 * time.Second)+ for {+ stillValid := false+ for _, key := range fetchPublicKeys(t, cpPort) {+ for _, old := range retiring {+ if bytes.Equal(key, old) {+ stillValid = true+ }+ }+ }+ if !stillValid {+ return time.Now()+ }+ if time.Now().After(deadline) {+ t.Fatal("timed out waiting for the retiring key to leave /keys")+ }+ time.Sleep(200 * time.Millisecond)+ }+}++// waitForLeaseRenewedAfter returns once the control plane reports a router+// lease renewal later than after; the renewal carries the router's biscuit+// and is refused when that biscuit's key is no longer valid.+func waitForLeaseRenewedAfter(t *testing.T, cpPort int, after time.Time) {+ t.Helper()+ deadline := time.Now().Add(10 * time.Second)+ for {+ for _, lease := range fetchAdminStatus(t, cpPort, "test-admin-token").ActiveRouters {+ if lease.LastRenewal.After(after) {+ return+ }+ }+ if time.Now().After(deadline) {+ t.Fatalf("no router lease renewed after %s: the router's credential was not re-issued", after.Format(time.RFC3339Nano))+ }+ time.Sleep(200 * time.Millisecond)+ }+}++func publicKeysFrom(raw [][]byte) []ed25519.PublicKey {+ keys := make([]ed25519.PublicKey, 0, len(raw))+ for _, key := range raw {+ keys = append(keys, ed25519.PublicKey(key))+ }+ return keys+}++func readLog(t *testing.T, path string) string {+ t.Helper()+ data, err := os.ReadFile(path)+ if err != nil {+ return fmt.Sprintf("(no log: %v)", err)+ }+ return string(data)+}