Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
test: a member that was offline during a key rotation resumes and joins
Antonio Ojeac31ba58cfc5e
diff --git a/tests/integration/rotation_offline_test.go b/tests/integration/rotation_offline_test.gonew file mode 100644index 00000000..78fe129c--- /dev/null+++ b/tests/integration/rotation_offline_test.go@@ -0,0 +1,247 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package integration_test++import (+ "context"+ "crypto/ed25519"+ "encoding/base64"+ "fmt"+ "os"+ "path/filepath"+ "sort"+ "testing"+ "time"++ "github.com/google/sam/api"+ "github.com/google/sam/internal/identity"+ "github.com/libp2p/go-libp2p"+ "github.com/libp2p/go-libp2p/core/peer"+ libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"+)++// TestKeyRotationWhileMembersOffline is the outage the testnets had twice:+// the control plane rotates its signing key while a member is not running.+// The routers refresh their credentials under the new key within a second+// of the rotation. The member comes back with what it persisted, a+// credential and a trusted key set that both predate the rotation, and+// must still join: pull the new key from the control plane, where the+// retiring key vouches for it, refresh its credential under it, and be+// admitted by routers whose credentials it could not have verified with the+// keys it had on disk. Both SDKs and the Go node are each enrolled, stopped,+// and started again from their state after the rotation, with no enrollment+// token to fall back on, so a member that re-enrolled instead of resuming+// would fail here too.+func TestKeyRotationWhileMembersOffline(t *testing.T) {+ mesh := startSDKMesh(t)+ ctx := context.Background()+ nodeBin := buildBinary(t, "./cmd/sam-node")++ // The members, enrolled under the current key, then stopped. Each keeps+ // its state directory; a token is read only to enroll, so the file can+ // go once the member holds a credential.+ type offlineSDKMember struct {+ launcher sdkRunner+ stateDir string+ peerID string+ biscuit string+ }+ var sdkMembers []offlineSDKMember+ for _, launcher := range sdkMemberLaunchers {+ cmd, skip := launcher.cmd(mesh.root)+ if skip != "" {+ t.Logf("%s SDK skipped: %s", launcher.name, skip)+ continue+ }+ stateDir := filepath.Join(t.TempDir(), "state")+ m := launchSDKMember(t, launcher.name, cmd, mesh.root, mesh.baseURL, mesh.adminToken, "SAM_SDK_STATE_DIR="+stateDir)+ sdkMembers = append(sdkMembers, offlineSDKMember{launcher: launcher, stateDir: stateDir, peerID: m.report.PeerID, biscuit: m.report.Biscuit})+ m.quit(t)+ }++ // The Go node: enrolled through OIDC, its identity and mesh config in+ // its data directory, its credential readable on the owner socket.+ nodeHome := filepath.Join(t.TempDir(), "node")+ if err := os.MkdirAll(nodeHome, 0o755); err != nil {+ t.Fatal(err)+ }+ nodeEnv := append(os.Environ(), "HOME="+nodeHome, "XDG_CONFIG_HOME="+filepath.Join(nodeHome, ".config"))+ socketDir, err := os.MkdirTemp("", "sam-rot-")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = os.RemoveAll(socketDir) })+ nodeSocket := filepath.Join(socketDir, "node.sock")+ nodeArgs := []string{"run",+ "--control-plane", mesh.baseURL,+ "--allow-loopback",+ "--api-token-path", tokenPath(t, "node-token"),+ "--socket-path", nodeSocket,+ "--log-level", "debug",+ }+ node := launchNode(t, nodeBin, nodeEnv, nodeHome,+ append(nodeArgs, "--jwt", mesh.mintToken(map[string]interface{}{"sub": "mock-user", "roles": []string{api.RoleNode}}))...)+ node.waitForAPI(t)+ socketClient := identityEvidenceSocketClient(nodeSocket)+ waitForIdentityEvidenceSocket(t, socketClient)+ var nodeBefore api.IdentityEvidenceResponse+ getIdentityEvidenceJSON(t, socketClient, "/sam/identity", &nodeBefore)+ nodePeer, err := peer.Decode(nodeBefore.PeerId)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := verifyBiscuitForApplication(nodeBefore.Biscuit, nodePeer, []ed25519.PublicKey{mesh.cpPub}); err != nil {+ t.Fatalf("node credential before the rotation does not verify under the current key: %v", err)+ }+ node.kill()++ // The rotation, as the sam-control-plane binary makes it: a new current+ // key, the old one retiring over a grace period, the event on the mesh.+ // Every router refreshes its credential under the new key; a member+ // that trusts only the old key can verify none of them any more.+ newPub, newPriv, err := ed25519.GenerateKey(nil)+ if err != nil {+ t.Fatal(err)+ }+ if err := mesh.store.RotateKeys(ctx, newPriv, newPub, time.Hour); err != nil {+ t.Fatalf("RotateKeys: %v", err)+ }+ if err := mesh.publisher.PublishEvent(ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil {+ t.Fatalf("publish KEY_ROTATION: %v", err)+ }+ waitForRoutersUnderKey(t, ctx, mesh, newPub)++ // The SDK members start again from their state directories. The token+ // path names no file: the member resumes or fails, it cannot enroll.+ for _, s := range sdkMembers {+ cmd, _ := s.launcher.cmd(mesh.root)+ m := launchSDKMember(t, s.launcher.name+" resumed", cmd, mesh.root, mesh.baseURL, mesh.adminToken,+ "SAM_SDK_STATE_DIR="+s.stateDir,+ "SAM_BOOTSTRAP_TOKEN_PATH="+filepath.Join(t.TempDir(), "no-token"))+ if m.report.PeerID != s.peerID {+ t.Fatalf("%s came back as %s, want the identity it persisted, %s", m.name, m.report.PeerID, s.peerID)+ }+ assertAdmittedByEveryRouter(t, m, mesh.routerAddrs)+ if m.report.Biscuit == s.biscuit {+ t.Fatalf("%s still holds the credential minted before the rotation", m.name)+ }+ assertSignedOnlyBy(t, m.name, m.report.Biscuit, m.report.PeerID, newPub, mesh.cpPub)+ if res := m.sync(t); !res.OK {+ t.Fatalf("%s: pull after the resume failed: %s", m.name, res.Error)+ }+ // The node across the mesh admits the resumed member too.+ if cred := m.auth(t, mesh.samNode.p2pAddr); cred.PeerID != mesh.samNode.peerID.String() {+ t.Fatalf("%s verified the node as %+v after the resume", m.name, cred)+ }+ m.quit(t)+ }++ // The Go node starts from its stored identity, with no token: it pulls+ // from the control plane before it starts and comes up under the new key.+ node = launchNode(t, nodeBin, nodeEnv, nodeHome, nodeArgs...)+ node.waitForAPI(t)+ waitForIdentityEvidenceSocket(t, socketClient)+ var nodeAfter api.IdentityEvidenceResponse+ getIdentityEvidenceJSON(t, socketClient, "/sam/identity", &nodeAfter)+ if nodeAfter.PeerId != nodeBefore.PeerId {+ t.Fatalf("node came back as %s, want its stored identity %s", nodeAfter.PeerId, nodeBefore.PeerId)+ }+ if _, err := verifyBiscuitForApplication(nodeAfter.Biscuit, nodePeer, []ed25519.PublicKey{newPub}); err != nil {+ t.Fatalf("node credential after the resume does not verify under the new key: %v\n--- node.log ---\n%s", err, node.log())+ }+ if _, err := verifyBiscuitForApplication(nodeAfter.Biscuit, nodePeer, []ed25519.PublicKey{mesh.cpPub}); err == nil {+ t.Fatal("node credential after the resume still verifies under the retiring key: it was not refreshed")+ }+ waitForPeerOnRouter(t, mesh.cpPort, mesh.adminToken, nodeBefore.PeerId, 10*time.Second)+ node.kill()+}++// waitForRoutersUnderKey returns once every router presents a credential+// signed by pub in the auth handshake. A Go peer holding a credential under+// the retiring key, still valid, runs the handshake.+func waitForRoutersUnderKey(t *testing.T, ctx context.Context, mesh *sdkMesh, pub ed25519.PublicKey) {+ t.Helper()+ h, err := libp2p.New(libp2p.NoListenAddrs, libp2p.Security(libp2ptls.ID, libp2ptls.New))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = h.Close() }()+ deadline := time.Now().Add(10 * time.Second)+ for _, routerAddr := range mesh.routerAddrs {+ info, err := peer.AddrInfoFromString(routerAddr)+ if err != nil {+ t.Fatal(err)+ }+ connectCtx, cancel := context.WithTimeout(ctx, 10*time.Second)+ err = h.Connect(connectCtx, *info)+ cancel()+ if err != nil {+ t.Fatalf("could not connect to router %s: %v", routerAddr, err)+ }+ for {+ routerBiscuit := authHandshake(t, ctx, h, info.ID, goHostBiscuit(t, mesh.cpPriv, h.ID()))+ if err := identity.VerifyBiscuitRole(routerBiscuit, pub, api.RoleRouter, 5*time.Second); err == nil {+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("router %s did not refresh its credential under the new key", info.ID)+ }+ time.Sleep(200 * time.Millisecond)+ }+ }+}++// assertAdmittedByEveryRouter checks the join report names every router the+// control plane lists, each verified as holding the router role.+func assertAdmittedByEveryRouter(t *testing.T, m *sdkMember, routerAddrs []string) {+ t.Helper()+ var admitted, want []string+ for _, r := range m.report.Routers {+ admitted = append(admitted, r.PeerID)+ if !contains(r.Roles, api.RoleRouter) {+ t.Fatalf("%s: router %s credential roles %v lack %s", m.name, r.PeerID, r.Roles, api.RoleRouter)+ }+ }+ for _, a := range routerAddrs {+ want = append(want, extractPeerID(a))+ }+ sort.Strings(admitted)+ sort.Strings(want)+ if fmt.Sprint(admitted) != fmt.Sprint(want) {+ t.Fatalf("%s was admitted by %v, want every router %v", m.name, admitted, want)+ }+}++// assertSignedOnlyBy checks a base64 credential verifies for peerID under+// signer and under none of the others.+func assertSignedOnlyBy(t *testing.T, who, encoded, peerID string, signer ed25519.PublicKey, others ...ed25519.PublicKey) {+ t.Helper()+ raw, err := base64.StdEncoding.DecodeString(encoded)+ if err != nil {+ t.Fatalf("%s credential is not base64: %v", who, err)+ }+ id, err := peer.Decode(peerID)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := identity.VerifyBiscuitAndGetExpiry(raw, id, []ed25519.PublicKey{signer}, 5*time.Second); err != nil {+ t.Fatalf("%s credential does not verify under the new key: %v", who, err)+ }+ for _, other := range others {+ if _, err := identity.VerifyBiscuitAndGetExpiry(raw, id, []ed25519.PublicKey{other}, 5*time.Second); err == nil {+ t.Fatalf("%s credential still verifies under a retiring key", who)+ }+ }+}