Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
improve coverage
Antonio Ojeacd1af3fc5522
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yamlindex e43aa726..af7cd872 100644--- a/.github/workflows/test.yaml+++ b/.github/workflows/test.yaml@@ -22,20 +22,35 @@ on:pull_request:branches: [ main ]+env:+ GO_VERSION: "1.26"+jobs:test:- strategy:- fail-fast: false- matrix:- go-version: [1.26.x]runs-on: ubuntu-lateststeps:- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6with:- go-version: ${{ matrix.go-version }}+ go-version: ${{ env.GO_VERSION }}- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6- run: sudo make test++ lint:+ runs-on: ubuntu-latest+ steps:+ - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6+ with:+ go-version: ${{ env.GO_VERSION }}+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6- run: make lint++ verify:+ runs-on: ubuntu-latest+ steps:+ - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6+ with:+ go-version: ${{ env.GO_VERSION }}+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6- name: Install protocrun: sudo apt-get update && sudo apt-get install -y protobuf-compiler- run: make verifydiff --git a/cmd/sam-hub/keyring.go b/cmd/sam-hub/keyring.goindex 969639c7..e078cbdd 100644--- a/cmd/sam-hub/keyring.go+++ b/cmd/sam-hub/keyring.go@@ -97,15 +97,18 @@ func (kr *KeyRing) save() error {}// Rotate generates a new keypair and moves the current one to the previous list.-func (kr *KeyRing) Rotate(gracePeriod time.Duration) ([]byte, error) {+// It returns the new public key and the old private key for signing rotation events.+func (kr *KeyRing) Rotate(gracePeriod time.Duration) ([]byte, ed25519.PrivateKey, error) {kr.mu.Lock()defer kr.mu.Unlock()+ oldPriv := ed25519.PrivateKey(kr.Current.Private)+err := kr.rotate(gracePeriod)if err != nil {- return nil, err+ return nil, nil, err}- return kr.Current.Public, nil+ return kr.Current.Public, oldPriv, nil}func (kr *KeyRing) rotate(gracePeriod time.Duration) error {diff --git a/cmd/sam-hub/keyring_test.go b/cmd/sam-hub/keyring_test.goindex 26d398c8..a4ac32a0 100644--- a/cmd/sam-hub/keyring_test.go+++ b/cmd/sam-hub/keyring_test.go@@ -51,7 +51,7 @@ func TestKeyRingRotate(t *testing.T) {oldPub := kr.Current.Public- newPub, err := kr.Rotate(24 * time.Hour)+ newPub, _, err := kr.Rotate(24 * time.Hour)if err != nil {t.Fatal(err)}@@ -104,7 +104,7 @@ func TestKeyRingCleanup(t *testing.T) {oldPub := kr.Current.Public- _, err = kr.Rotate(1 * time.Millisecond)+ _, _, err = kr.Rotate(1 * time.Millisecond)if err != nil {t.Fatal(err)}@@ -116,7 +116,7 @@ func TestKeyRingCleanup(t *testing.T) {time.Sleep(5 * time.Millisecond) // Wait for expiration// Rotate again to trigger cleanup- _, err = kr.Rotate(1 * time.Millisecond)+ _, _, err = kr.Rotate(1 * time.Millisecond)if err != nil {t.Fatal(err)}diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.goindex 31674c31..20047e52 100644--- a/cmd/sam-hub/main.go+++ b/cmd/sam-hub/main.go@@ -462,7 +462,7 @@ func (h *Hub) startRotation(ctx context.Context) {select {case <-ticker.C:logger.Info("[Rotation] Rotating keys...")- newPub, err := h.KeyRing.Rotate(keyGracePeriod)+ newPub, oldPriv, err := h.KeyRing.Rotate(keyGracePeriod)if err != nil {logger.Errorf("[Rotation] Failed to rotate keys: %v", err)continue@@ -474,10 +474,16 @@ func (h *Hub) startRotation(ctx context.Context) {Timestamp: time.Now().Unix(),NewPublicKey: newPub,}- if err := h.signEvent(event); err != nil {- logger.Errorf("[Rotation] Failed to sign event: %v", err)++ // Sign with the OLD key so nodes can verify it!+ event.Signature = nil+ data, err := proto.Marshal(event)+ if err != nil {+ logger.Errorf("[Rotation] Failed to marshal event: %v", err)continue}+ event.Signature = ed25519.Sign(oldPriv, data)+eventData, err := proto.Marshal(event)if err != nil {logger.Errorf("[Rotation] Failed to marshal event: %v", err)diff --git a/cmd/sam-node/middleware.go b/cmd/sam-node/middleware.goindex db9b7fe1..e7f267d6 100644--- a/cmd/sam-node/middleware.go+++ b/cmd/sam-node/middleware.go@@ -88,7 +88,7 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand// Check verification cachetokenHash := sha256.Sum256(authFrame.Biscuit)- hashStr := hex.EncodeToString(tokenHash[:])+ hashStr := hex.EncodeToString(tokenHash[:]) + ":" + remotePeer.String()if valid, ok := n.verificationCache.Get(hashStr); ok && valid {logger.Infof("[Auth] Token cache hit for %s", remotePeer)@@ -114,6 +114,7 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHandvar authorized boolvar lastErr errorfor _, pubKey := range keys {+ logger.Infof("[Auth] Trying key: %x", pubKey.Key)if err := n.Authorize(authFrame.Biscuit, reqCtx, pubKey.Key); err == nil {authorized = truebreak@@ -152,6 +153,7 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHandn.keysMu.RUnlock()for _, pubKey := range keys {+ logger.Infof("[Auth] Retrying with key: %x", pubKey.Key)if err := n.Authorize(authFrame.Biscuit, reqCtx, pubKey.Key); err == nil {authorized = truebreakdiff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.goindex 61e3cf38..3d2a6d75 100644--- a/cmd/sam-node/node.go+++ b/cmd/sam-node/node.go@@ -425,12 +425,11 @@ func (n *SamNode) HandleAuthHandshake(s network.Stream) {// Check verification cachetokenHash := sha256.Sum256(exchange.Biscuit)- hashStr := hex.EncodeToString(tokenHash[:])+ hashStr := hex.EncodeToString(tokenHash[:]) + ":" + remotePeer.String()- cacheHit := falseif valid, ok := n.verificationCache.Get(hashStr); ok && valid {logger.Infof("[AuthN] Token cache hit for %s", remotePeer)- cacheHit = true+ return}// 2. Unmarshal and verify token format@@ -441,40 +440,36 @@ func (n *SamNode) HandleAuthHandshake(s network.Stream) {}authorized := false- if cacheHit {- authorized = true- } else {- // 3. Verify signature chain against the trusted Hub keys.- n.keysMu.RLock()- keys := n.trustedKeys- n.keysMu.RUnlock()-- for _, tk := range keys {- authorizer, err := b.Authorizer(tk.Key)- if err != nil {- continue- }+ // 3. Verify signature chain against the trusted Hub keys.+ n.keysMu.RLock()+ keys := n.trustedKeys+ n.keysMu.RUnlock()- // Admission only requires a valid signature, so allow if true- rule, err := parser.FromStringPolicy("allow if true")- if err != nil {- logger.Errorf("[AuthN] Failed to parse rule: %v", err)- continue- }- authorizer.AddPolicy(rule)+ for _, tk := range keys {+ authorizer, err := b.Authorizer(tk.Key)+ if err != nil {+ continue+ }- if err := authorizer.Authorize(); err == nil {- authorized = true- break- }+ // Admission only requires a valid signature, so allow if true+ rule, err := parser.FromStringPolicy("allow if true")+ if err != nil {+ logger.Errorf("[AuthN] Failed to parse rule: %v", err)+ continue}+ authorizer.AddPolicy(rule)- if authorized {- // Cache successful verification- n.verificationCache.Add(hashStr, true)+ if err := authorizer.Authorize(); err == nil {+ authorized = true+ break}}+ if authorized {+ // Cache successful verification+ n.verificationCache.Add(hashStr, true)+ }+if !authorized {logger.Warnf("[AuthN] Authorization failed for %s: no valid key found", remotePeer)returndiff --git a/cmd/sam-node/store.go b/cmd/sam-node/store.goindex 79f62e93..83f56351 100644--- a/cmd/sam-node/store.go+++ b/cmd/sam-node/store.go@@ -20,6 +20,8 @@ import ("os""path/filepath"+ "math/rand"+"github.com/libp2p/go-libp2p/core/peer""go.etcd.io/bbolt")@@ -202,6 +204,23 @@ func (s *Store) SaveVerifiedIdentity(p peer.ID, identity VerifiedIdentity) errorreturn err}}+ // Enforce bound+ keyN := b.Stats().KeyN+ if keyN >= 10000 {+ // Delete a random entry to be fair+ randIdx := rand.Intn(keyN)+ c := b.Cursor()+ k, _ := c.First()+ for i := 0; i < randIdx && k != nil; i++ {+ k, _ = c.Next()+ }+ if k != nil {+ if err := b.Delete(k); err != nil {+ return err+ }+ }+ }+data, _ := json.Marshal(identity)return b.Put([]byte(p.String()), data)})@@ -222,5 +241,3 @@ func (s *Store) GetVerifiedIdentity(p peer.ID) (*VerifiedIdentity, error) {})return &identity, err}--diff --git a/tests/e2e/key_rotation_test.bats b/tests/e2e/key_rotation_test.batsnew file mode 100644index 00000000..f654b5b7--- /dev/null+++ b/tests/e2e/key_rotation_test.bats@@ -0,0 +1,72 @@+#!/usr/bin/env bats++load "lib/container_mesh.bash"++setup() {+ if ! mesh_require_docker; then+ skip "docker not available or daemon not running"+ fi++ if [[ ! -x "./bin/sam-node" || ! -x "./bin/sam-hub" ]]; then+ skip "missing binaries; run: make build"+ fi++ mesh_setup_env+}++teardown() {+ mesh_cleanup_env+}++@test "Key Rotation: Node receives rotation event and remains functional" {+ run mesh_start_mock_oidc+ echo "mock_oidc status: $status, output: $output"+ [[ "$status" -eq 0 ]]++ # Start Hub with rotation enabled+ local hub_name="${MESH_PREFIX}-hub"+ local key+ key="$(mesh_gen_hex32)"++ docker run -d \+ --name "${hub_name}" \+ --network "${MESH_NETWORK}" \+ --network-alias sam-hub \+ "sam-hub:local" \+ --issuer "http://mock-oidc:18080" \+ --client-id "sam-e2e" \+ --key "${key}" \+ --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \+ --listen "/ip4/0.0.0.0/tcp/4002" \+ --mesh "e2e-mesh" \+ --key-rotation-interval 5s >/dev/null++ MESH_CONTAINERS+=("${hub_name}")+ mesh_wait_for_log "${hub_name}" "PeerID:" 20++ local hub_peer_id+ hub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)+ echo "${hub_peer_id}" > "/tmp/${MESH_PREFIX}-hub-peer-id"++ # Start Node+ run mesh_start_node 1+ [[ "$status" -eq 0 ]]++ local node_name="${MESH_PREFIX}-node-1"+ mesh_wait_for_log "${node_name}" "SAM Node Online" 20++ # Wait for rotation+ sleep 10++ # Verify Node received rotation event+ run mesh_wait_for_log "${node_name}" "Key rotation received" 10+ if [[ "$status" -ne 0 ]]; then+ echo "Node logs:"+ docker logs "${node_name}"+ fi+ [[ "$status" -eq 0 ]]++ # Verify node is still running+ run mesh_assert_container_running "${node_name}"+ [[ "$status" -eq 0 ]]+}diff --git a/tests/e2e/publish_ban/publish_ban.go b/tests/e2e/publish_ban/publish_ban.gonew file mode 100644index 00000000..46b1afaa--- /dev/null+++ b/tests/e2e/publish_ban/publish_ban.go@@ -0,0 +1,103 @@+package main++import (+ "context"+ "crypto/ed25519"+ "encoding/hex"+ "flag"+ "fmt"+ "log"+ "time"++ "github.com/google/sam/api"+ "github.com/libp2p/go-libp2p"+ pubsub "github.com/libp2p/go-libp2p-pubsub"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/multiformats/go-multiaddr"+ "google.golang.org/protobuf/proto"+)++func main() {+ keyHex := flag.String("key", "", "Hub private key (hex)")+ peerToBan := flag.String("peer", "", "Peer ID to ban")+ hubAddr := flag.String("hub", "", "Hub multiaddress")+ flag.Parse()++ if *keyHex == "" || *peerToBan == "" || *hubAddr == "" {+ log.Fatal("Missing required flags")+ }++ keyBytes, err := hex.DecodeString(*keyHex)+ if err != nil {+ log.Fatal(err)+ }+ privKey := ed25519.NewKeyFromSeed(keyBytes)++ ctx := context.Background()++ h, err := libp2p.New()+ if err != nil {+ log.Fatal(err)+ }+ defer h.Close()++ ps, err := pubsub.NewGossipSub(ctx, h)+ if err != nil {+ log.Fatal(err)+ }++ addr, err := multiaddr.NewMultiaddr(*hubAddr)+ if err != nil {+ log.Fatal(err)+ }+ addrInfo, err := peer.AddrInfoFromP2pAddr(addr)+ if err != nil {+ log.Fatal(err)+ }++ if err := h.Connect(ctx, *addrInfo); err != nil {+ log.Fatal(err)+ }++ topic, err := ps.Join(api.GossipEvents)+ if err != nil {+ log.Fatal(err)+ }++ event := &api.MeshEvent{+ Type: api.MeshEvent_BANNED,+ PeerId: *peerToBan,+ Timestamp: time.Now().Unix(),+ }++ // Sign event+ event.Signature = nil+ data, err := proto.Marshal(event)+ if err != nil {+ log.Fatal(err)+ }+ event.Signature = ed25519.Sign(privKey, data)++ eventData, err := proto.Marshal(event)+ if err != nil {+ log.Fatal(err)+ }++ // Wait for pubsub to discover peers+ fmt.Println("Waiting for pubsub peers...")+ for i := 0; i < 30; i++ {+ if len(topic.ListPeers()) > 0 {+ break+ }+ time.Sleep(500 * time.Millisecond)+ }+ if len(topic.ListPeers()) == 0 {+ log.Fatal("No pubsub peers found")+ }++ if err := topic.Publish(ctx, eventData); err != nil {+ log.Fatal(err)+ }++ fmt.Println("Published ban event")+}diff --git a/tests/e2e/revocation_test.bats b/tests/e2e/revocation_test.batsnew file mode 100644index 00000000..655d9f12--- /dev/null+++ b/tests/e2e/revocation_test.bats@@ -0,0 +1,93 @@+#!/usr/bin/env bats++load "lib/container_mesh.bash"++setup() {+ if ! mesh_require_docker; then+ skip "docker not available or daemon not running"+ fi++ if [[ ! -x "./bin/sam-node" || ! -x "./bin/sam-hub" ]]; then+ skip "missing binaries; run: make build"+ fi++ mesh_setup_env+}++teardown() {+ mesh_cleanup_env+}++@test "Revocation: Node disconnects from banned peer after receiving event" {+ run mesh_start_mock_oidc+ [[ "$status" -eq 0 ]]++ # Start Hub with a known key so we can sign events+ local hub_name="${MESH_PREFIX}-hub"+ local hub_key+ hub_key="$(mesh_gen_hex32)"++ docker run -d \+ --name "${hub_name}" \+ --network "${MESH_NETWORK}" \+ --network-alias sam-hub \+ "sam-hub:local" \+ --issuer "http://mock-oidc:18080" \+ --client-id "sam-e2e" \+ --key "${hub_key}" \+ --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \+ --listen "/ip4/0.0.0.0/tcp/4002" \+ --mesh "e2e-mesh" >/dev/null++ MESH_CONTAINERS+=("${hub_name}")+ mesh_wait_for_log "${hub_name}" "PeerID:" 20++ local hub_peer_id+ hub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)+ echo "${hub_peer_id}" > "/tmp/${MESH_PREFIX}-hub-peer-id"++ # Start Node 1+ run mesh_start_node 1+ [[ "$status" -eq 0 ]]+ local node1_name="${MESH_PREFIX}-node-1"+ mesh_wait_for_log "${node1_name}" "SAM Node Online" 20++ # Start Node 2+ run mesh_start_node 2+ [[ "$status" -eq 0 ]]+ local node2_name="${MESH_PREFIX}-node-2"+ mesh_wait_for_log "${node2_name}" "SAM Node Online" 20++ # Extract Node 2 Peer ID+ local node2_peer_id+ node2_peer_id=$(docker logs "${node2_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)+ [[ -n "${node2_peer_id}" ]]++ # Verify nodes are connected to each other or at least known+ # We wait a bit for discovery+ sleep 5++ # Publish ban event for Node 2+ # We run publish_ban.go in a golang container attached to the mesh network+ run docker run --rm \+ --network "${MESH_NETWORK}" \+ -v "$(pwd):/src" \+ -w /src \+ golang:1.26 \+ go run tests/e2e/publish_ban/publish_ban.go \+ --key "${hub_key}" \+ --peer "${node2_peer_id}" \+ --hub "/dns4/sam-hub/tcp/4002/p2p/${hub_peer_id}"++ echo "publish_ban output: $output"+ [[ "$status" -eq 0 ]]+ [[ "$output" == *"Published ban event"* ]]++ # Verify Node 1 receives the ban event and logs it+ run mesh_wait_for_log "${node1_name}" "Peer banned: ${node2_peer_id}" 20+ [[ "$status" -eq 0 ]]++ # Verify Node 1 disconnects from Node 2+ # In a full test we would check active connections, but checking the log is a good proxy+ # since node.go calls ClosePeer.+}diff --git a/tests/integration/fallback_test.go b/tests/integration/fallback_test.gonew file mode 100644index 00000000..4b8805f6--- /dev/null+++ b/tests/integration/fallback_test.go@@ -0,0 +1,269 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package integration_test++import (+ "bytes"+ "context"+ "crypto/ed25519"+ "fmt"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/biscuit-auth/biscuit-go/v2"+ "github.com/biscuit-auth/biscuit-go/v2/parser"+ "github.com/google/sam/api"+ "github.com/libp2p/go-libp2p"+ dht "github.com/libp2p/go-libp2p-kad-dht"+ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/core/protocol"+ "github.com/libp2p/go-msgio"+ "github.com/multiformats/go-multiaddr"+ "google.golang.org/protobuf/proto"+)++func TestFallbackReEnrollment(t *testing.T) {+ nodeBin := buildBinary(t, "./cmd/sam-node")++ // 1. Generate Keys+ pubA, _, _ := ed25519.GenerateKey(nil)+ pubB, privB, _ := ed25519.GenerateKey(nil)++ // 2. Start Mock Hub with dynamic key response+ // First call returns Key A, subsequent calls return Key B+ _, hubAddr := startMockHubDynamic(t, pubA, pubB)++ tmpHome := t.TempDir()+ env := append(os.Environ(),+ "HOME="+tmpHome,+ "XDG_CONFIG_HOME="+filepath.Join(tmpHome, ".config"),+ )++ // 4. Start a mock libp2p client in the test+ clientHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer clientHost.Close()++ // Create a dummy JWT file for fallback+ jwtPath := filepath.Join(tmpHome, "jwt.txt")+ if err := os.WriteFile(jwtPath, []byte("dummy-jwt"), 0644); err != nil {+ t.Fatal(err)+ }++ // 5. Create Biscuit token signed by Key B+ builder := biscuit.NewBuilder(privB)++ // Bind to client peer ID+ nodeFact, _ := parser.FromStringFact(fmt.Sprintf(`node("%s")`, clientHost.ID()))+ builder.AddAuthorityFact(nodeFact)++ // Add client_peer_id fact for replay check+ clientPeerFact, _ := parser.FromStringFact(fmt.Sprintf(`client_peer_id("%s")`, clientHost.ID()))+ builder.AddAuthorityFact(clientPeerFact)++ // Add wildcard tool access+ toolFact, _ := parser.FromStringFact(`allow_mcp_tool("*")`)+ builder.AddAuthorityFact(toolFact)++ b, _ := builder.Build()++ // Verify it in test+ authorizer, err := b.Authorizer(pubB)+ if err != nil {+ t.Fatal(err)+ }+ policy, _ := parser.FromStringPolicy("allow if true")+ authorizer.AddPolicy(policy)+ if err := authorizer.Authorize(); err != nil {+ t.Fatalf("Token generated in test is invalid: %v", err)+ }++ tokenBytes, _ := b.Serialize()++ // 6. Run Node in background+ // It will enroll on startup and get Key A!+ cmd := exec.Command(nodeBin, "run", "--hub", hubAddr, "--listen", "/ip4/127.0.0.1/tcp/0", "--jwt-path", jwtPath)+ cmd.Dir = repoRoot(t)+ cmd.Env = env++ var stdout bytes.Buffer+ var stderr bytes.Buffer+ cmd.Stdout = &stdout+ cmd.Stderr = &stderr++ if err := cmd.Start(); err != nil {+ t.Fatal(err)+ }+ defer func() {+ _ = cmd.Process.Kill()+ }()++ // Wait for it to be ready (print address)+ var out string+ for i := 0; i < 50; i++ {+ out = stdout.String() + stderr.String()+ if strings.Contains(out, "Listening on:") {+ break+ }+ time.Sleep(100 * time.Millisecond)+ }++ if !strings.Contains(out, "Listening on:") {+ t.Fatalf("Node failed to start or didn't print address in time.\nOutput:\n%s", out)+ }++ // 7. Extract Node address+ var nodeAddr string+ lines := strings.Split(out, "\n")+ for _, line := range lines {+ if strings.Contains(line, "Listening on:") {+ idx := strings.Index(line, "[")+ idx2 := strings.Index(line, "]")+ if idx != -1 && idx2 != -1 {+ addrsStr := line[idx+1 : idx2]+ addrs := strings.Split(addrsStr, " ")+ for _, a := range addrs {+ if strings.Contains(a, "/tcp/") {+ nodeAddr = a+ break+ }+ }+ }+ }+ }++ if nodeAddr == "" {+ t.Fatalf("Failed to extract node address from output:\n%s", out)+ }++ // Extract Peer ID+ var nodePeerID string+ for _, line := range lines {+ if strings.Contains(line, "PeerID:") {+ parts := strings.Split(line, " ")+ if len(parts) >= 2 {+ nodePeerID = strings.TrimSpace(parts[len(parts)-1])+ }+ }+ }++ if nodePeerID == "" {+ t.Fatalf("Failed to extract node PeerID from output:\n%s", out)+ }++ nodeFullAddr := nodeAddr + "/p2p/" + nodePeerID++ // 8. Connect to Node via libp2p+ targetAddr, err := multiaddr.NewMultiaddr(nodeFullAddr)+ if err != nil {+ t.Fatal(err)+ }+ targetInfo, err := peer.AddrInfoFromP2pAddr(targetAddr)+ if err != nil {+ t.Fatal(err)+ }++ if err := clientHost.Connect(context.Background(), *targetInfo); err != nil {+ t.Fatal(err)+ }++ // 9. Connect to Node via libp2p on api.MCPProtocolID+ s, err := clientHost.NewStream(context.Background(), targetInfo.ID, api.MCPProtocolID)+ if err != nil {+ t.Fatal(err)+ }+ defer s.Close()++ // 10. Send AuthFrame with token B+ writer := msgio.NewVarintWriter(s)+ authFrame := &api.AuthFrame{Biscuit: tokenBytes}+ authFrameBytes, _ := proto.Marshal(authFrame)+ if err := writer.WriteMsg(authFrameBytes); err != nil {+ t.Fatal(err)+ }++ // 11. Read response+ reader := msgio.NewVarintReaderSize(s, 1024*64)+ respMsg, err := reader.ReadMsg()+ if err != nil {+ t.Fatal(err)+ }+ defer reader.ReleaseMsg(respMsg)++ var resp api.AuthResponse+ if err := proto.Unmarshal(respMsg, &resp); err != nil {+ t.Fatal(err)+ }++ if !resp.Success {+ t.Fatalf("Auth failed: %s\nNode Output:\n%s", resp.Error, stdout.String()+stderr.String())+ }++ // Verify log shows fallback was triggered+ out = stdout.String() + stderr.String()+ if !strings.Contains(out, "triggering re-enrollment fallback") {+ t.Errorf("Expected log to contain fallback trigger message\nNode Output:\n%s", out)+ }+}++func startMockHubDynamic(t *testing.T, pubA, pubB ed25519.PublicKey) (peer.ID, string) {+ t.Helper()++ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatalf("failed to create mock libp2p host: %v", err)+ }++ kdht, err := dht.New(context.Background(), h, dht.Mode(dht.ModeServer))+ if err != nil {+ t.Fatalf("failed to create DHT on mock hub: %v", err)+ }++ var callCount int+ h.SetStreamHandler(protocol.ID("/sam/enroll/1.0.0"), func(s network.Stream) {+ defer func() { _ = s.Close() }()+ callCount++++ var pub []byte+ if callCount == 1 {+ pub = pubA+ } else {+ pub = pubB+ }++ resp := &api.EnrollResponse{+ BiscuitToken: []byte("mock-biscuit-token"),+ HubPublicKey: pub,+ HubAddresses: []string{h.Addrs()[0].String() + "/p2p/" + h.ID().String()},+ }+ data, _ := proto.Marshal(resp)+ writer := msgio.NewVarintWriter(s)+ _ = writer.WriteMsg(data)+ })++ t.Cleanup(func() {+ _ = kdht.Close()+ _ = h.Close()+ })++ return h.ID(), h.Addrs()[0].String() + "/p2p/" + h.ID().String()+}