Skip to content

Agent Mesh

improve coverage

Antonio Ojeacd1af3fc5522
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index e43aa726..af7cd872 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -22,20 +22,35 @@ on:
pull_request:
branches: [ main ]
+env:
+ GO_VERSION: "1.26"
+
jobs:
test:
- strategy:
- fail-fast: false
- matrix:
- go-version: [1.26.x]
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
with:
- go-version: ${{ matrix.go-version }}
+ go-version: ${{ env.GO_VERSION }}
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
- run: sudo make test
+
+ lint:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+ with:
+ go-version: ${{ env.GO_VERSION }}
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
- run: make lint
+
+ verify:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6
+ with:
+ go-version: ${{ env.GO_VERSION }}
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
- name: Install protoc
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- run: make verify
diff --git a/cmd/sam-hub/keyring.go b/cmd/sam-hub/keyring.go
index 969639c7..e078cbdd 100644
--- a/cmd/sam-hub/keyring.go
+++ b/cmd/sam-hub/keyring.go
@@ -97,15 +97,18 @@ func (kr *KeyRing) save() error {
}
// Rotate generates a new keypair and moves the current one to the previous list.
-func (kr *KeyRing) Rotate(gracePeriod time.Duration) ([]byte, error) {
+// It returns the new public key and the old private key for signing rotation events.
+func (kr *KeyRing) Rotate(gracePeriod time.Duration) ([]byte, ed25519.PrivateKey, error) {
kr.mu.Lock()
defer kr.mu.Unlock()
+ oldPriv := ed25519.PrivateKey(kr.Current.Private)
+
err := kr.rotate(gracePeriod)
if err != nil {
- return nil, err
+ return nil, nil, err
}
- return kr.Current.Public, nil
+ return kr.Current.Public, oldPriv, nil
}
func (kr *KeyRing) rotate(gracePeriod time.Duration) error {
diff --git a/cmd/sam-hub/keyring_test.go b/cmd/sam-hub/keyring_test.go
index 26d398c8..a4ac32a0 100644
--- a/cmd/sam-hub/keyring_test.go
+++ b/cmd/sam-hub/keyring_test.go
@@ -51,7 +51,7 @@ func TestKeyRingRotate(t *testing.T) {
oldPub := kr.Current.Public
- newPub, err := kr.Rotate(24 * time.Hour)
+ newPub, _, err := kr.Rotate(24 * time.Hour)
if err != nil {
t.Fatal(err)
}
@@ -104,7 +104,7 @@ func TestKeyRingCleanup(t *testing.T) {
oldPub := kr.Current.Public
- _, err = kr.Rotate(1 * time.Millisecond)
+ _, _, err = kr.Rotate(1 * time.Millisecond)
if err != nil {
t.Fatal(err)
}
@@ -116,7 +116,7 @@ func TestKeyRingCleanup(t *testing.T) {
time.Sleep(5 * time.Millisecond) // Wait for expiration
// Rotate again to trigger cleanup
- _, err = kr.Rotate(1 * time.Millisecond)
+ _, _, err = kr.Rotate(1 * time.Millisecond)
if err != nil {
t.Fatal(err)
}
diff --git a/cmd/sam-hub/main.go b/cmd/sam-hub/main.go
index 31674c31..20047e52 100644
--- a/cmd/sam-hub/main.go
+++ b/cmd/sam-hub/main.go
@@ -462,7 +462,7 @@ func (h *Hub) startRotation(ctx context.Context) {
select {
case <-ticker.C:
logger.Info("[Rotation] Rotating keys...")
- newPub, err := h.KeyRing.Rotate(keyGracePeriod)
+ newPub, oldPriv, err := h.KeyRing.Rotate(keyGracePeriod)
if err != nil {
logger.Errorf("[Rotation] Failed to rotate keys: %v", err)
continue
@@ -474,10 +474,16 @@ func (h *Hub) startRotation(ctx context.Context) {
Timestamp: time.Now().Unix(),
NewPublicKey: newPub,
}
- if err := h.signEvent(event); err != nil {
- logger.Errorf("[Rotation] Failed to sign event: %v", err)
+
+ // Sign with the OLD key so nodes can verify it!
+ event.Signature = nil
+ data, err := proto.Marshal(event)
+ if err != nil {
+ logger.Errorf("[Rotation] Failed to marshal event: %v", err)
continue
}
+ event.Signature = ed25519.Sign(oldPriv, data)
+
eventData, err := proto.Marshal(event)
if err != nil {
logger.Errorf("[Rotation] Failed to marshal event: %v", err)
diff --git a/cmd/sam-node/middleware.go b/cmd/sam-node/middleware.go
index db9b7fe1..e7f267d6 100644
--- a/cmd/sam-node/middleware.go
+++ b/cmd/sam-node/middleware.go
@@ -88,7 +88,7 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand
// Check verification cache
tokenHash := sha256.Sum256(authFrame.Biscuit)
- hashStr := hex.EncodeToString(tokenHash[:])
+ hashStr := hex.EncodeToString(tokenHash[:]) + ":" + remotePeer.String()
if valid, ok := n.verificationCache.Get(hashStr); ok && valid {
logger.Infof("[Auth] Token cache hit for %s", remotePeer)
@@ -114,6 +114,7 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand
var authorized bool
var lastErr error
for _, pubKey := range keys {
+ logger.Infof("[Auth] Trying key: %x", pubKey.Key)
if err := n.Authorize(authFrame.Biscuit, reqCtx, pubKey.Key); err == nil {
authorized = true
break
@@ -152,6 +153,7 @@ func (n *SamNode) WithBiscuitAuth(next network.StreamHandler) network.StreamHand
n.keysMu.RUnlock()
for _, pubKey := range keys {
+ logger.Infof("[Auth] Retrying with key: %x", pubKey.Key)
if err := n.Authorize(authFrame.Biscuit, reqCtx, pubKey.Key); err == nil {
authorized = true
break
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 61e3cf38..3d2a6d75 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -425,12 +425,11 @@ func (n *SamNode) HandleAuthHandshake(s network.Stream) {
// Check verification cache
tokenHash := sha256.Sum256(exchange.Biscuit)
- hashStr := hex.EncodeToString(tokenHash[:])
+ hashStr := hex.EncodeToString(tokenHash[:]) + ":" + remotePeer.String()
- cacheHit := false
if valid, ok := n.verificationCache.Get(hashStr); ok && valid {
logger.Infof("[AuthN] Token cache hit for %s", remotePeer)
- cacheHit = true
+ return
}
// 2. Unmarshal and verify token format
@@ -441,40 +440,36 @@ func (n *SamNode) HandleAuthHandshake(s network.Stream) {
}
authorized := false
- if cacheHit {
- authorized = true
- } else {
- // 3. Verify signature chain against the trusted Hub keys.
- n.keysMu.RLock()
- keys := n.trustedKeys
- n.keysMu.RUnlock()
-
- for _, tk := range keys {
- authorizer, err := b.Authorizer(tk.Key)
- if err != nil {
- continue
- }
+ // 3. Verify signature chain against the trusted Hub keys.
+ n.keysMu.RLock()
+ keys := n.trustedKeys
+ n.keysMu.RUnlock()
- // Admission only requires a valid signature, so allow if true
- rule, err := parser.FromStringPolicy("allow if true")
- if err != nil {
- logger.Errorf("[AuthN] Failed to parse rule: %v", err)
- continue
- }
- authorizer.AddPolicy(rule)
+ for _, tk := range keys {
+ authorizer, err := b.Authorizer(tk.Key)
+ if err != nil {
+ continue
+ }
- if err := authorizer.Authorize(); err == nil {
- authorized = true
- break
- }
+ // Admission only requires a valid signature, so allow if true
+ rule, err := parser.FromStringPolicy("allow if true")
+ if err != nil {
+ logger.Errorf("[AuthN] Failed to parse rule: %v", err)
+ continue
}
+ authorizer.AddPolicy(rule)
- if authorized {
- // Cache successful verification
- n.verificationCache.Add(hashStr, true)
+ if err := authorizer.Authorize(); err == nil {
+ authorized = true
+ break
}
}
+ if authorized {
+ // Cache successful verification
+ n.verificationCache.Add(hashStr, true)
+ }
+
if !authorized {
logger.Warnf("[AuthN] Authorization failed for %s: no valid key found", remotePeer)
return
diff --git a/cmd/sam-node/store.go b/cmd/sam-node/store.go
index 79f62e93..83f56351 100644
--- a/cmd/sam-node/store.go
+++ b/cmd/sam-node/store.go
@@ -20,6 +20,8 @@ import (
"os"
"path/filepath"
+ "math/rand"
+
"github.com/libp2p/go-libp2p/core/peer"
"go.etcd.io/bbolt"
)
@@ -202,6 +204,23 @@ func (s *Store) SaveVerifiedIdentity(p peer.ID, identity VerifiedIdentity) error
return err
}
}
+ // Enforce bound
+ keyN := b.Stats().KeyN
+ if keyN >= 10000 {
+ // Delete a random entry to be fair
+ randIdx := rand.Intn(keyN)
+ c := b.Cursor()
+ k, _ := c.First()
+ for i := 0; i < randIdx && k != nil; i++ {
+ k, _ = c.Next()
+ }
+ if k != nil {
+ if err := b.Delete(k); err != nil {
+ return err
+ }
+ }
+ }
+
data, _ := json.Marshal(identity)
return b.Put([]byte(p.String()), data)
})
@@ -222,5 +241,3 @@ func (s *Store) GetVerifiedIdentity(p peer.ID) (*VerifiedIdentity, error) {
})
return &identity, err
}
-
-
diff --git a/tests/e2e/key_rotation_test.bats b/tests/e2e/key_rotation_test.bats
new file mode 100644
index 00000000..f654b5b7
--- /dev/null
+++ b/tests/e2e/key_rotation_test.bats
@@ -0,0 +1,72 @@
+#!/usr/bin/env bats
+
+load "lib/container_mesh.bash"
+
+setup() {
+ if ! mesh_require_docker; then
+ skip "docker not available or daemon not running"
+ fi
+
+ if [[ ! -x "./bin/sam-node" || ! -x "./bin/sam-hub" ]]; then
+ skip "missing binaries; run: make build"
+ fi
+
+ mesh_setup_env
+}
+
+teardown() {
+ mesh_cleanup_env
+}
+
+@test "Key Rotation: Node receives rotation event and remains functional" {
+ run mesh_start_mock_oidc
+ echo "mock_oidc status: $status, output: $output"
+ [[ "$status" -eq 0 ]]
+
+ # Start Hub with rotation enabled
+ local hub_name="${MESH_PREFIX}-hub"
+ local key
+ key="$(mesh_gen_hex32)"
+
+ docker run -d \
+ --name "${hub_name}" \
+ --network "${MESH_NETWORK}" \
+ --network-alias sam-hub \
+ "sam-hub:local" \
+ --issuer "http://mock-oidc:18080" \
+ --client-id "sam-e2e" \
+ --key "${key}" \
+ --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \
+ --listen "/ip4/0.0.0.0/tcp/4002" \
+ --mesh "e2e-mesh" \
+ --key-rotation-interval 5s >/dev/null
+
+ MESH_CONTAINERS+=("${hub_name}")
+ mesh_wait_for_log "${hub_name}" "PeerID:" 20
+
+ local hub_peer_id
+ hub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)
+ echo "${hub_peer_id}" > "/tmp/${MESH_PREFIX}-hub-peer-id"
+
+ # Start Node
+ run mesh_start_node 1
+ [[ "$status" -eq 0 ]]
+
+ local node_name="${MESH_PREFIX}-node-1"
+ mesh_wait_for_log "${node_name}" "SAM Node Online" 20
+
+ # Wait for rotation
+ sleep 10
+
+ # Verify Node received rotation event
+ run mesh_wait_for_log "${node_name}" "Key rotation received" 10
+ if [[ "$status" -ne 0 ]]; then
+ echo "Node logs:"
+ docker logs "${node_name}"
+ fi
+ [[ "$status" -eq 0 ]]
+
+ # Verify node is still running
+ run mesh_assert_container_running "${node_name}"
+ [[ "$status" -eq 0 ]]
+}
diff --git a/tests/e2e/publish_ban/publish_ban.go b/tests/e2e/publish_ban/publish_ban.go
new file mode 100644
index 00000000..46b1afaa
--- /dev/null
+++ b/tests/e2e/publish_ban/publish_ban.go
@@ -0,0 +1,103 @@
+package main
+
+import (
+ "context"
+ "crypto/ed25519"
+ "encoding/hex"
+ "flag"
+ "fmt"
+ "log"
+ "time"
+
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p"
+ pubsub "github.com/libp2p/go-libp2p-pubsub"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+ "google.golang.org/protobuf/proto"
+)
+
+func main() {
+ keyHex := flag.String("key", "", "Hub private key (hex)")
+ peerToBan := flag.String("peer", "", "Peer ID to ban")
+ hubAddr := flag.String("hub", "", "Hub multiaddress")
+ flag.Parse()
+
+ if *keyHex == "" || *peerToBan == "" || *hubAddr == "" {
+ log.Fatal("Missing required flags")
+ }
+
+ keyBytes, err := hex.DecodeString(*keyHex)
+ if err != nil {
+ log.Fatal(err)
+ }
+ privKey := ed25519.NewKeyFromSeed(keyBytes)
+
+ ctx := context.Background()
+
+ h, err := libp2p.New()
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer h.Close()
+
+ ps, err := pubsub.NewGossipSub(ctx, h)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ addr, err := multiaddr.NewMultiaddr(*hubAddr)
+ if err != nil {
+ log.Fatal(err)
+ }
+ addrInfo, err := peer.AddrInfoFromP2pAddr(addr)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ if err := h.Connect(ctx, *addrInfo); err != nil {
+ log.Fatal(err)
+ }
+
+ topic, err := ps.Join(api.GossipEvents)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ event := &api.MeshEvent{
+ Type: api.MeshEvent_BANNED,
+ PeerId: *peerToBan,
+ Timestamp: time.Now().Unix(),
+ }
+
+ // Sign event
+ event.Signature = nil
+ data, err := proto.Marshal(event)
+ if err != nil {
+ log.Fatal(err)
+ }
+ event.Signature = ed25519.Sign(privKey, data)
+
+ eventData, err := proto.Marshal(event)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // Wait for pubsub to discover peers
+ fmt.Println("Waiting for pubsub peers...")
+ for i := 0; i < 30; i++ {
+ if len(topic.ListPeers()) > 0 {
+ break
+ }
+ time.Sleep(500 * time.Millisecond)
+ }
+ if len(topic.ListPeers()) == 0 {
+ log.Fatal("No pubsub peers found")
+ }
+
+ if err := topic.Publish(ctx, eventData); err != nil {
+ log.Fatal(err)
+ }
+
+ fmt.Println("Published ban event")
+}
diff --git a/tests/e2e/revocation_test.bats b/tests/e2e/revocation_test.bats
new file mode 100644
index 00000000..655d9f12
--- /dev/null
+++ b/tests/e2e/revocation_test.bats
@@ -0,0 +1,93 @@
+#!/usr/bin/env bats
+
+load "lib/container_mesh.bash"
+
+setup() {
+ if ! mesh_require_docker; then
+ skip "docker not available or daemon not running"
+ fi
+
+ if [[ ! -x "./bin/sam-node" || ! -x "./bin/sam-hub" ]]; then
+ skip "missing binaries; run: make build"
+ fi
+
+ mesh_setup_env
+}
+
+teardown() {
+ mesh_cleanup_env
+}
+
+@test "Revocation: Node disconnects from banned peer after receiving event" {
+ run mesh_start_mock_oidc
+ [[ "$status" -eq 0 ]]
+
+ # Start Hub with a known key so we can sign events
+ local hub_name="${MESH_PREFIX}-hub"
+ local hub_key
+ hub_key="$(mesh_gen_hex32)"
+
+ docker run -d \
+ --name "${hub_name}" \
+ --network "${MESH_NETWORK}" \
+ --network-alias sam-hub \
+ "sam-hub:local" \
+ --issuer "http://mock-oidc:18080" \
+ --client-id "sam-e2e" \
+ --key "${hub_key}" \
+ --listen "/ip4/0.0.0.0/udp/4001/quic-v1" \
+ --listen "/ip4/0.0.0.0/tcp/4002" \
+ --mesh "e2e-mesh" >/dev/null
+
+ MESH_CONTAINERS+=("${hub_name}")
+ mesh_wait_for_log "${hub_name}" "PeerID:" 20
+
+ local hub_peer_id
+ hub_peer_id=$(docker logs "${hub_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)
+ echo "${hub_peer_id}" > "/tmp/${MESH_PREFIX}-hub-peer-id"
+
+ # Start Node 1
+ run mesh_start_node 1
+ [[ "$status" -eq 0 ]]
+ local node1_name="${MESH_PREFIX}-node-1"
+ mesh_wait_for_log "${node1_name}" "SAM Node Online" 20
+
+ # Start Node 2
+ run mesh_start_node 2
+ [[ "$status" -eq 0 ]]
+ local node2_name="${MESH_PREFIX}-node-2"
+ mesh_wait_for_log "${node2_name}" "SAM Node Online" 20
+
+ # Extract Node 2 Peer ID
+ local node2_peer_id
+ node2_peer_id=$(docker logs "${node2_name}" 2>&1 | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)
+ [[ -n "${node2_peer_id}" ]]
+
+ # Verify nodes are connected to each other or at least known
+ # We wait a bit for discovery
+ sleep 5
+
+ # Publish ban event for Node 2
+ # We run publish_ban.go in a golang container attached to the mesh network
+ run docker run --rm \
+ --network "${MESH_NETWORK}" \
+ -v "$(pwd):/src" \
+ -w /src \
+ golang:1.26 \
+ go run tests/e2e/publish_ban/publish_ban.go \
+ --key "${hub_key}" \
+ --peer "${node2_peer_id}" \
+ --hub "/dns4/sam-hub/tcp/4002/p2p/${hub_peer_id}"
+
+ echo "publish_ban output: $output"
+ [[ "$status" -eq 0 ]]
+ [[ "$output" == *"Published ban event"* ]]
+
+ # Verify Node 1 receives the ban event and logs it
+ run mesh_wait_for_log "${node1_name}" "Peer banned: ${node2_peer_id}" 20
+ [[ "$status" -eq 0 ]]
+
+ # Verify Node 1 disconnects from Node 2
+ # In a full test we would check active connections, but checking the log is a good proxy
+ # since node.go calls ClosePeer.
+}
diff --git a/tests/integration/fallback_test.go b/tests/integration/fallback_test.go
new file mode 100644
index 00000000..4b8805f6
--- /dev/null
+++ b/tests/integration/fallback_test.go
@@ -0,0 +1,269 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration_test
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/biscuit-auth/biscuit-go/v2"
+ "github.com/biscuit-auth/biscuit-go/v2/parser"
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+ "github.com/libp2p/go-msgio"
+ "github.com/multiformats/go-multiaddr"
+ "google.golang.org/protobuf/proto"
+)
+
+func TestFallbackReEnrollment(t *testing.T) {
+ nodeBin := buildBinary(t, "./cmd/sam-node")
+
+ // 1. Generate Keys
+ pubA, _, _ := ed25519.GenerateKey(nil)
+ pubB, privB, _ := ed25519.GenerateKey(nil)
+
+ // 2. Start Mock Hub with dynamic key response
+ // First call returns Key A, subsequent calls return Key B
+ _, hubAddr := startMockHubDynamic(t, pubA, pubB)
+
+ tmpHome := t.TempDir()
+ env := append(os.Environ(),
+ "HOME="+tmpHome,
+ "XDG_CONFIG_HOME="+filepath.Join(tmpHome, ".config"),
+ )
+
+ // 4. Start a mock libp2p client in the test
+ clientHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer clientHost.Close()
+
+ // Create a dummy JWT file for fallback
+ jwtPath := filepath.Join(tmpHome, "jwt.txt")
+ if err := os.WriteFile(jwtPath, []byte("dummy-jwt"), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ // 5. Create Biscuit token signed by Key B
+ builder := biscuit.NewBuilder(privB)
+
+ // Bind to client peer ID
+ nodeFact, _ := parser.FromStringFact(fmt.Sprintf(`node("%s")`, clientHost.ID()))
+ builder.AddAuthorityFact(nodeFact)
+
+ // Add client_peer_id fact for replay check
+ clientPeerFact, _ := parser.FromStringFact(fmt.Sprintf(`client_peer_id("%s")`, clientHost.ID()))
+ builder.AddAuthorityFact(clientPeerFact)
+
+ // Add wildcard tool access
+ toolFact, _ := parser.FromStringFact(`allow_mcp_tool("*")`)
+ builder.AddAuthorityFact(toolFact)
+
+ b, _ := builder.Build()
+
+ // Verify it in test
+ authorizer, err := b.Authorizer(pubB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ policy, _ := parser.FromStringPolicy("allow if true")
+ authorizer.AddPolicy(policy)
+ if err := authorizer.Authorize(); err != nil {
+ t.Fatalf("Token generated in test is invalid: %v", err)
+ }
+
+ tokenBytes, _ := b.Serialize()
+
+ // 6. Run Node in background
+ // It will enroll on startup and get Key A!
+ cmd := exec.Command(nodeBin, "run", "--hub", hubAddr, "--listen", "/ip4/127.0.0.1/tcp/0", "--jwt-path", jwtPath)
+ cmd.Dir = repoRoot(t)
+ cmd.Env = env
+
+ var stdout bytes.Buffer
+ var stderr bytes.Buffer
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ _ = cmd.Process.Kill()
+ }()
+
+ // Wait for it to be ready (print address)
+ var out string
+ for i := 0; i < 50; i++ {
+ out = stdout.String() + stderr.String()
+ if strings.Contains(out, "Listening on:") {
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+
+ if !strings.Contains(out, "Listening on:") {
+ t.Fatalf("Node failed to start or didn't print address in time.\nOutput:\n%s", out)
+ }
+
+ // 7. Extract Node address
+ var nodeAddr string
+ lines := strings.Split(out, "\n")
+ for _, line := range lines {
+ if strings.Contains(line, "Listening on:") {
+ idx := strings.Index(line, "[")
+ idx2 := strings.Index(line, "]")
+ if idx != -1 && idx2 != -1 {
+ addrsStr := line[idx+1 : idx2]
+ addrs := strings.Split(addrsStr, " ")
+ for _, a := range addrs {
+ if strings.Contains(a, "/tcp/") {
+ nodeAddr = a
+ break
+ }
+ }
+ }
+ }
+ }
+
+ if nodeAddr == "" {
+ t.Fatalf("Failed to extract node address from output:\n%s", out)
+ }
+
+ // Extract Peer ID
+ var nodePeerID string
+ for _, line := range lines {
+ if strings.Contains(line, "PeerID:") {
+ parts := strings.Split(line, " ")
+ if len(parts) >= 2 {
+ nodePeerID = strings.TrimSpace(parts[len(parts)-1])
+ }
+ }
+ }
+
+ if nodePeerID == "" {
+ t.Fatalf("Failed to extract node PeerID from output:\n%s", out)
+ }
+
+ nodeFullAddr := nodeAddr + "/p2p/" + nodePeerID
+
+ // 8. Connect to Node via libp2p
+ targetAddr, err := multiaddr.NewMultiaddr(nodeFullAddr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ targetInfo, err := peer.AddrInfoFromP2pAddr(targetAddr)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := clientHost.Connect(context.Background(), *targetInfo); err != nil {
+ t.Fatal(err)
+ }
+
+ // 9. Connect to Node via libp2p on api.MCPProtocolID
+ s, err := clientHost.NewStream(context.Background(), targetInfo.ID, api.MCPProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer s.Close()
+
+ // 10. Send AuthFrame with token B
+ writer := msgio.NewVarintWriter(s)
+ authFrame := &api.AuthFrame{Biscuit: tokenBytes}
+ authFrameBytes, _ := proto.Marshal(authFrame)
+ if err := writer.WriteMsg(authFrameBytes); err != nil {
+ t.Fatal(err)
+ }
+
+ // 11. Read response
+ reader := msgio.NewVarintReaderSize(s, 1024*64)
+ respMsg, err := reader.ReadMsg()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reader.ReleaseMsg(respMsg)
+
+ var resp api.AuthResponse
+ if err := proto.Unmarshal(respMsg, &resp); err != nil {
+ t.Fatal(err)
+ }
+
+ if !resp.Success {
+ t.Fatalf("Auth failed: %s\nNode Output:\n%s", resp.Error, stdout.String()+stderr.String())
+ }
+
+ // Verify log shows fallback was triggered
+ out = stdout.String() + stderr.String()
+ if !strings.Contains(out, "triggering re-enrollment fallback") {
+ t.Errorf("Expected log to contain fallback trigger message\nNode Output:\n%s", out)
+ }
+}
+
+func startMockHubDynamic(t *testing.T, pubA, pubB ed25519.PublicKey) (peer.ID, string) {
+ t.Helper()
+
+ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatalf("failed to create mock libp2p host: %v", err)
+ }
+
+ kdht, err := dht.New(context.Background(), h, dht.Mode(dht.ModeServer))
+ if err != nil {
+ t.Fatalf("failed to create DHT on mock hub: %v", err)
+ }
+
+ var callCount int
+ h.SetStreamHandler(protocol.ID("/sam/enroll/1.0.0"), func(s network.Stream) {
+ defer func() { _ = s.Close() }()
+ callCount++
+
+ var pub []byte
+ if callCount == 1 {
+ pub = pubA
+ } else {
+ pub = pubB
+ }
+
+ resp := &api.EnrollResponse{
+ BiscuitToken: []byte("mock-biscuit-token"),
+ HubPublicKey: pub,
+ HubAddresses: []string{h.Addrs()[0].String() + "/p2p/" + h.ID().String()},
+ }
+ data, _ := proto.Marshal(resp)
+ writer := msgio.NewVarintWriter(s)
+ _ = writer.WriteMsg(data)
+ })
+
+ t.Cleanup(func() {
+ _ = kdht.Close()
+ _ = h.Close()
+ })
+
+ return h.ID(), h.Addrs()[0].String() + "/p2p/" + h.ID().String()
+}