Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package integration_test
import (
"bufio"
"bytes"
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"reflect"
"slices"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/controlplane"
"github.com/google/agentmesh/internal/identity"
"github.com/google/agentmesh/internal/storage"
"github.com/libp2p/go-libp2p"
libp2phttp "github.com/libp2p/go-libp2p-http"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
circuitpb "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/pb"
circuitproto "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
circuitutil "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/util"
libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
"github.com/libp2p/go-msgio"
"github.com/multiformats/go-multiaddr"
"google.golang.org/protobuf/proto"
)
// sdkMesh is the real mesh the SDK tests run against: a control plane
// publishing gossip events, a agentmesh-router, and a agentmesh-node serving the MCP
// service "calc" from a backend the test runs.
type sdkMesh struct {
root string
store *storage.SQLStore
publisher *controlplane.P2PMeshAdapter
baseURL string
adminToken string
cpPort int
cpPriv ed25519.PrivateKey
cpPub ed25519.PublicKey
routerAddr string
routerPeer string
// routerAddrs lists every router; routerAddr is one of them, for tests
// that need a router. The control plane hands them out in a random order
// per response, so members spread across them.
routerAddrs []string
samNode *backgroundNode
nodeAPI string
// egressSeen records what sdkMeshEgressHost received: the node presents
// sdkMeshEgressCredential to it and nothing of the caller.
egressSeen func() []*http.Request
// mintToken mints an OIDC token the control plane accepts, as a platform's
// workload identity token would be.
mintToken func(map[string]interface{}) string
}
// sdkMeshLabels is what every provider in the mesh attests, the node and the
// SDK members alike: the inputs of the "one pair of two wrong fails" case of
// internal/node/labels_gate_test.go, so the caller-side check can be run
// against a real credential from every implementation.
var sdkMeshLabels = map[string]string{"region": "na-us", "team": "platform"}
// The egress destination the agentmesh-node serves, assigned to it by label, and
// the credential the platform delivered to the node's host for it.
const (
sdkMeshEgressHost = "api.example.test"
sdkMeshEgressCredential = "ghp_node_secret"
)
// sdkMeshLabelRequirements are the caller requirements the matrix runs against
// sdkMeshLabels: both pairs attested, so a requirement of several pairs is
// met; one pair of two wrong, so it is refused (api.LabelCheck joins them with
// `,`); one pair that matches; the coarser value of a finer claim, so there is
// no hierarchy.
var sdkMeshLabelRequirements = []struct {
name string
required map[string]string
allowed bool
}{
{"every pair attested", map[string]string{"region": "na-us", "team": "platform"}, true},
{"one pair of two wrong fails", map[string]string{"region": "eu", "team": "platform"}, false},
{"exact match", map[string]string{"team": "platform"}, true},
{"no built-in hierarchy", map[string]string{"region": "na"}, false},
}
const sdkMeshAdminToken = "test-admin-token"
func startSDKMesh(t *testing.T) *sdkMesh {
t.Helper()
root := repoRoot(t)
oidcURL, mintToken := startCustomMockOIDC(t)
var egressMu sync.Mutex
var egressSeen []*http.Request
store, err := storage.NewSQLStore("sqlite", filepath.Join(t.TempDir(), "cp.db"))
if err != nil {
t.Fatalf("failed to create store: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
srv, err := controlplane.NewServer(controlplane.Options{
ListenAddr: "127.0.0.1:0",
AdminToken: sdkMeshAdminToken,
OIDCIssuer: oidcURL,
AllowedAudiences: []string{"agentmesh-audience"},
AutoApproveEnrollment: true,
}, store)
if err != nil {
t.Fatalf("failed to create control plane: %v", err)
}
if err := srv.Start(); err != nil {
t.Fatalf("failed to start control plane: %v", err)
}
t.Cleanup(func() { _ = srv.Close() })
baseURL := "http://" + srv.Addr()
_, portStr, _ := net.SplitHostPort(srv.Addr())
cpPort, _ := strconv.Atoi(portStr)
// Bans and rotations reach the mesh as gossip events through the routers,
// as the agentmesh-control-plane binary publishes them.
publisher, err := controlplane.NewMeshPublisher(context.Background(), store, 500*time.Millisecond)
if err != nil {
t.Fatalf("failed to start mesh event publisher: %v", err)
}
t.Cleanup(func() { _ = publisher.Close() })
srv.SetMeshAdapter(publisher)
// The router enrolls through OIDC with group "routers" and the node with
// user mock-user. Every member holds the node role, which may reach any
// service on any target and declare the two labels the matrix uses; the
// SDK members get it from their bootstrap token and the node from its
// binding. The node alone carries team=platform, which is what selects it
// to serve the egress destination.
egress := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
egressMu.Lock()
egressSeen = append(egressSeen, r.Clone(context.Background()))
egressMu.Unlock()
w.WriteHeader(http.StatusNoContent)
}))
t.Cleanup(egress.Close)
policyFile := filepath.Join(t.TempDir(), "policy.yaml")
policy := fmt.Sprintf(`roles:
- name: %s
allowed_services: []
allowed_targets: ["*"]
allowed_labels: ["zone=*"]
- name: %s
allowed_services: ["*"]
allowed_publish: ["*"]
allowed_targets: ["*"]
allowed_labels: ["region=*", "team=*"]
bindings:
- role: %s
members: ["group:routers"]
- role: %s
members: ["user:mock-user"]
egress:
- name: %s
target_url: %q
credential: github
served_by: ["team=platform"]
`, api.RoleRouter, api.RoleNode, api.RoleRouter, api.RoleNode, sdkMeshEgressHost, egress.URL)
if err := os.WriteFile(policyFile, []byte(policy), 0o600); err != nil {
t.Fatal(err)
}
injectPolicyYAML(t, cpPort, sdkMeshAdminToken, policyFile)
// Two routers, as the testnets run several: a member authenticates with
// every one and reserves a relay slot on the first the control plane
// lists for it, a different one from member to member.
dirA, dirB := t.TempDir(), t.TempDir()
var routerA, routerB, nodeBin string
var rwg sync.WaitGroup
rwg.Add(3)
go func() {
defer rwg.Done()
routerA, _ = startRouter(t, dirA, cpPort, mintToken, "router-a")
}()
go func() {
defer rwg.Done()
routerB, _ = startRouter(t, dirB, cpPort, mintToken, "router-b")
}()
go func() {
defer rwg.Done()
nodeBin = buildBinary(t, "./cmd/agentmesh-node")
}()
rwg.Wait()
routerAddrs := []string{routerA, routerB}
routerAddr := routerA
cpPriv, cpPub, err := store.GetCurrentKey(context.Background())
if err != nil {
t.Fatalf("failed to load control plane signing key: %v", err)
}
// The Go node: a agentmesh-node enrolled through OIDC, listening on loopback,
// serving the MCP service "calc" from a backend this test runs.
backend := httptest.NewServer(newBoundaryMCPHandler(t))
t.Cleanup(backend.Close)
// A stock A2A agent behind the node; its card names its own address.
agentCard := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/.well-known/agent-card.json" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, sdkMeshStockAgentCard)
}))
t.Cleanup(agentCard.Close)
nodeHome := filepath.Join(t.TempDir(), "node")
if err := os.MkdirAll(nodeHome, 0o755); err != nil {
t.Fatal(err)
}
secrets := filepath.Join(nodeHome, "secrets")
if err := os.MkdirAll(secrets, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(secrets, "github"), []byte(sdkMeshEgressCredential+"\n"), 0o600); err != nil {
t.Fatal(err)
}
samNode := launchNode(t, nodeBin,
append(os.Environ(), "HOME="+nodeHome, "XDG_CONFIG_HOME="+filepath.Join(nodeHome, ".config")),
nodeHome, "run",
"--control-plane", baseURL,
"--jwt", mintToken(map[string]interface{}{"sub": "mock-user", "roles": []string{api.RoleNode}}),
"--allow-loopback",
"--api-token-path", tokenPath(t, "node-token"),
"--discovery-interval", "100ms",
"--config", writeNodeConfig(t, nodeHome, sdkMeshLabels,
svcDecl{Type: "mcp", Name: "calc", TargetURL: backend.URL},
svcDecl{Type: "a2a", Name: sdkMeshAgentName, TargetURL: agentCard.URL}),
"--secrets-dir", secrets,
"--log-level", "debug",
)
return &sdkMesh{
root: root,
store: store,
publisher: publisher,
baseURL: baseURL,
adminToken: sdkMeshAdminToken,
cpPort: cpPort,
cpPriv: cpPriv,
cpPub: cpPub,
routerAddr: routerAddr,
routerPeer: extractPeerID(routerAddr),
routerAddrs: routerAddrs,
samNode: samNode,
nodeAPI: samNode.waitForAPI(t),
egressSeen: func() []*http.Request {
egressMu.Lock()
defer egressMu.Unlock()
return append([]*http.Request(nil), egressSeen...)
},
mintToken: mintToken,
}
}
const sdkMeshAgentName = "echo-agent"
const sdkMeshStockAgentCard = `{"name":"echo-agent","description":"stock agent behind a agentmesh-node","version":"1.0.0",` +
`"capabilities":{"streaming":true},` +
`"supportedInterfaces":[{"url":"http://127.0.0.1:7777/","protocolBinding":"JSONRPC","protocolVersion":"1.0"},` +
`{"url":"127.0.0.1:50051","protocolBinding":"GRPC","protocolVersion":"1.0"}],` +
`"signatures":[{"protected":"eyJhbGciOiJFUzI1NiJ9","signature":"c3RhbGU"}],` +
`"skills":[],"defaultInputModes":["text/plain"],"defaultOutputModes":["text/plain"]}`
// sdkMember is a running SDK conformance-join runner: a mesh member written
// in another language that the test drives over stdin/stdout. Both runners
// (sdk/js/src/conformance-join.ts, sdk/python/src/agent_mesh/conformance_join.py)
// speak the same line protocol.
type sdkMember struct {
name string
report sdkJoinReport
cmd *exec.Cmd
stdin io.WriteCloser
lines *bufio.Scanner
stderr *bytes.Buffer
}
// sdkJoinReport is the first line a runner prints once on the mesh.
type sdkJoinReport struct {
SDK string `json:"sdk"`
PeerID string `json:"peer_id"`
Routers []struct {
PeerID string `json:"peer_id"`
Addr string `json:"addr"`
Roles []string `json:"roles"`
} `json:"routers"`
RelayAddresses []string `json:"relay_addresses"`
DirectAddresses []string `json:"direct_addresses"`
Biscuit string `json:"biscuit"`
}
// sdkAuthResult answers an {"cmd":"auth"} command.
type sdkAuthResult struct {
OK bool `json:"ok"`
Error string `json:"error"`
PeerID string `json:"peer_id"`
Roles []string `json:"roles"`
Labels map[string]string `json:"labels"`
Expiration int64 `json:"expiration"`
}
var sdkMemberLaunchers = []sdkRunner{
{
name: "js",
cmd: func(root string) (*exec.Cmd, string) {
entry := filepath.Join(root, "sdk", "js", "dist", "conformance-join.js")
if _, err := exec.LookPath("node"); err != nil {
return nil, "node is not installed"
}
if _, err := os.Stat(entry); err != nil {
return nil, "sdk/js is not built (cd sdk/js && npm ci && npm run build)"
}
return exec.Command("node", entry), ""
},
},
{
name: "python",
cmd: func(root string) (*exec.Cmd, string) {
python := filepath.Join(root, "sdk", "python", ".venv", "bin", "python")
if _, err := os.Stat(python); err != nil {
var lookErr error
if python, lookErr = exec.LookPath("python3"); lookErr != nil {
return nil, "python3 is not installed"
}
}
if err := exec.Command(python, "-c", "import agent_mesh.session").Run(); err != nil {
return nil, "agent_mesh is not importable with libp2p (pip install -e sdk/python)"
}
return exec.Command(python, "-m", "agent_mesh.conformance_join"), ""
},
},
}
// TestNativeSDKsMesh runs one mesh: a control plane, a agentmesh-router, a
// agentmesh-node serving an MCP service, and one member per SDK, all real. It
// then walks the connectivity matrix. Every member authenticates with the
// router and gets a relay reservation, which the router grants only to
// admitted peers. Each SDK member reaches the agentmesh-node directly and the
// other SDK member both directly and through the router, and on every path
// both ends verify each other's credential with the /mesh/auth/1.0.0
// handshake. The agentmesh-node reaches each SDK member through the router. A Go
// peer that is itself admitted does the same and also checks that a forged
// credential gets no answer. Then each SDK member discovers the agentmesh-node's
// service in the DHT, lists and calls its tools over /mesh/mcp/1.0.0 through
// the router, reads the node's own catalog, and is refused a service the
// node does not have. Every member and the agentmesh-node attest the same labels,
// and the caller-side label requirement is run as one matrix from each SDK
// against the node and from the node against each SDK agent, so the three
// implementations agree on what a requirement of several pairs means. Then
// each SDK member accepts A2A requests for its
// agent: the agentmesh-node reaches it by peer ID through its egress proxy
// (/mesh/<peer>/a2a/agent/...), the other SDK member does the same over the
// mesh, the agent is in no discovery table, the member speaks no
// /mesh/mcp/1.0.0, and a Go peer holding a role the policy grants nothing is
// refused with 403. Any SDK whose toolchain is missing is skipped, and the
// matrix shrinks to the members present.
func TestNativeSDKsMesh(t *testing.T) {
mesh := startSDKMesh(t)
root, baseURL, adminToken, cpPort := mesh.root, mesh.baseURL, mesh.adminToken, mesh.cpPort
routerAddr, cpPriv, cpPub := mesh.routerAddr, mesh.cpPriv, mesh.cpPub
store, publisher, samNode, nodeAPI := mesh.store, mesh.publisher, mesh.samNode, mesh.nodeAPI
const serviceName = "calc"
ctx := context.Background()
// One member per SDK, each listening directly on loopback too so both
// the direct and the relayed path can be walked.
type memberSlot struct {
launcher sdkRunner
cmd *exec.Cmd
}
var slots []memberSlot
for _, launcher := range sdkMemberLaunchers {
cmd, skip := launcher.cmd(root)
if skip != "" {
t.Logf("%s SDK skipped: %s", launcher.name, skip)
continue
}
slots = append(slots, memberSlot{launcher: launcher, cmd: cmd})
}
if len(slots) == 0 {
t.Skip("no SDK toolchain available; see sdk/README.md")
}
members := make([]*sdkMember, len(slots))
var mwg sync.WaitGroup
for i := range slots {
i := i
mwg.Add(1)
go func() {
defer mwg.Done()
members[i] = startSDKMember(t, slots[i].launcher.name, slots[i].cmd, root, baseURL, adminToken, mesh.routerAddrs)
}()
}
mwg.Wait()
// The router reports every member connected to the control plane.
waitForPeerOnRouter(t, cpPort, adminToken, samNode.peerID.String(), 5*time.Second)
for _, m := range members {
waitForPeerOnRouter(t, cpPort, adminToken, m.report.PeerID, 5*time.Second)
}
goPeer := newAdmittedGoPeer(t, ctx, cpPriv, mesh.routerAddrs)
for _, m := range members {
m := m
t.Run(m.name, func(t *testing.T) {
// SDK -> agentmesh-node, directly: the node's HandleAuthHandshake
// verifies the member's biscuit and answers with its own.
nodeCred := m.auth(t, samNode.p2pAddr)
if nodeCred.PeerID != samNode.peerID.String() || !contains(nodeCred.Roles, api.RoleNode) {
t.Fatalf("%s verified the node as %+v", m.name, nodeCred)
}
if !strings.Contains(samNode.log(), "Successfully authenticated peer "+m.report.PeerID) {
t.Errorf("agentmesh-node log does not record admitting %s", m.report.PeerID)
}
// agentmesh-node -> SDK, through the router: the node reaches the member
// on its relayed address, which the router only serves for
// authenticated peers on both ends.
connectPeerWithToken(t, nodeAPI, "node-token", m.report.RelayAddresses[0])
// SDK -> every other SDK, directly and through the router.
for _, other := range members {
if other == m {
continue
}
for _, addr := range []string{pickDirectAddr(t, other.report.DirectAddresses, other.report.PeerID), other.report.RelayAddresses[0]} {
cred := m.auth(t, addr)
if cred.PeerID != other.report.PeerID || !contains(cred.Roles, api.RoleNode) {
t.Fatalf("%s verified %s at %s as %+v", m.name, other.name, addr, cred)
}
}
}
// Go peer -> SDK, through the router, both ways of the handshake.
target := multiaddr.StringCast(m.report.RelayAddresses[0])
sdkPeer, _ := peer.Decode(m.report.PeerID)
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := goPeer.Connect(dialCtx, peer.AddrInfo{ID: sdkPeer, Addrs: []multiaddr.Multiaddr{target}}); err != nil {
t.Fatalf("go peer could not reach %s at %s: %v", m.name, target, err)
}
memberBiscuit := authHandshake(t, ctx, goPeer, sdkPeer, goHostBiscuit(t, cpPriv, goPeer.ID()))
if _, err := identity.VerifyBiscuitAndGetExpiry(memberBiscuit, sdkPeer, []ed25519.PublicKey{cpPub}, 5*time.Second); err != nil {
t.Fatalf("%s answered with a credential that does not verify: %v", m.name, err)
}
if err := identity.VerifyBiscuitRole(memberBiscuit, cpPub, api.RoleNode, 5*time.Second); err != nil {
t.Fatalf("%s credential lacks the node role: %v", m.name, err)
}
if want, _ := base64.StdEncoding.DecodeString(m.report.Biscuit); !bytes.Equal(want, memberBiscuit) {
t.Fatal("member answered with a different biscuit than it reported holding")
}
refuseForgedFrame(t, ctx, goPeer, sdkPeer)
// A peer that is not on the mesh cannot be reached: the address
// names an unknown peer behind the router.
stranger, _ := peer.Decode("12D3KooWA4Xop1JaT3MHxwYMkCepYsv4iPVopMXwCz5iHYdBfeSB")
if res := m.authRaw(t, routerAddr+"/p2p-circuit/p2p/"+stranger.String()); res.OK {
t.Fatalf("%s reached a peer that is not on the mesh", m.name)
}
// Services. The node advertises calc in the DHT once its backend
// answered a probe; the member finds it there and calls it through
// the router, the way agentmesh-node's call_remote_tool would.
providers := m.discoverUntil(t, "mcp", serviceName, samNode.peerID.String(), 20*time.Second)
if len(providers) != 1 {
t.Fatalf("%s discovered %+v for %s, want only the node", m.name, providers, serviceName)
}
// By peer ID: the node reserved on a router of its choosing, and
// the member finds the relayed path through whichever of the
// routers it joined through relays for the node.
nodeRelayAddr := samNode.peerID.String()
if tools := m.tools(t, nodeRelayAddr, "mcp://"+serviceName); !contains(tools, "add") {
t.Fatalf("%s listed %v on mcp://%s, want add", m.name, tools, serviceName)
}
if res := m.call(t, nodeRelayAddr, "mcp://"+serviceName, "add", map[string]any{"a": 1, "b": 2}); res.IsError || !contains(res.Text, "fake-result:add") {
t.Fatalf("%s calling add: %+v", m.name, res)
}
// The node's own catalog is the empty target.
if tools := m.tools(t, samNode.p2pAddr, ""); !contains(tools, "list_local_services") {
t.Fatalf("%s listed %v on the node's catalog", m.name, tools)
}
if res := m.call(t, samNode.p2pAddr, "", "list_local_services", nil); res.IsError || !strings.Contains(strings.Join(res.Text, "\n"), serviceName) {
t.Fatalf("%s list_local_services: %+v", m.name, res)
}
// A service the node does not have gets the stream closed on it.
if res := m.callRaw(t, nodeRelayAddr, "mcp://no-such-service", "add", nil); res.OK {
t.Fatalf("%s called a service the node does not serve: %+v", m.name, res)
}
// A stock client bootstraps from the node's agent card, so the SDK
// serves it rewritten for the mesh, as the node's egress proxy does.
res := m.http(t, nodeRelayAddr, "a2a://"+sdkMeshAgentName, "/.well-known/agent-card.json")
var card struct {
SupportedInterfaces []struct {
URL string `json:"url"`
ProtocolBinding string `json:"protocolBinding"`
} `json:"supportedInterfaces"`
Capabilities struct {
Streaming bool `json:"streaming"`
} `json:"capabilities"`
Signatures []json.RawMessage `json:"signatures"`
}
if res.Status != 200 || json.Unmarshal([]byte(res.Body), &card) != nil {
t.Fatalf("%s fetching the node's agent card: %+v", m.name, res)
}
meshBase := "http://mesh/mesh/" + samNode.peerID.String() + "/a2a/" + sdkMeshAgentName
if len(card.SupportedInterfaces) != 1 || card.SupportedInterfaces[0].URL != meshBase || card.SupportedInterfaces[0].ProtocolBinding != "JSONRPC" {
t.Errorf("%s got interfaces %+v, want one JSONRPC interface at %s", m.name, card.SupportedInterfaces, meshBase)
}
// Streaming stays as the agent declares it: the SDK's transport streams.
if !card.Capabilities.Streaming || len(card.Signatures) != 0 {
t.Errorf("%s got a card with streaming=%v (want true) and %d signatures (want none)", m.name, card.Capabilities.Streaming, len(card.Signatures))
}
// The bare service root serves the same card, as the node does for a2a-go.
if root := m.http(t, nodeRelayAddr, "a2a://"+sdkMeshAgentName, "/"); root.Body != res.Body {
t.Errorf("%s got a different card at the service root: %s", m.name, root.Body)
}
})
}
// The caller-side label requirement means the same thing in every
// implementation: a requirement of several pairs is met only when every
// one of them is attested, and a value is matched whole. The node and
// every SDK member attest the same labels, so the matrix below runs each
// requirement from each caller against a credential each provider minted:
// SDK -> node over /mesh/mcp/1.0.0, and node -> SDK through the egress
// proxy's X-Mesh-Required-Labels, which runs checkPeerLabels.
t.Run("required-labels", func(t *testing.T) {
for _, m := range members {
if got := m.auth(t, samNode.p2pAddr).Labels; got["region"] != sdkMeshLabels["region"] || got["team"] != sdkMeshLabels["team"] {
t.Fatalf("%s sees the node's labels as %v, want %v", m.name, got, sdkMeshLabels)
}
}
for _, tc := range sdkMeshLabelRequirements {
t.Run(tc.name, func(t *testing.T) {
nodeRelayAddr := samNode.peerID.String()
for _, m := range members {
res := m.toolsRequiring(t, nodeRelayAddr, "mcp://"+serviceName, tc.required)
if res.OK != tc.allowed {
t.Errorf("%s requiring %v of the node: ok=%v (%s), want %v", m.name, tc.required, res.OK, res.Error, tc.allowed)
}
if !tc.allowed && res.OK == false && !strings.Contains(res.Error, "LabelsNotSatisfied") {
t.Errorf("%s requiring %v of the node was refused for another reason: %s", m.name, tc.required, res.Error)
}
}
})
}
})
// A member whose egress floor nobody attests is refused before anything is
// sent, MCP and HTTP alike; the members above carry a floor the mesh does
// satisfy (launchSDKMember). MCP to an SDK member fails before the floor: no /mesh/mcp.
t.Run("egress-floor", func(t *testing.T) {
for _, m := range members {
m := m
t.Run(m.name, func(t *testing.T) {
t.Parallel()
cmd := exec.Command(m.cmd.Path, m.cmd.Args[1:]...)
floored := launchSDKMember(t, m.name+"-floored", cmd, root, baseURL, adminToken, "AGENTMESH_SDK_EGRESS_REQUIRE_LABELS=team=nobody")
byFloor := func(what, target, err string) {
if !strings.Contains(err, "LabelsNotSatisfied") {
t.Errorf("%s %s to %s was not refused by the floor: %q", floored.name, what, target, err)
}
}
nodeRelayAddr := samNode.peerID.String()
byFloor("tools", "the node", floored.toolsRequiring(t, nodeRelayAddr, "mcp://"+serviceName, nil).Error)
byFloor("call", "the node", floored.callRaw(t, nodeRelayAddr, "mcp://"+serviceName, "add", nil).Error)
byFloor("http", "the node", floored.httpRaw(t, nodeRelayAddr, "egress://"+sdkMeshEgressHost, "/").Error)
for _, m := range members {
byFloor("http", m.name, floored.httpRaw(t, m.report.PeerID, "a2a://agent", "/card").Error)
if res := floored.toolsRequiring(t, m.report.PeerID, "mcp://"+serviceName, nil); res.OK {
t.Errorf("%s listed tools of %s, an SDK member: %v", floored.name, m.name, res.Tools)
}
if res := floored.callRaw(t, m.report.PeerID, "mcp://"+serviceName, "add", nil); res.OK {
t.Errorf("%s called a tool of %s, an SDK member: %+v", floored.name, m.name, res)
}
}
floored.quit(t)
})
}
})
// A relay that accepted the circuit while nobody speaks on the far end,
// as a caller sees one whose destination never completes its handshake.
// The dial ends at the SDK's dial timeout with an error and the circuit
// is let go, in every SDK. Every member dials at once, so this costs the
// test one timeout, not one per SDK.
t.Run("stalled-relay", func(t *testing.T) {
relayAddr, released := startStalledRelay(t)
stranger := "12D3KooWA4Xop1JaT3MHxwYMkCepYsv4iPVopMXwCz5iHYdBfeSB"
for _, m := range members {
m.write(t, map[string]string{"cmd": "auth", "addr": relayAddr + "/p2p-circuit/p2p/" + stranger, "timeout_ms": "400"})
}
for _, m := range members {
var res sdkAuthResult
if line := m.readLine(t, 20*time.Second); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: auth answered %q", m.name, line)
}
if res.OK {
t.Fatalf("%s reached a peer through a relay that delivered nothing", m.name)
}
t.Logf("%s gave up on the stalled circuit: %s", m.name, res.Error)
}
deadline := time.Now().Add(5 * time.Second)
for released.Load() < int32(len(members)) && time.Now().Before(deadline) {
time.Sleep(100 * time.Millisecond)
}
if got := released.Load(); got != int32(len(members)) {
t.Fatalf("%d of %d stalled circuits were let go; the rest are leaked on the relay", got, len(members))
}
})
// Every SDK member is an agent: it accepts A2A requests for a2a://agent,
// answered in the runner's process, reachable by peer ID through the
// router. It publishes nothing; the policy rules it evaluates are the
// ones the control plane renders.
for _, m := range members {
m.accept(t, "agent")
}
guest := newGuestGoPeer(t, ctx, cpPriv, mesh.routerAddrs)
for _, m := range members {
m := m
t.Run(m.name+"-accepts", func(t *testing.T) {
sdkPeer, _ := peer.Decode(m.report.PeerID)
// agentmesh-node -> SDK agent, through the egress proxy: the node
// verifies the member's credential, then the member authorizes
// the node's and answers with who called.
status, body := egressGet(t, nodeAPI, "node-token", "/mesh/"+m.report.PeerID+"/a2a/agent/card")
if status != 200 {
t.Fatalf("agentmesh-node egress to %s agent: status %d, body %s", m.name, status, body)
}
var seen struct {
SDK string `json:"sdk"`
Peer string `json:"peer"`
Path string `json:"path"`
}
if json.Unmarshal([]byte(body), &seen) != nil || seen.SDK != m.name || seen.Peer != samNode.peerID.String() || seen.Path != "/card" {
t.Fatalf("agentmesh-node egress to %s agent answered %s", m.name, body)
}
// The node's caller-side requirement against the member's
// credential, the same matrix the members ran against the node's.
for _, tc := range sdkMeshLabelRequirements {
want := http.StatusForbidden
if tc.allowed {
want = http.StatusOK
}
status, body := egressGetRequiring(t, nodeAPI, "node-token", "/mesh/"+m.report.PeerID+"/a2a/agent/card", tc.required)
if status != want {
t.Errorf("agentmesh-node requiring %v of %s (%s): status %d %s, want %d", tc.required, m.name, tc.name, status, strings.TrimSpace(body), want)
}
}
// The agent is not in the discovery table: nothing was announced.
if providers := m.discoverOnce(t, "a2a", "agent"); len(providers) != 0 {
t.Fatalf("%s discovered %+v for a2a://agent, want nothing: an agent is not published", m.name, providers)
}
// SDK -> SDK: the other member reaches the agent by peer ID
// through the router, with no lookup.
for _, other := range members {
if other == m {
continue
}
res := other.http(t, m.report.PeerID, "a2a://agent", "/card")
if res.Status != 200 || !strings.Contains(res.Body, other.report.PeerID) {
t.Fatalf("%s calling a2a://agent on %s: %+v", other.name, m.name, res)
}
// Another agent name the policy grants: authorized, then 404.
if r := other.http(t, m.report.PeerID, "a2a://other", "/card"); r.Status != 404 {
t.Fatalf("%s calling an agent name %s does not answer: %+v", other.name, m.name, r)
}
}
// An SDK member serves no MCP: the stream is refused at the
// protocol negotiation, before any frame is read.
if err := tryMCPStream(ctx, guest, sdkPeer, multiaddr.StringCast(m.report.RelayAddresses[0])); err == nil {
t.Fatalf("%s accepted a /mesh/mcp/1.0.0 stream; an SDK member serves no MCP", m.name)
}
// A peer the control plane vouches for, but whose role the policy
// grants nothing, is refused with 403 on the HTTP path.
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := guest.Connect(dialCtx, peer.AddrInfo{ID: sdkPeer, Addrs: []multiaddr.Multiaddr{multiaddr.StringCast(m.report.RelayAddresses[0])}}); err != nil {
t.Fatalf("guest peer could not reach %s: %v", m.name, err)
}
if status, _ := libp2pHTTPGet(t, ctx, guest, sdkPeer, guestHostBiscuit(t, cpPriv, guest.ID()), "/a2a/agent/card"); status != 403 {
t.Fatalf("%s answered a guest's HTTP request with %d, want 403", m.name, status)
}
// The same peer with a node-role token is answered: the refusal was the policy, not the peer.
if status, body := libp2pHTTPGet(t, ctx, guest, sdkPeer, goHostBiscuit(t, cpPriv, guest.ID()), "/a2a/agent/card"); status != 200 || !strings.Contains(body, guest.ID().String()) {
t.Fatalf("%s answered a node-role HTTP request with %d %s", m.name, status, body)
}
})
}
// Everyone who ran the handshake against a member is in its admitted set.
for _, m := range members {
peers := m.peers(t)
if !contains(peers, goPeer.ID().String()) {
t.Errorf("%s admitted peers %v lack the Go peer %s", m.name, peers, goPeer.ID())
}
for _, other := range members {
if other != m && !contains(peers, other.report.PeerID) {
t.Errorf("%s admitted peers %v lack %s (%s)", m.name, peers, other.name, other.report.PeerID)
}
}
}
// Control plane state reaches the members while they run. A ban travels
// as a gossip event through the router and is enforced without a pull;
// a key rotation lands with the next pull, and a credential that
// predates it is refreshed under the new key.
t.Run("ban-and-rotation", func(t *testing.T) {
// An enrolled peer the control plane can ban: a node record, admitted
// by every member over the router.
outcast := newAdmittedGoPeer(t, ctx, cpPriv, mesh.routerAddrs)
outcastBiscuit := goHostBiscuit(t, cpPriv, outcast.ID())
outcastPub, err := crypto.MarshalPublicKey(outcast.Peerstore().PubKey(outcast.ID()))
if err != nil {
t.Fatal(err)
}
if err := store.EnrollNode(ctx, &storage.EnrolledNode{
PeerID: outcast.ID().String(),
PublicKey: outcastPub,
Biscuit: outcastBiscuit,
Role: api.RoleNode,
EnrollmentType: "BOOTSTRAP",
EnrolledAt: time.Now(),
ExpiresAt: time.Now().Add(time.Hour),
}); err != nil {
t.Fatalf("failed to enroll the outcast: %v", err)
}
for _, m := range members {
sdkPeer, _ := peer.Decode(m.report.PeerID)
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
if err := outcast.Connect(dialCtx, peer.AddrInfo{ID: sdkPeer, Addrs: []multiaddr.Multiaddr{multiaddr.StringCast(m.report.RelayAddresses[0])}}); err != nil {
cancel()
t.Fatalf("outcast could not reach %s: %v", m.name, err)
}
cancel()
authHandshake(t, ctx, outcast, sdkPeer, outcastBiscuit)
if !contains(m.peers(t), outcast.ID().String()) {
t.Fatalf("%s did not admit the outcast before the ban", m.name)
}
}
// The operator bans it. The event goes control plane -> router -> members.
revokeBody, _ := proto.Marshal(&api.TokenRevokeRequest{PeerId: outcast.ID().String()})
req, _ := http.NewRequest(http.MethodPost, baseURL+"/admin/revoke", bytes.NewReader(revokeBody))
req.Header.Set("Authorization", "Bearer "+adminToken)
req.Header.Set("Content-Type", "application/x-protobuf")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("POST /admin/revoke: %v", err)
}
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("POST /admin/revoke: %s", resp.Status)
}
for _, m := range members {
state := m.bannedUntil(t, outcast.ID().String(), 10*time.Second)
if contains(state.AuthenticatedPeers, outcast.ID().String()) {
t.Errorf("%s kept the outcast admitted after the ban", m.name)
}
// Its credential still verifies; it is refused all the same.
sdkPeer, _ := peer.Decode(m.report.PeerID)
if err := tryAuthHandshake(ctx, outcast, sdkPeer, multiaddr.StringCast(m.report.RelayAddresses[0]), outcastBiscuit); err == nil {
t.Errorf("%s admitted the banned outcast again", m.name)
}
// A pull agrees with the event and does not lift the ban.
if res := m.sync(t); !contains(res.Banned, outcast.ID().String()) {
t.Errorf("%s: /info does not list the outcast as banned after the pull: %+v", m.name, res)
}
}
// The control plane rotates its signing key; the retiring key stays
// valid for its grace period, and the rotation is announced as the
// agentmesh-control-plane binary announces it. A member learns the new key
// from the event or from its next pull, and since its credential
// predates the rotation, refreshes it under the new key.
newPub, newPriv, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
if err := store.RotateKeys(ctx, newPriv, newPub, time.Hour); err != nil {
t.Fatalf("RotateKeys: %v", err)
}
if err := publisher.PublishEvent(ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil {
t.Fatalf("publish KEY_ROTATION: %v", err)
}
for _, m := range members {
// Idempotent: the event may already have brought a pull forward.
res := m.sync(t)
if !contains(res.TrustedKeys, hex.EncodeToString(newPub)) || !contains(res.TrustedKeys, hex.EncodeToString(cpPub)) {
t.Fatalf("%s trusts %v after the rotation, want both keys", m.name, res.TrustedKeys)
}
if res.Biscuit == m.report.Biscuit {
t.Fatalf("%s still holds the credential minted before the rotation", m.name)
}
fresh, err := base64.StdEncoding.DecodeString(res.Biscuit)
if err != nil {
t.Fatal(err)
}
sdkPeer, _ := peer.Decode(m.report.PeerID)
if _, err := identity.VerifyBiscuitAndGetExpiry(fresh, sdkPeer, []ed25519.PublicKey{newPub}, 5*time.Second); err != nil {
t.Fatalf("%s refreshed credential does not verify under the new key: %v", m.name, err)
}
if _, err := identity.VerifyBiscuitAndGetExpiry(fresh, sdkPeer, []ed25519.PublicKey{cpPub}, 5*time.Second); err == nil {
t.Fatalf("%s refreshed credential still verifies under the retiring key", m.name)
}
// The mesh keeps working under the new credential: the agentmesh-node
// learned the key from the same event.
deadline := time.Now().Add(10 * time.Second)
for {
cred := m.authRaw(t, samNode.p2pAddr)
if cred.OK && cred.PeerID == samNode.peerID.String() {
break
}
if time.Now().After(deadline) {
t.Fatalf("%s could not authenticate with the node under its refreshed credential: %+v", m.name, cred)
}
time.Sleep(200 * time.Millisecond)
}
}
})
for _, m := range members {
m.quit(t)
}
}
// TestNativeSDKsAcrossRouters is the datapath between two members that joined
// through different routers: one member per SDK on router A and one on router
// B, none with an address of its own, so the relayed path is the only one.
// Every member on one router calls the agent of every member on the other by
// peer ID, over /libp2p-http, and authenticates with it. Which router a peer
// happened to reserve on must not decide whether it can be reached; the
// caller finds the peer through the routers and is admitted by the one that
// relays for it on the way.
func TestNativeSDKsAcrossRouters(t *testing.T) {
mesh := startSDKMesh(t)
if len(mesh.routerAddrs) < 2 {
t.Fatalf("the fixture runs %d router(s), want two", len(mesh.routerAddrs))
}
type placed struct {
*sdkMember
router string
routerAddr string
}
type launchSpec struct {
launcher sdkRunner
cmd *exec.Cmd
router string
routerAddr string
}
var specs []launchSpec
for _, launcher := range sdkMemberLaunchers {
for _, routerAddr := range mesh.routerAddrs {
cmd, skip := launcher.cmd(mesh.root)
if skip != "" {
t.Logf("%s SDK skipped: %s", launcher.name, skip)
break
}
specs = append(specs, launchSpec{
launcher: launcher,
cmd: cmd,
router: extractPeerID(routerAddr),
routerAddr: routerAddr,
})
}
}
if len(specs) == 0 {
t.Skip("no SDK toolchain available; see sdk/README.md")
}
members := make([]placed, len(specs))
var wg sync.WaitGroup
for i, sp := range specs {
i, sp := i, sp
wg.Add(1)
go func() {
defer wg.Done()
m := launchSDKMember(t, sp.launcher.name+"@"+sp.router[len(sp.router)-6:], sp.cmd, mesh.root, mesh.baseURL, mesh.adminToken, "AGENTMESH_SDK_ROUTERS="+sp.router)
m.accept(t, "agent")
members[i] = placed{m, sp.router, sp.routerAddr}
}()
}
wg.Wait()
for _, m := range members {
if got := routerIDs(m.report); !reflect.DeepEqual(got, []string{m.router}) {
t.Fatalf("%s joined through %v, want %s only", m.name, got, m.router)
}
for _, a := range m.report.RelayAddresses {
if !strings.HasPrefix(a, m.routerAddr) {
t.Fatalf("%s reserved on %s, want %s", m.name, a, m.routerAddr)
}
}
if len(m.report.DirectAddresses) != 0 {
t.Fatalf("%s listens on %v, want the relayed path to be the only one", m.name, m.report.DirectAddresses)
}
waitForPeerOnRouter(t, mesh.cpPort, mesh.adminToken, m.report.PeerID, 5*time.Second)
}
// A pair that talked once stays connected, so only the first contact
// between two members has to find its way through the other router.
connected := map[[2]string]bool{}
for _, caller := range members {
for _, agent := range members {
if caller.router == agent.router {
continue
}
t.Run(caller.name+"-reaches-"+agent.name, func(t *testing.T) {
pair := [2]string{caller.report.PeerID, agent.report.PeerID}
if pair[0] > pair[1] {
pair[0], pair[1] = pair[1], pair[0]
}
firstContact := !connected[pair]
connected[pair] = true
// The agent names the verified caller in its answer.
res := caller.http(t, agent.report.PeerID, "a2a://agent", "/hello")
if res.Status != 200 || !strings.Contains(res.Body, caller.report.PeerID) {
t.Fatalf("%s -> %s: %d %s", caller.name, agent.name, res.Status, res.Body)
}
verified := caller.auth(t, agent.report.PeerID)
if verified.PeerID != agent.report.PeerID {
t.Fatalf("%s authenticated %s, want %s", caller.name, verified.PeerID, agent.report.PeerID)
}
// The agent's router admitted the caller on the way and stays admitted.
if got := caller.routers(t); firstContact && !contains(got, agent.router) {
t.Fatalf("%s admitted routers %v after reaching a peer on %s", caller.name, got, agent.router)
}
})
}
}
for _, m := range members {
m.quit(t)
}
}
// A router rescheduled keeps its key and comes back on another address. A
// member that reserved on it advertises a dead relayed address until it
// reserves again, and it must do so at the router's new address: here a
// literal one, as agentmesh-one hands out, so there is no name to re-resolve and
// only the control plane's list, which the member pulls, names it.
func TestNativeSDKsFollowAMovedRouter(t *testing.T) {
mesh := startSDKMesh(t)
keys := t.TempDir()
routerAddr, stop := startRouter(t, keys, mesh.cpPort, mesh.mintToken, "router-c")
router := extractPeerID(routerAddr)
var members []*sdkMember
for _, launcher := range sdkMemberLaunchers {
cmd, skip := launcher.cmd(mesh.root)
if skip != "" {
t.Logf("%s SDK skipped: %s", launcher.name, skip)
continue
}
m := launchSDKMember(t, launcher.name, cmd, mesh.root, mesh.baseURL, mesh.adminToken, "AGENTMESH_SDK_ROUTERS="+router, "AGENTMESH_SDK_RELAY_CHECK_SECONDS=0.5")
if got := routerIDs(m.report); !reflect.DeepEqual(got, []string{router}) {
t.Fatalf("%s joined through %v, want %s only", m.name, got, router)
}
for _, a := range m.report.RelayAddresses {
if !strings.HasPrefix(a, routerAddr) {
t.Fatalf("%s reserved on %s, want %s", m.name, a, routerAddr)
}
}
m.accept(t, "agent")
members = append(members, m)
}
if len(members) == 0 {
t.Skip("no SDK toolchain available; see sdk/README.md")
}
// The same key on another port; the router's lease replaces its address.
stop()
movedAddr, _ := startRouter(t, keys, mesh.cpPort, mesh.mintToken, "router-c")
if extractPeerID(movedAddr) != router || movedAddr == routerAddr {
t.Fatalf("router came back as %s, want %s on another port", movedAddr, routerAddr)
}
for _, m := range members {
if got := m.sync(t).RouterAddresses; !contains(got, movedAddr) || contains(got, routerAddr) {
t.Fatalf("%s pulled routers %v, want %s and not %s", m.name, got, movedAddr, routerAddr)
}
deadline := time.Now().Add(15 * time.Second)
for {
relayed := m.relayAddresses(t)
if len(relayed) > 0 && strings.HasPrefix(relayed[0], movedAddr) {
break
}
if time.Now().After(deadline) {
t.Fatalf("%s still advertises %v, want the router at %s", m.name, relayed, movedAddr)
}
time.Sleep(200 * time.Millisecond)
}
}
// A peer that knows the router where it is now reaches each member there.
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
goPeer := newAdmittedGoPeer(t, ctx, mesh.cpPriv, []string{movedAddr})
for _, m := range members {
sdkPeer, err := peer.Decode(m.report.PeerID)
if err != nil {
t.Fatal(err)
}
if err := goPeer.Connect(ctx, peer.AddrInfo{ID: sdkPeer, Addrs: []multiaddr.Multiaddr{multiaddr.StringCast(m.relayAddresses(t)[0])}}); err != nil {
t.Fatalf("go peer could not reach %s through the moved router: %v", m.name, err)
}
if status, body := libp2pHTTPGet(t, ctx, goPeer, sdkPeer, goHostBiscuit(t, mesh.cpPriv, goPeer.ID()), "/a2a/agent/card"); status != 200 || !strings.Contains(body, goPeer.ID().String()) {
t.Fatalf("%s answered %d %s through the moved router", m.name, status, body)
}
}
for _, m := range members {
m.quit(t)
}
}
// TestNativeSDKsHoldTwoRoutersAndHandleGoAway verifies that when the control
// plane lists three routers in /info, each SDK member picks two by preference
// and load, authenticates and reserves a relay slot on those two only, and
// when one held router stops and sends a /mesh/goaway/1.0.0 DRAINING frame,
// the member records the go-away reason, drops the draining router, and
// attaches to the remaining router from the catalog.
func TestNativeSDKsHoldTwoRoutersAndHandleGoAway(t *testing.T) {
mesh := startSDKMesh(t)
routerJWT := mesh.mintToken(map[string]interface{}{
"sub": "router-c",
"groups": []string{"routers"},
"roles": []string{api.RoleRouter},
})
routerC := startLabelledRouter(t, buildBinary(t, "./cmd/agentmesh-router"), t.TempDir(), "router-c", mesh.cpPort, routerJWT, "zone=preferred")
waitForActiveRouters(t, mesh.cpPort, 3, 10*time.Second)
var members []*sdkMember
for _, launcher := range sdkMemberLaunchers {
cmd, skip := launcher.cmd(mesh.root)
if skip != "" {
t.Logf("%s SDK skipped: %s", launcher.name, skip)
continue
}
m := launchSDKMember(t, launcher.name, cmd, mesh.root, mesh.baseURL, mesh.adminToken,
"AGENTMESH_SDK_ROUTER_PREFER=zone=preferred",
)
if len(m.report.Routers) != 2 {
t.Fatalf("%s joined through %d routers (%v), want 2 of the 3 routers", m.name, len(m.report.Routers), routerIDs(m.report))
}
if m.report.Routers[0].PeerID != routerC.peerID.String() {
t.Fatalf("%s picked %v, want preferred router %s first", m.name, routerIDs(m.report), routerC.peerID)
}
if len(m.report.RelayAddresses) != 2 {
t.Fatalf("%s reserved %d relay addresses (%v), want 2", m.name, len(m.report.RelayAddresses), m.report.RelayAddresses)
}
m.accept(t, "agent")
members = append(members, m)
}
if len(members) == 0 {
t.Skip("no SDK toolchain available; see sdk/README.md")
}
// Send SIGTERM to router-c: it drains its members over /mesh/goaway/1.0.0
// with RouterGoAway_DRAINING before closing.
if err := routerC.cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
wantRemaining := []string{extractPeerID(mesh.routerAddrs[0]), extractPeerID(mesh.routerAddrs[1])}
slices.Sort(wantRemaining)
for _, m := range members {
deadline := time.Now().Add(15 * time.Second)
for {
ans := m.routersAnswer(t)
got := slices.Clone(ans.Routers)
slices.Sort(got)
if ans.GoAwayReceived["DRAINING"] == 1 && slices.Equal(got, wantRemaining) && len(ans.RelayAddresses) == 2 {
break
}
if time.Now().After(deadline) {
t.Fatalf("%s after router-c drained: routers=%v relay_addresses=%v go_away_received=%v, want routers=%v and DRAINING=1",
m.name, ans.Routers, ans.RelayAddresses, ans.GoAwayReceived, wantRemaining)
}
time.Sleep(150 * time.Millisecond)
}
}
for _, m := range members {
m.quit(t)
}
}
// routerIDs lists the routers a join report says admitted the member.
func routerIDs(report sdkJoinReport) []string {
var ids []string
for _, r := range report.Routers {
ids = append(ids, r.PeerID)
}
return ids
}
func startSDKMember(t *testing.T, name string, cmd *exec.Cmd, root, baseURL, adminToken string, routerAddrs []string) *sdkMember {
t.Helper()
m := launchSDKMember(t, name, cmd, root, baseURL, adminToken, "AGENTMESH_SDK_LISTEN_ADDRS=/ip4/127.0.0.1/tcp/0")
// Both routers in the fixture admit the member and grant it a relay reservation.
var admitted []string
for _, r := range m.report.Routers {
admitted = append(admitted, r.PeerID)
if !contains(r.Roles, api.RoleRouter) {
t.Fatalf("%s: router %s credential roles %v lack %s", name, r.PeerID, r.Roles, api.RoleRouter)
}
}
var want []string
for _, a := range routerAddrs {
want = append(want, extractPeerID(a))
}
sort.Strings(admitted)
sort.Strings(want)
if !reflect.DeepEqual(admitted, want) {
t.Fatalf("%s report routers %v, want every router the control plane listed %v", name, admitted, want)
}
if len(m.report.RelayAddresses) != len(m.report.Routers) {
t.Fatalf("%s: got %d relay addresses (%v), want one for each of %d routers", name, len(m.report.RelayAddresses), m.report.RelayAddresses, len(m.report.Routers))
}
for _, r := range m.report.Routers {
wantRelay := r.Addr + "/p2p-circuit/p2p/" + m.report.PeerID
if !contains(m.report.RelayAddresses, wantRelay) {
t.Fatalf("%s relay addresses %v lack %s for router %s", name, m.report.RelayAddresses, wantRelay, r.PeerID)
}
}
return m
}
// launchSDKMember starts a conformance-join runner with a fresh bootstrap
// token, the labels the matrix uses and an egress floor every provider in the
// mesh satisfies, plus env (a later entry wins), and reads its join report.
func launchSDKMember(t *testing.T, name string, cmd *exec.Cmd, root, baseURL, adminToken string, env ...string) *sdkMember {
t.Helper()
tokenPath := filepath.Join(t.TempDir(), "bootstrap.token")
if err := os.WriteFile(tokenPath, []byte(mintBootstrapToken(t, baseURL, adminToken)+"\n"), 0o600); err != nil {
t.Fatalf("failed to write bootstrap token: %v", err)
}
return launchSDKRunner(t, name, cmd, root, baseURL, append([]string{
"AGENTMESH_BOOTSTRAP_TOKEN_PATH=" + tokenPath,
"AGENTMESH_SDK_LABELS=" + labelsEnv(sdkMeshLabels),
"AGENTMESH_SDK_EGRESS_REQUIRE_LABELS=region=" + sdkMeshLabels["region"],
}, env...)...)
}
// launchSDKRunner starts a conformance-join runner against baseURL with env
// naming how it enrolls (a later entry wins), and reads its join report.
func launchSDKRunner(t *testing.T, name string, cmd *exec.Cmd, root, baseURL string, env ...string) *sdkMember {
t.Helper()
cmd.Env = append(append(os.Environ(),
"PYTHONPATH="+filepath.Join(root, "sdk", "python", "src"),
"AGENTMESH_CONTROL_PLANE_URL="+baseURL,
"AGENTMESH_SDK_STATE_DIR="+filepath.Join(t.TempDir(), "state"),
), env...)
cmd.Dir = root
m := &sdkMember{name: name, cmd: cmd, stderr: &bytes.Buffer{}}
cmd.Stderr = m.stderr
stdin, err := cmd.StdinPipe()
if err != nil {
t.Fatal(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
}
m.stdin = stdin
m.lines = bufio.NewScanner(stdout)
m.lines.Buffer(make([]byte, 0, 64*1024), 1024*1024)
if err := cmd.Start(); err != nil {
t.Fatalf("failed to start %s member: %v", name, err)
}
t.Cleanup(func() {
_ = stdin.Close()
_ = cmd.Process.Kill()
if cmd.ProcessState == nil {
_ = cmd.Wait()
}
if t.Failed() {
t.Logf("%s member stderr:\n%s", name, m.stderr.String())
}
})
if line := m.readLine(t, 20*time.Second); json.Unmarshal(line, &m.report) != nil {
t.Fatalf("%s member printed no join report, got: %q", name, line)
}
if _, err := peer.Decode(m.report.PeerID); err != nil {
t.Fatalf("%s report peer_id %q: %v", name, m.report.PeerID, err)
}
return m
}
func (m *sdkMember) send(t *testing.T, command map[string]string) []byte {
t.Helper()
m.write(t, command)
return m.readLine(t, 20*time.Second)
}
// write sends a command without waiting for its answer; readLine collects it.
func (m *sdkMember) write(t *testing.T, command map[string]string) {
t.Helper()
line, _ := json.Marshal(command)
if _, err := m.stdin.Write(append(line, '\n')); err != nil {
t.Fatalf("%s member: write command: %v", m.name, err)
}
}
// authRaw asks the member to connect to addr and run the auth handshake.
func (m *sdkMember) authRaw(t *testing.T, addr string) sdkAuthResult {
t.Helper()
var res sdkAuthResult
if line := m.send(t, map[string]string{"cmd": "auth", "addr": addr}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: auth answered %q", m.name, line)
}
return res
}
// auth is authRaw that must succeed.
func (m *sdkMember) auth(t *testing.T, addr string) sdkAuthResult {
t.Helper()
res := m.authRaw(t, addr)
if !res.OK {
t.Fatalf("%s member could not authenticate with %s: %s", m.name, addr, res.Error)
}
if res.Expiration <= time.Now().Unix() {
t.Fatalf("%s member accepted a credential from %s that is already expired", m.name, addr)
}
return res
}
func (m *sdkMember) peers(t *testing.T) []string {
t.Helper()
var res struct {
AuthenticatedPeers []string `json:"authenticated_peers"`
}
if line := m.send(t, map[string]string{"cmd": "peers"}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: peers answered %q", m.name, line)
}
return res.AuthenticatedPeers
}
// sdkBanned answers a {"cmd":"banned"} command.
type sdkBanned struct {
Banned []string `json:"banned"`
AuthenticatedPeers []string `json:"authenticated_peers"`
}
// bannedUntil polls the member's ban set, without asking it to pull, until
// wantPeer is in it: this is how long a gossip event takes to land.
func (m *sdkMember) bannedUntil(t *testing.T, wantPeer string, timeout time.Duration) sdkBanned {
t.Helper()
deadline := time.Now().Add(timeout)
for {
var res sdkBanned
if line := m.send(t, map[string]string{"cmd": "banned"}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: banned answered %q", m.name, line)
}
if contains(res.Banned, wantPeer) {
return res
}
if time.Now().After(deadline) {
t.Fatalf("%s member never learned the ban of %s from the gossip event; banned=%v\nstderr:\n%s", m.name, wantPeer, res.Banned, m.stderr.String())
}
time.Sleep(200 * time.Millisecond)
}
}
// sdkSyncResult answers a {"cmd":"sync"} command.
type sdkSyncResult struct {
OK bool `json:"ok"`
Error string `json:"error"`
KeysChanged bool `json:"keys_changed"`
Refreshed bool `json:"refreshed"`
TrustedKeys []string `json:"trusted_keys"`
Biscuit string `json:"biscuit"`
Banned []string `json:"banned"`
RouterAddresses []string `json:"router_addresses"`
}
// sync asks the member to pull from the control plane now; every part must land.
func (m *sdkMember) sync(t *testing.T) sdkSyncResult {
t.Helper()
var res sdkSyncResult
if line := m.send(t, map[string]string{"cmd": "sync"}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: sync answered %q", m.name, line)
}
if !res.OK {
t.Fatalf("%s member: sync failed: %s", m.name, res.Error)
}
return res
}
// sdkProvider is one entry of a {"cmd":"discover"} answer.
type sdkProvider struct {
PeerID string `json:"peer_id"`
Addrs []string `json:"addrs"`
}
// discoverUntil repeats the DHT lookup until wantPeer is among the providers
// or the deadline passes; a node advertises a service only after probing
// its backend, on its own schedule.
func (m *sdkMember) discoverUntil(t *testing.T, serviceType, name, wantPeer string, timeout time.Duration) []sdkProvider {
t.Helper()
deadline := time.Now().Add(timeout)
var last []sdkProvider
for {
var res struct {
OK bool `json:"ok"`
Error string `json:"error"`
Providers []sdkProvider `json:"providers"`
}
if line := m.send(t, map[string]string{"cmd": "discover", "type": serviceType, "name": name}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: discover answered %q", m.name, line)
}
if !res.OK {
t.Fatalf("%s member: discover failed: %s", m.name, res.Error)
}
last = res.Providers
for _, p := range res.Providers {
if p.PeerID == wantPeer {
return res.Providers
}
}
if time.Now().After(deadline) {
t.Fatalf("%s member never discovered %s as a provider of %s://%s; last answer %+v", m.name, wantPeer, serviceType, name, last)
}
time.Sleep(500 * time.Millisecond)
}
}
// discoverOnce is one DHT lookup, whatever it returns.
func (m *sdkMember) discoverOnce(t *testing.T, serviceType, name string) []sdkProvider {
t.Helper()
var res struct {
OK bool `json:"ok"`
Error string `json:"error"`
Providers []sdkProvider `json:"providers"`
}
if line := m.send(t, map[string]string{"cmd": "discover", "type": serviceType, "name": name}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: discover answered %q", m.name, line)
}
if !res.OK {
t.Fatalf("%s member: discover failed: %s", m.name, res.Error)
}
return res.Providers
}
func (m *sdkMember) tools(t *testing.T, addr, service string) []string {
t.Helper()
res := m.toolsRequiring(t, addr, service, nil)
if !res.OK {
t.Fatalf("%s member could not list tools of %q at %s: %s", m.name, service, addr, res.Error)
}
return res.Tools
}
// sdkToolsResult answers a {"cmd":"tools"} command.
type sdkToolsResult struct {
OK bool `json:"ok"`
Error string `json:"error"`
Tools []string `json:"tools"`
}
// toolsRequiring lists a provider's tools with a caller-side label requirement
// on the provider's credential, and reports a refusal instead of failing.
func (m *sdkMember) toolsRequiring(t *testing.T, addr, service string, required map[string]string) sdkToolsResult {
t.Helper()
command := map[string]any{"cmd": "tools", "addr": addr, "service": service}
if required != nil {
command["required_labels"] = required
}
line, _ := json.Marshal(command)
if _, err := m.stdin.Write(append(line, '\n')); err != nil {
t.Fatalf("%s member: write command: %v", m.name, err)
}
var res sdkToolsResult
if answer := m.readLine(t, 20*time.Second); json.Unmarshal(answer, &res) != nil {
t.Fatalf("%s member: tools answered %q", m.name, answer)
}
return res
}
// sdkCallResult answers a {"cmd":"call"} command.
type sdkCallResult struct {
OK bool `json:"ok"`
Error string `json:"error"`
IsError bool `json:"is_error"`
Text []string `json:"text"`
}
func (m *sdkMember) callRaw(t *testing.T, addr, service, tool string, args map[string]any) sdkCallResult {
t.Helper()
if args == nil {
args = map[string]any{}
}
line, _ := json.Marshal(map[string]any{"cmd": "call", "addr": addr, "service": service, "tool": tool, "args": args})
if _, err := m.stdin.Write(append(line, '\n')); err != nil {
t.Fatalf("%s member: write command: %v", m.name, err)
}
var res sdkCallResult
if answer := m.readLine(t, 20*time.Second); json.Unmarshal(answer, &res) != nil {
t.Fatalf("%s member: call answered %q", m.name, answer)
}
return res
}
// call is callRaw that must reach the tool.
func (m *sdkMember) call(t *testing.T, addr, service, tool string, args map[string]any) sdkCallResult {
t.Helper()
res := m.callRaw(t, addr, service, tool, args)
if !res.OK {
t.Fatalf("%s member could not call %s on %q at %s: %s", m.name, tool, service, addr, res.Error)
}
return res
}
// accept asks the member to accept A2A requests for its agent, answered in
// its own process.
func (m *sdkMember) accept(t *testing.T, name string) {
t.Helper()
var res struct {
OK bool `json:"ok"`
Error string `json:"error"`
Service string `json:"service"`
}
if line := m.send(t, map[string]string{"cmd": "accept", "name": name}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: accept answered %q", m.name, line)
}
if !res.OK {
t.Fatalf("%s member could not accept a2a://%s: %s", m.name, name, res.Error)
}
if res.Service != "a2a://"+name {
t.Fatalf("%s member accepts %q after asking for a2a://%s", m.name, res.Service, name)
}
}
// sdkHTTPResult answers an {"cmd":"http"} command.
type sdkHTTPResult struct {
OK bool `json:"ok"`
Error string `json:"error"`
Status int `json:"status"`
Body string `json:"body"`
}
// httpRaw asks the member to call an inference or A2A service over
// /libp2p-http, and reports a refusal instead of failing.
func (m *sdkMember) httpRaw(t *testing.T, addr, service, path string) sdkHTTPResult {
t.Helper()
var res sdkHTTPResult
if line := m.send(t, map[string]string{"cmd": "http", "addr": addr, "service": service, "path": path}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: http answered %q", m.name, line)
}
return res
}
// http is httpRaw that must be answered.
func (m *sdkMember) http(t *testing.T, addr, service, path string) sdkHTTPResult {
t.Helper()
res := m.httpRaw(t, addr, service, path)
if !res.OK {
t.Fatalf("%s member could not call %s%s at %s: %s", m.name, service, path, addr, res.Error)
}
return res
}
// routers asks the member which routers admitted it so far.
func (m *sdkMember) routers(t *testing.T) []string {
t.Helper()
return m.routersAnswer(t).Routers
}
// relayAddresses are the `.../p2p-circuit/p2p/<member>` addresses the member
// holds a reservation for right now.
func (m *sdkMember) relayAddresses(t *testing.T) []string {
t.Helper()
return m.routersAnswer(t).RelayAddresses
}
func (m *sdkMember) routersAnswer(t *testing.T) (res struct {
OK bool `json:"ok"`
Error string `json:"error"`
Routers []string `json:"routers"`
RelayAddresses []string `json:"relay_addresses"`
GoAwayReceived map[string]int `json:"go_away_received"`
}) {
t.Helper()
if line := m.send(t, map[string]string{"cmd": "routers"}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: routers answered %q", m.name, line)
}
if !res.OK {
t.Fatalf("%s member: routers: %s", m.name, res.Error)
}
return res
}
func (m *sdkMember) quit(t *testing.T) {
t.Helper()
m.send(t, map[string]string{"cmd": "quit"})
_ = m.stdin.Close()
exited := make(chan error, 1)
go func() { exited <- m.cmd.Wait() }()
select {
case err := <-exited:
if err != nil {
t.Errorf("%s member exited with %v\nstderr:\n%s", m.name, err, m.stderr.String())
}
case <-time.After(15 * time.Second):
_ = m.cmd.Process.Kill()
<-exited
t.Errorf("%s member did not exit after quit\nstderr:\n%s", m.name, m.stderr.String())
}
}
// readLine returns the member's next stdout line or fails after timeout; a
// runner that hangs must not hang the test.
func (m *sdkMember) readLine(t *testing.T, timeout time.Duration) []byte {
t.Helper()
type result struct {
line []byte
err error
}
ch := make(chan result, 1)
go func() {
if m.lines.Scan() {
ch <- result{line: append([]byte(nil), m.lines.Bytes()...)}
return
}
err := m.lines.Err()
if err == nil {
err = io.EOF
}
ch <- result{err: err}
}()
select {
case r := <-ch:
if r.err != nil {
t.Fatalf("%s member stdout ended: %v\nstderr:\n%s", m.name, r.err, m.stderr.String())
}
return r.line
case <-time.After(timeout):
t.Fatalf("%s member printed nothing within %v\nstderr:\n%s", m.name, timeout, m.stderr.String())
return nil
}
}
// newAdmittedGoPeer is a libp2p host configured like agentmesh-node's, connected
// to every router and past their auth handshakes, so each relays for it.
func newAdmittedGoPeer(t *testing.T, ctx context.Context, cpPriv ed25519.PrivateKey, routerAddrs []string) host.Host {
t.Helper()
h, err := libp2p.New(
libp2p.NoListenAddrs,
libp2p.Security(libp2ptls.ID, libp2ptls.New),
libp2p.EnableRelay(),
)
if err != nil {
t.Fatalf("failed to create go peer: %v", err)
}
t.Cleanup(func() { _ = h.Close() })
for _, routerAddr := range routerAddrs {
info, err := peer.AddrInfoFromString(routerAddr)
if err != nil {
t.Fatalf("router address %s: %v", routerAddr, err)
}
connectCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
err = h.Connect(connectCtx, *info)
cancel()
if err != nil {
t.Fatalf("go peer could not connect to router %s: %v", routerAddr, err)
}
routerBiscuit := authHandshake(t, ctx, h, info.ID, goHostBiscuit(t, cpPriv, h.ID()))
if err := identity.VerifyBiscuitRole(routerBiscuit, cpPriv.Public().(ed25519.PublicKey), api.RoleRouter, 5*time.Second); err != nil {
t.Fatalf("router credential lacks the router role: %v", err)
}
}
return h
}
func goHostBiscuit(t *testing.T, cpPriv ed25519.PrivateKey, id peer.ID) []byte {
t.Helper()
b, err := identity.MintBootstrapBiscuitToken(cpPriv, id, api.RoleNode, time.Now().Add(time.Hour), nil, nil)
if err != nil {
t.Fatalf("failed to mint biscuit for %s: %v", id, err)
}
return b
}
// guestHostBiscuit is a valid control plane credential for a role the mesh
// policy does not know, so it grants nothing anywhere.
func guestHostBiscuit(t *testing.T, cpPriv ed25519.PrivateKey, id peer.ID) []byte {
t.Helper()
b, err := identity.MintBootstrapBiscuitToken(cpPriv, id, "mesh:role:guest", time.Now().Add(time.Hour), nil, nil)
if err != nil {
t.Fatalf("failed to mint guest biscuit for %s: %v", id, err)
}
return b
}
// newGuestGoPeer is a Go peer admitted by the routers with a node-role token;
// what it presents to providers is chosen per request.
func newGuestGoPeer(t *testing.T, ctx context.Context, cpPriv ed25519.PrivateKey, routerAddrs []string) host.Host {
t.Helper()
return newAdmittedGoPeer(t, ctx, cpPriv, routerAddrs)
}
// tryMCPStream opens /mesh/mcp/1.0.0 to target through addr and reports
// whether the peer speaks it at all.
func tryMCPStream(ctx context.Context, h host.Host, target peer.ID, addr multiaddr.Multiaddr) error {
ctx, cancel := context.WithTimeout(network.WithAllowLimitedConn(ctx, "mcp"), 10*time.Second)
defer cancel()
if err := h.Connect(ctx, peer.AddrInfo{ID: target, Addrs: []multiaddr.Multiaddr{addr}}); err != nil {
return err
}
s, err := h.NewStream(ctx, target, api.MCPProtocolID)
if err != nil {
return err
}
_ = s.Close()
return nil
}
// libp2pHTTPGet is one GET over /libp2p-http with the biscuit in
// X-Mesh-Biscuit, the way agentmesh-node's egress proxy sends it.
func libp2pHTTPGet(t *testing.T, ctx context.Context, h host.Host, target peer.ID, biscuit []byte, path string) (int, string) {
t.Helper()
client := &http.Client{Transport: libp2phttp.NewTransport(h), Timeout: 15 * time.Second}
req, err := http.NewRequestWithContext(network.WithAllowLimitedConn(ctx, "http"), http.MethodGet, "libp2p://"+target.String()+path, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set(api.HeaderMeshBiscuit, base64.StdEncoding.EncodeToString(biscuit))
resp, err := client.Do(req)
if err != nil {
t.Fatalf("GET %s on %s: %v", path, target, err)
}
defer func() { _ = resp.Body.Close() }()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}
// egressGet is a GET through agentmesh-node's egress proxy with the node API token.
func egressGet(t *testing.T, apiAddr, token, path string) (int, string) {
t.Helper()
return egressGetRequiring(t, apiAddr, token, path, nil)
}
// egressGetRequiring is egressGet with a caller-side label requirement on the
// destination, as X-Mesh-Required-Labels carries it.
func egressGetRequiring(t *testing.T, apiAddr, token, path string, required map[string]string) (int, string) {
t.Helper()
req, err := http.NewRequest(http.MethodGet, "http://"+apiAddr+path, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set(api.HeaderMeshAuthentication, "Bearer "+token)
if required != nil {
req.Header.Set(api.HeaderMeshRequiredLabels, labelsEnv(required))
}
resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
if err != nil {
t.Fatalf("GET %s through the sidecar: %v", path, err)
}
defer func() { _ = resp.Body.Close() }()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}
// labelsEnv renders labels as "k=v,k2=v2", the form the runners' AGENTMESH_SDK_LABELS
// and agentmesh-node's X-Mesh-Required-Labels both take, keys sorted.
func labelsEnv(labels map[string]string) string {
keys := make([]string, 0, len(labels))
for k := range labels {
keys = append(keys, k)
}
sort.Strings(keys)
pairs := make([]string, 0, len(keys))
for _, k := range keys {
pairs = append(pairs, k+"="+labels[k])
}
return strings.Join(pairs, ",")
}
// authHandshake runs the client side of /mesh/auth/1.0.0 against target and
// returns the credential target answers with.
func authHandshake(t *testing.T, ctx context.Context, h host.Host, target peer.ID, biscuit []byte) []byte {
t.Helper()
streamCtx, cancel := context.WithTimeout(network.WithAllowLimitedConn(ctx, "auth"), 10*time.Second)
defer cancel()
s, err := h.NewStream(streamCtx, target, api.AuthProtocolID)
if err != nil {
t.Fatalf("open auth stream to %s: %v", target, err)
}
defer func() { _ = s.Close() }()
_ = s.SetDeadline(time.Now().Add(10 * time.Second))
frame, _ := proto.Marshal(&api.AuthFrame{Biscuit: biscuit})
if err := msgio.NewVarintWriter(s).WriteMsg(frame); err != nil {
t.Fatalf("write auth frame to %s: %v", target, err)
}
msg, err := msgio.NewVarintReaderSize(s, 64*1024).ReadMsg()
if err != nil {
t.Fatalf("read auth response from %s: %v", target, err)
}
var resp api.AuthResponse
if err := proto.Unmarshal(msg, &resp); err != nil {
t.Fatalf("auth response from %s: %v", target, err)
}
if !resp.Success {
t.Fatalf("%s refused the handshake: %s", target, resp.Error)
}
return resp.Biscuit
}
// tryAuthHandshake reconnects to target through addr if needed and runs the
// client side of /mesh/auth/1.0.0, returning the error a refusal produces: a
// connection the gate denies, or a stream closed without an answer.
func tryAuthHandshake(ctx context.Context, h host.Host, target peer.ID, addr multiaddr.Multiaddr, biscuit []byte) error {
dialCtx, cancel := context.WithTimeout(network.WithAllowLimitedConn(ctx, "auth"), 10*time.Second)
defer cancel()
if err := h.Connect(dialCtx, peer.AddrInfo{ID: target, Addrs: []multiaddr.Multiaddr{addr}}); err != nil {
return err
}
s, err := h.NewStream(dialCtx, target, api.AuthProtocolID)
if err != nil {
return err
}
defer func() { _ = s.Close() }()
_ = s.SetDeadline(time.Now().Add(5 * time.Second))
frame, _ := proto.Marshal(&api.AuthFrame{Biscuit: biscuit})
if err := msgio.NewVarintWriter(s).WriteMsg(frame); err != nil {
return err
}
msg, err := msgio.NewVarintReaderSize(s, 64*1024).ReadMsg()
if err != nil {
return err
}
var resp api.AuthResponse
if err := proto.Unmarshal(msg, &resp); err != nil {
return err
}
if !resp.Success {
return fmt.Errorf("refused: %s", resp.Error)
}
return nil
}
// refuseForgedFrame checks that target answers a frame it cannot verify
// with a closed stream and nothing else, as agentmesh-node does.
func refuseForgedFrame(t *testing.T, ctx context.Context, h host.Host, target peer.ID) {
t.Helper()
s, err := h.NewStream(network.WithAllowLimitedConn(ctx, "auth"), target, api.AuthProtocolID)
if err != nil {
t.Fatalf("open auth stream for the forged frame: %v", err)
}
defer func() { _ = s.Close() }()
forged, _ := proto.Marshal(&api.AuthFrame{Biscuit: []byte("not a biscuit")})
if err := msgio.NewVarintWriter(s).WriteMsg(forged); err != nil {
t.Fatalf("write forged frame: %v", err)
}
_ = s.SetReadDeadline(time.Now().Add(5 * time.Second))
if msg, err := msgio.NewVarintReaderSize(s, 64*1024).ReadMsg(); err == nil {
t.Fatalf("%s answered a forged frame with %d bytes", target, len(msg))
}
}
// startStalledRelay is a circuit relay that answers every CONNECT with OK and
// forwards nothing: what a caller sees when the relay's destination accepted
// the circuit but never completes its side of the handshake. It speaks no
// other protocol, so no member admits it as a router; a caller reaches it
// only by an explicit /p2p-circuit address. Returns its address and the
// count of circuits the callers let go of, by resetting or closing them.
func startStalledRelay(t *testing.T) (string, *atomic.Int32) {
t.Helper()
h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
if err != nil {
t.Fatalf("failed to create the stalled relay: %v", err)
}
t.Cleanup(func() { _ = h.Close() })
released := new(atomic.Int32)
h.SetStreamHandler(circuitproto.ProtoIDv2Hop, func(s network.Stream) {
defer func() { _ = s.Close() }()
var msg circuitpb.HopMessage
if err := circuitutil.NewDelimitedReader(s, 4096).ReadMsg(&msg); err != nil || msg.GetType() != circuitpb.HopMessage_CONNECT {
return
}
ok := circuitpb.HopMessage{Type: circuitpb.HopMessage_STATUS.Enum(), Status: circuitpb.Status_OK.Enum()}
if err := circuitutil.NewDelimitedWriter(s).WriteMsg(&ok); err != nil {
return
}
// The caller's handshake bytes arrive and nothing answers them; the
// read ends when the caller resets or closes the circuit.
_, _ = io.Copy(io.Discard, s)
released.Add(1)
})
return h.Addrs()[0].String() + "/p2p/" + h.ID().String(), released
}
// pickDirectAddr is the member's loopback TCP address with its peer ID.
func pickDirectAddr(t *testing.T, addrs []string, peerID string) string {
t.Helper()
for _, a := range addrs {
if strings.HasPrefix(a, "/ip4/127.0.0.1/tcp/") {
if strings.Contains(a, "/p2p/") {
return a
}
return a + "/p2p/" + peerID
}
}
t.Fatalf("no loopback TCP address among %v", addrs)
return ""
}
func contains(values []string, want string) bool {
for _, v := range values {
if v == want {
return true
}
}
return false
}