Skip to content

Agent Mesh

node: serve /mesh/ingress only to routers and inventory every protocol's gate

Antonio Ojeadb5292f1f032
diff --git a/internal/node/auth_gate.go b/internal/node/auth_gate.go
index 11a54d35..0872d957 100644
--- a/internal/node/auth_gate.go
+++ b/internal/node/auth_gate.go
@@ -30,6 +30,10 @@ import (
// never authenticated while the stream waited.
const streamScopeUnauthenticated = "unauthenticated"
+// streamScopeNotRouter labels a stream the gate refused on a router-only
+// protocol: its peer authenticated, but not as mesh:role:router.
+const streamScopeNotRouter = "not_router"
+
// authWait bounds how long a stream waits for its peer's handshake: a
// router's auth deadline for the connection, so a peer that never
// authenticates is cut off with it.
@@ -66,12 +70,40 @@ func (h *meshHost) NewStream(ctx context.Context, p peer.ID, pids ...protocol.ID
return h.Host.NewStream(ctx, p, pids...)
}
+// routerHost is meshHost for a protocol only routers speak to a member, such
+// as external HTTP ingress forwarding. A handler installed through it is
+// served only to a peer that authenticated as mesh:role:router: a router this
+// node attached to, or one that completed the handshake with this node and
+// carried the role in its credential. Any other authenticated peer is refused
+// before the handler reads a byte, so a member cannot reach the handler's
+// parser at all; the handler's own credential check then only confirms what
+// the gate decided.
+type routerHost struct {
+ meshHost
+}
+
+func (h *routerHost) SetStreamHandler(pid protocol.ID, handler network.StreamHandler) {
+ h.Host.SetStreamHandler(pid, h.n.requireRouter(handler))
+}
+
+func (h *routerHost) SetStreamHandlerMatch(pid protocol.ID, match func(protocol.ID) bool, handler network.StreamHandler) {
+ h.Host.SetStreamHandlerMatch(pid, match, h.n.requireRouter(handler))
+}
+
// isPeerAuthenticated reports whether p is a router this node authenticated
// with on an open session, or a peer whose handshake with this node holds.
func (n *AgentMeshNode) isPeerAuthenticated(p peer.ID) bool {
return n.isAuthenticatedAndConnected(p) || n.isAdmitted(p)
}
+// isAuthenticatedRouter reports whether p authenticated as a router: on a
+// session this node opened to it after verifying role("mesh:role:router"),
+// or by completing the handshake with this node with a credential that
+// carried the role.
+func (n *AgentMeshNode) isAuthenticatedRouter(p peer.ID) bool {
+ return n.isAuthenticatedAndConnected(p) || n.isAdmittedRouter(p)
+}
+
// ensurePeerAuthenticated returns once p is authenticated: for an unattached
// router the control plane lists, by running the handshake (one at a time per
// router). A router this node is already attached to whose session dropped is
@@ -108,6 +140,23 @@ func (n *AgentMeshNode) requireAuthenticated(next network.StreamHandler) network
}
}
+// requireRouter wraps a stream handler so it runs only for a peer that
+// authenticated as a router. The peer gets the same wait for its handshake
+// as requireAuthenticated; a peer that then holds any other role is refused
+// at once, logged, and counted under streamScopeNotRouter.
+func (n *AgentMeshNode) requireRouter(next network.StreamHandler) network.StreamHandler {
+ return n.requireAuthenticated(func(s network.Stream) {
+ p := s.Conn().RemotePeer()
+ if !n.isAuthenticatedRouter(p) {
+ streamsRefusedTotal.WithLabelValues(streamScopeNotRouter, string(s.Protocol())).Inc()
+ logger.Warnf("[AuthZ] Refusing %s stream from %s: peer is authenticated but not a router", s.Protocol(), p)
+ _ = s.Reset()
+ return
+ }
+ next(s)
+ })
+}
+
// isCatalogRouter reports whether p is a router the control plane lists.
func (n *AgentMeshNode) isCatalogRouter(p peer.ID) bool {
n.mu.Lock()
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.go
index eea3dd64..aaaf3a94 100644
--- a/internal/node/controlplane_sync.go
+++ b/internal/node/controlplane_sync.go
@@ -293,7 +293,7 @@ func (n *AgentMeshNode) banPeer(p peer.ID, bannedAtMs int64) {
if n.revokedPeers != nil {
n.revokedPeers.Add(p.String(), bannedAtMs)
}
- n.authPeers.Delete(p)
+ n.forgetAdmission(p)
if n.Host != nil {
_ = n.Host.Network().ClosePeer(p)
}
diff --git a/internal/node/ingress.go b/internal/node/ingress.go
index dffe2bc4..c0bdab10 100644
--- a/internal/node/ingress.go
+++ b/internal/node/ingress.go
@@ -112,9 +112,12 @@ func (n *AgentMeshNode) logIngressURLs(info *api.ServiceInfo) {
// StartExternalIngressServer starts the HTTP-over-libp2p server on
// api.IngressProtocolID (/mesh/ingress/1.0.0) that receives external HTTP
-// requests forwarded by enrolled routers.
+// requests forwarded by enrolled routers. The listener is built on routerHost,
+// so a stream from any peer that did not authenticate as a router is reset at
+// the gate before the HTTP server reads it; the handler's X-Mesh-Biscuit check
+// below is the second, independent layer.
func (n *AgentMeshNode) StartExternalIngressServer(ctx context.Context) error {
- gated := &meshHost{Host: n.Host, n: n}
+ gated := &routerHost{meshHost{Host: n.Host, n: n}}
listener, err := gostream.Listen(gated, api.IngressProtocolID)
if err != nil {
return err
diff --git a/internal/node/node.go b/internal/node/node.go
index a706c9fa..d1e1a61c 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -131,13 +131,43 @@ func (a *nodeRelayACL) AllowConnect(src peer.ID, srcAddr multiaddr.Multiaddr, de
// has not lapsed since. The handshake only proves the token was valid at that
// instant, so without this the relay ACL would honour an admission forever.
func (n *AgentMeshNode) isAdmitted(p peer.ID) bool {
- v, ok := n.authPeers.Load(p)
+ return admissionHolds(&n.authPeers, p)
+}
+
+// isAdmittedRouter reports whether a peer completed the auth handshake with a
+// credential carrying role("mesh:role:router") and that admission holds. It
+// requires isAdmitted too, so dropping a peer from authPeers (a ban, an
+// eviction) withdraws its router standing with it.
+func (n *AgentMeshNode) isAdmittedRouter(p peer.ID) bool {
+ return n.isAdmitted(p) && admissionHolds(&n.authRouters, p)
+}
+
+// admitPeer records a completed handshake: every admitted peer in authPeers,
+// and a router additionally in authRouters. A peer re-admitted without the
+// role loses it.
+func (n *AgentMeshNode) admitPeer(p peer.ID, expiry time.Time, router bool) {
+ n.authPeers.Store(p, expiry)
+ if router {
+ n.authRouters.Store(p, expiry)
+ } else {
+ n.authRouters.Delete(p)
+ }
+}
+
+// forgetAdmission drops every record of p's handshake.
+func (n *AgentMeshNode) forgetAdmission(p peer.ID) {
+ n.authPeers.Delete(p)
+ n.authRouters.Delete(p)
+}
+
+func admissionHolds(m *sync.Map, p peer.ID) bool {
+ v, ok := m.Load(p)
if !ok {
return false
}
expiry, ok := v.(time.Time)
if !ok || !time.Now().Before(expiry) {
- n.authPeers.Delete(p)
+ m.Delete(p)
return false
}
return true
@@ -183,12 +213,17 @@ type AgentMeshNode struct {
delegatedBiscuitCache *lru.Cache[string, *api.TokenExchangeResponse]
stsTokenCache *lru.Cache[string, *api.STSTokenResponse]
peerLabelGate *lru.Cache[string, time.Time]
- authPeers sync.Map
- trustedKeys []TrustedKey
- keysMu sync.RWMutex
- MeshPolicyRules []biscuit.Rule
- meshPolicyTexts []string
- MeshPolicyMu sync.RWMutex
+ // authPeers is every peer that completed the handshake with this node,
+ // keyed by peer.ID, with the instant its admission lapses. authRouters
+ // holds the subset whose credential carried mesh:role:router, with the
+ // same instant, for the protocols only a router may open (routerHost).
+ authPeers sync.Map
+ authRouters sync.Map
+ trustedKeys []TrustedKey
+ keysMu sync.RWMutex
+ MeshPolicyRules []biscuit.Rule
+ meshPolicyTexts []string
+ MeshPolicyMu sync.RWMutex
// pendingEgress holds assignments that arrived before Start created the
// service registry (SyncControlPlane runs first); Start applies them.
pendingEgress []*api.EgressDestination
@@ -2073,15 +2108,24 @@ func (n *AgentMeshNode) HandleAuthHandshake(s network.Stream) {
}
// One verifier call enforces signature, expiry and the authority-block peer
- // binding, and reports when the admission lapses.
- expiry, err := identity.VerifyBiscuitAndGetExpiry(exchange.Biscuit, remotePeer, n.getTrustedPublicKeys(), n.BiscuitTimeout)
+ // binding, and reports when the admission lapses, together with the key
+ // that verified so the role can be read under that same key.
+ b, verifyingKey, expiry, err := identity.VerifyBiscuitAdmission(exchange.Biscuit, remotePeer, n.getTrustedPublicKeys(), n.BiscuitTimeout)
if err != nil {
logger.Warnf("[AuthN] Authorization failed for %s: %v", remotePeer, err)
return
}
- n.authPeers.Store(remotePeer, expiry)
- logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)
+ // A router that connects to this node (to forward external ingress, for
+ // one) is admitted like any peer, and additionally remembered as a router
+ // so routerHost protocols can serve it and refuse everyone else.
+ isRouter := identity.RequireRole(b, verifyingKey, api.RoleRouter, n.BiscuitTimeout) == nil
+ n.admitPeer(remotePeer, expiry, isRouter)
+ if isRouter {
+ logger.Infof("[AuthN] Successfully authenticated peer %s as router", remotePeer)
+ } else {
+ logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)
+ }
// Mutual response with our identity, mirroring the router handler, so
// peers can verify this node's attested facts (e.g. region).
diff --git a/internal/node/protocol_inventory_test.go b/internal/node/protocol_inventory_test.go
new file mode 100644
index 00000000..dfa68322
--- /dev/null
+++ b/internal/node/protocol_inventory_test.go
@@ -0,0 +1,279 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "bufio"
+ "context"
+ "net/http"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p"
+ pubsub "github.com/libp2p/go-libp2p-pubsub"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+ "github.com/libp2p/go-libp2p/p2p/protocol/autonatv2"
+ circuit "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto"
+ "github.com/libp2p/go-libp2p/p2p/protocol/identify"
+ "github.com/libp2p/go-libp2p/p2p/protocol/ping"
+ "github.com/libp2p/go-msgio"
+ "google.golang.org/protobuf/proto"
+
+ "github.com/google/agentmesh/api"
+)
+
+// Every protocol a started node registers on its libp2p host belongs to
+// exactly one of these sets. A protocol in none of them fails
+// TestNodeProtocolInventoryIsClassified: whoever adds a stream handler has to
+// say here how an unauthenticated peer is kept out of it, and the gated set
+// is then checked live, not by reading the code.
+var (
+ // handshakeProtocols are the only protocols open to a peer that has not
+ // authenticated: what the handshake itself needs (identify, the mesh
+ // handshake) and libp2p's own reachability plumbing. Nothing here reads
+ // mesh data.
+ handshakeProtocols = []protocol.ID{
+ api.AuthProtocolID,
+ identify.ID,
+ identify.IDPush,
+ ping.ID,
+ autonatv2.DialProtocol,
+ autonatv2.DialBackProtocol,
+ circuit.ProtoIDv2Stop,
+ }
+
+ // credentialProtocols authenticate inside the stream: the first frame or
+ // request carries a Biscuit, and the handler refuses the stream itself
+ // when it does not verify. Each has its own test of that refusal.
+ credentialProtocols = []protocol.ID{
+ api.MCPProtocolID,
+ api.GoAwayProtocolID,
+ "/libp2p-http",
+ }
+
+ // gatedProtocols are served through meshHost or routerHost: a stream from
+ // a peer that has not completed the handshake is reset and counted before
+ // the handler sees it. The test below opens each one from a stranger and
+ // checks exactly that.
+ gatedProtocols = []protocol.ID{
+ api.IngressProtocolID,
+ pubsub.FloodSubID,
+ api.DHTProtocolID,
+ }
+)
+
+func TestNodeProtocolInventoryIsClassified(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+
+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ node := r.startNode(t, ctx, r.routerAddrs)
+
+ registered := node.Host.Mux().Protocols()
+ slices.Sort(registered)
+ var unclassified []protocol.ID
+ var gated []protocol.ID
+ for _, p := range registered {
+ switch {
+ case slices.Contains(handshakeProtocols, p), slices.Contains(credentialProtocols, p):
+ case slices.Contains(gatedProtocols, p):
+ gated = append(gated, p)
+ default:
+ unclassified = append(unclassified, p)
+ }
+ }
+ if len(unclassified) > 0 {
+ t.Fatalf("node registers protocols that no set in protocol_inventory_test.go names: %v\n"+
+ "Add each to handshakeProtocols (open before the handshake, nothing else may be), "+
+ "credentialProtocols (the handler verifies a Biscuit in the stream) or "+
+ "gatedProtocols (registered through meshHost/routerHost).", unclassified)
+ }
+ if !slices.Contains(gated, api.IngressProtocolID) {
+ t.Fatalf("%s is not registered on a started node; registered: %v", api.IngressProtocolID, registered)
+ }
+
+ stranger, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = stranger.Close() })
+ if err := stranger.Connect(ctx, peer.AddrInfo{ID: node.Host.ID(), Addrs: node.Host.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+
+ for _, p := range gated {
+ t.Run(string(p), func(t *testing.T) {
+ before := counterValue(t, streamsRefusedTotal.WithLabelValues(streamScopeUnauthenticated, string(p)))
+
+ s, err := stranger.NewStream(ctx, node.Host.ID(), p)
+ if err != nil {
+ t.Fatalf("open %s: %v", p, err)
+ }
+ defer func() { _ = s.Reset() }()
+ if _, err := s.Write([]byte("GET / HTTP/1.1\r\nHost: x\r\n\r\n")); err != nil {
+ t.Fatalf("write: %v", err)
+ }
+ // The gate holds the stream for the handshake; nothing may come
+ // back meanwhile, whatever the handler behind it would answer.
+ buf := make([]byte, 64)
+ _ = s.SetReadDeadline(time.Now().Add(300 * time.Millisecond))
+ if n, err := s.Read(buf); err == nil {
+ t.Fatalf("%s answered %q to a peer that never authenticated", p, buf[:n])
+ }
+
+ // Dropping the connection ends the wait: the gate refuses the
+ // stream and counts it under this protocol.
+ if err := stranger.Network().ClosePeer(node.Host.ID()); err != nil {
+ t.Fatal(err)
+ }
+ waitForCounter(t, func() float64 {
+ return counterValue(t, streamsRefusedTotal.WithLabelValues(streamScopeUnauthenticated, string(p)))
+ }, before+1, 5*time.Second)
+
+ if err := stranger.Connect(ctx, peer.AddrInfo{ID: node.Host.ID(), Addrs: node.Host.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+// /mesh/ingress/1.0.0 is served only to a router. A member that completed
+// the handshake with the node is still refused at the gate, before the HTTP
+// server behind it reads a byte, and counted; a peer admitted as a router
+// reaches the handler.
+func TestExternalIngressProtocolServesOnlyRouters(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+
+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ node := r.startNode(t, ctx, r.routerAddrs)
+
+ request := "GET /a2a/assistant/.well-known/agent-card.json HTTP/1.1\r\nHost: mesh\r\n\r\n"
+
+ t.Run("member is refused at the gate", func(t *testing.T) {
+ member := newAdmittedPeer(t, ctx, node, r, api.RoleNode)
+ if !node.isAdmitted(member.ID()) || node.isAdmittedRouter(member.ID()) {
+ t.Fatalf("member admitted=%v router=%v, want true/false", node.isAdmitted(member.ID()), node.isAdmittedRouter(member.ID()))
+ }
+ before := counterValue(t, streamsRefusedTotal.WithLabelValues(streamScopeNotRouter, string(api.IngressProtocolID)))
+
+ s, err := member.NewStream(ctx, node.Host.ID(), api.IngressProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = s.Reset() }()
+ if _, err := s.Write([]byte(request)); err != nil {
+ t.Fatal(err)
+ }
+ buf := make([]byte, 64)
+ _ = s.SetReadDeadline(time.Now().Add(5 * time.Second))
+ if n, err := s.Read(buf); err == nil {
+ t.Fatalf("ingress protocol answered a member %q; the gate must reset the stream", buf[:n])
+ }
+ waitForCounter(t, func() float64 {
+ return counterValue(t, streamsRefusedTotal.WithLabelValues(streamScopeNotRouter, string(api.IngressProtocolID)))
+ }, before+1, 5*time.Second)
+ })
+
+ t.Run("router reaches the handler", func(t *testing.T) {
+ router := newAdmittedPeer(t, ctx, node, r, api.RoleRouter)
+ if !node.isAdmittedRouter(router.ID()) {
+ t.Fatal("peer admitted with the router role is not recorded as a router")
+ }
+
+ s, err := router.NewStream(ctx, node.Host.ID(), api.IngressProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = s.Close() }()
+ if _, err := s.Write([]byte(request)); err != nil {
+ t.Fatal(err)
+ }
+ _ = s.SetReadDeadline(time.Now().Add(5 * time.Second))
+ resp, err := http.ReadResponse(bufio.NewReader(s), nil)
+ if err != nil {
+ t.Fatalf("read HTTP response from ingress handler: %v", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ // The handler ran: with no X-Mesh-Biscuit it refuses the request
+ // itself, which is the second layer behind the gate.
+ if resp.StatusCode != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d from the handler's own credential check", resp.StatusCode, http.StatusUnauthorized)
+ }
+ })
+
+ t.Run("ban withdraws router standing", func(t *testing.T) {
+ router := newAdmittedPeer(t, ctx, node, r, api.RoleRouter)
+ node.banPeer(router.ID(), time.Now().UnixMilli())
+ if node.isAdmitted(router.ID()) || node.isAdmittedRouter(router.ID()) {
+ t.Fatal("banned peer still admitted")
+ }
+ })
+}
+
+// newAdmittedPeer is a libp2p host that connects to node and completes the
+// mesh handshake with a credential the fake router's control plane key signed
+// for the given role.
+func newAdmittedPeer(t *testing.T, ctx context.Context, node *AgentMeshNode, r *fakeRouter, role string) host.Host {
+ t.Helper()
+ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = h.Close() })
+ if err := h.Connect(ctx, peer.AddrInfo{ID: node.Host.ID(), Addrs: node.Host.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ s, err := h.NewStream(ctx, node.Host.ID(), api.AuthProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = s.Close() }()
+ frame, err := proto.Marshal(&api.AuthFrame{Biscuit: r.mint(h.ID().String(), role)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := msgio.NewVarintWriter(s).WriteMsg(frame); err != nil {
+ t.Fatal(err)
+ }
+ reply, err := msgio.NewVarintReaderSize(s, 1024*64).ReadMsg()
+ if err != nil {
+ t.Fatalf("handshake reply: %v", err)
+ }
+ var resp api.AuthResponse
+ if err := proto.Unmarshal(reply, &resp); err != nil || !resp.Success {
+ t.Fatalf("handshake refused: %v %s", err, resp.GetError())
+ }
+ return h
+}
+
+// waitForCounter polls get until it reaches want, and reports the last value
+// it saw if the deadline passes first.
+func waitForCounter(t *testing.T, get func() float64, want float64, within time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(within)
+ var got float64
+ for time.Now().Before(deadline) {
+ got = get()
+ if got >= want {
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatalf("counter = %v, want %v", got, want)
+}
diff --git a/site/content/docs/guides/external-ingress.md b/site/content/docs/guides/external-ingress.md
index 83eb4344..81385402 100644
--- a/site/content/docs/guides/external-ingress.md
+++ b/site/content/docs/guides/external-ingress.md
@@ -150,7 +150,11 @@ to announcing the service internally so mesh peers can still reach it.
node. Once admitted, both the router and the serving node (defense in depth
over `/mesh/ingress/1.0.0`) enforce the full intersection of mesh Datalog
RBAC, `TaskAuthorizationRule`, `ServiceIngress.allow`, and local
- `agentmesh-node.yaml` `attenuation`.
+ `agentmesh-node.yaml` `attenuation`. The node serves `/mesh/ingress/1.0.0`
+ only to a peer that authenticated as `mesh:role:router`: a stream from any
+ other peer, authenticated or not, is reset at the libp2p gate before the
+ HTTP server reads it and counted in
+ `agentmesh_node_streams_refused_total{scope="not_router"}`.
4. The external `Authorization` header is stripped before reaching the local
backend (replaced only if `target_auth_path` is configured on the node).
5. For `a2a` services, unauthenticated `GET` requests to the service root or