Skip to content

Agent Mesh

controlplane: a refused /sts/token names the origin peer too; actor_node for /oauth/* Biscuits

Ignacio Janeiro95300a5c182b
diff --git a/api/datalog.go b/api/datalog.go
index 6a153b86..32989f5b 100644
--- a/api/datalog.go
+++ b/api/datalog.go
@@ -79,10 +79,13 @@ const (
// Example Datalog: check if client_peer_id($id), connection_peer_id($id)
FactClientPeerID = "client_peer_id"
- // FactActorNode names the origin agentmesh-node PeerID that exchanged a delegated
- // subject credential at POST /token/exchange, for audit logs and for ext_proc
- // attributes["mesh"].actor_node. It grants nothing: no baseline rule and no
- // policy binding reads it.
+ // FactActorNode names the origin agentmesh-node PeerID of a delegated Biscuit:
+ // the node that exchanged the subject credential at POST /token/exchange. For
+ // a Biscuit the control plane mints at /oauth/*, it is the node whose
+ // credential came with the request, or else the actor_peer_id the client
+ // named (checked only for admission) or the control plane's own peer ID. It
+ // is for audit logs and for ext_proc attributes["mesh"].actor_node. It grants
+ // nothing: no baseline rule and no policy binding reads it.
FactActorNode = "actor_node"
// FactGroup defines the group claim extracted from the OIDC token.
diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go
index 1b38116e..35fe26d3 100644
--- a/internal/controlplane/sts.go
+++ b/internal/controlplane/sts.go
@@ -1039,11 +1039,16 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) {
claims, status, err := s.authorizeBiscuitForEgress(r.Context(), req.Biscuit, req.Destination)
if err != nil {
+ clientPeerID := ""
+ if claims != nil {
+ clientPeerID = claims.ClientPeerID
+ }
logger.Infow("Border Crossing",
"direction", "outbound",
"endpoint", "/sts/token",
"decision", "deny",
"actor_node", nodeRecord.PeerID,
+ "client_peer_id", clientPeerID,
"audience", audience,
"destination", req.Destination,
"reason", err.Error(),
@@ -1170,6 +1175,10 @@ func mustParseSTSRule(s string) biscuit.Rule {
return r
}
+// authorizeBiscuitForEgress verifies the caller's Biscuit and its tar_block chain
+// against egress://<destination>. Once the Biscuit has verified, it returns the
+// claims even when it refuses the request, so the caller's Border Crossing line
+// can name the origin peer the Biscuit is bound to.
func (s *Server) authorizeBiscuitForEgress(ctx context.Context, rawBiscuit []byte, destination string) (*identity.VerifiedBiscuitClaims, int, error) {
trustedKeys, err := s.store.GetAllValidPublicKeys(ctx)
if err != nil {
@@ -1180,22 +1189,22 @@ func (s *Server) authorizeBiscuitForEgress(ctx context.Context, rawBiscuit []byt
return nil, http.StatusForbidden, fmt.Errorf("invalid caller biscuit: %w", err)
}
if status, revErr := s.checkBiscuitRevocationAndBans(ctx, claims); revErr != nil {
- return nil, status, revErr
+ return claims, status, revErr
}
if claims.ClientPeerID == "" {
- return nil, http.StatusForbidden, errors.New("caller biscuit lacks client_peer_id")
+ return claims, http.StatusForbidden, errors.New("caller biscuit lacks client_peer_id")
}
clientPeer, err := peer.Decode(claims.ClientPeerID)
if err != nil {
- return nil, http.StatusForbidden, fmt.Errorf("invalid client_peer_id in biscuit: %w", err)
+ return claims, http.StatusForbidden, fmt.Errorf("invalid client_peer_id in biscuit: %w", err)
}
if err := identity.RequireAuthorityRequestBinding(claims.Biscuit, clientPeer); err != nil {
- return nil, http.StatusForbidden, err
+ return claims, http.StatusForbidden, err
}
authorizer, err := claims.Biscuit.Authorizer(claims.VerifyingKey, identity.AuthorizerOptions(s.config.BiscuitTimeout)...)
if err != nil {
- return nil, http.StatusForbidden, err
+ return claims, http.StatusForbidden, err
}
authorizer.AddFact(biscuit.Fact{
Predicate: biscuit.Predicate{
@@ -1232,11 +1241,11 @@ func (s *Server) authorizeBiscuitForEgress(ctx context.Context, rawBiscuit []byt
roles, bindings, err := s.store.GetMeshPolicy(ctx)
if err != nil && err != storage.ErrNotFound {
- return nil, http.StatusInternalServerError, fmt.Errorf("failed to load mesh policy: %w", err)
+ return claims, http.StatusInternalServerError, fmt.Errorf("failed to load mesh policy: %w", err)
}
egressDests, err := s.store.GetEgressDestinations(ctx)
if err != nil && err != storage.ErrNotFound {
- return nil, http.StatusInternalServerError, fmt.Errorf("failed to load egress destinations: %w", err)
+ return claims, http.StatusInternalServerError, fmt.Errorf("failed to load egress destinations: %w", err)
}
policyRules, _ := api.BuildPolicyRules(roles, bindings)
policyRules = append(policyRules, api.BuildEgressServingRules(egressDests)...)
@@ -1245,11 +1254,11 @@ func (s *Server) authorizeBiscuitForEgress(ctx context.Context, rawBiscuit []byt
}
if err := authorizer.Authorize(); err != nil {
- return nil, http.StatusForbidden, fmt.Errorf("standing policy denied egress://%s: %w", destination, err)
+ return claims, http.StatusForbidden, fmt.Errorf("standing policy denied egress://%s: %w", destination, err)
}
if err := api.EvaluateTaskRulesForDestination(claims.TaskRules, "egress", destination, time.Now()); err != nil {
- return nil, http.StatusForbidden, err
+ return claims, http.StatusForbidden, err
}
// Collect any roles derived via Datalog policy bindings so mesh_roles is complete.
diff --git a/internal/controlplane/sts_test.go b/internal/controlplane/sts_test.go
index 3ff020de..783f449f 100644
--- a/internal/controlplane/sts_test.go
+++ b/internal/controlplane/sts_test.go
@@ -15,6 +15,7 @@
package controlplane
import (
+ "bufio"
"bytes"
"context"
"crypto/ecdsa"
@@ -40,6 +41,7 @@ import (
"github.com/google/agentmesh/internal/identity"
"github.com/google/agentmesh/internal/node"
"github.com/google/agentmesh/internal/storage"
+ golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/proto"
@@ -1012,3 +1014,117 @@ func TestRevokeBiscuitIDPrunesExpiredEntries(t *testing.T) {
t.Error("live revocation missing")
}
}
+
+// TestSTSTokenBorderCrossingNamesTheOriginPeer checks that both Border Crossing
+// lines at /sts/token name the origin peer of the caller's Biscuit next to the
+// egress node, for a Task Biscuit attenuated from a Member Biscuit: the case
+// that carries client_peer_id but no actor_node.
+func TestSTSTokenBorderCrossingNamesTheOriginPeer(t *testing.T) {
+ issuer, mintOIDC := startCustomMockOIDC(t)
+ srv, store, baseURL := setupTestServer(t, issuer)
+ defer func() { _ = srv.Close() }()
+
+ roles := []*api.PolicyRole{
+ {Name: api.RoleNode, AllowedServices: []string{"egress://bigquery.googleapis.com"}, AllowedTargets: []string{"*"}},
+ }
+ bindings := []*api.PolicyBinding{
+ {Role: api.RoleNode, Members: []string{api.SystemAuthenticated}},
+ }
+ if err := store.SavePolicyDocument(context.Background(), roles, bindings, nil); err != nil {
+ t.Fatalf("SavePolicyDocument: %v", err)
+ }
+
+ _, originPeerID, originBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintOIDC)
+ egressPriv, egressPeerID, egressBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintOIDC)
+ taskBiscuit, err := identity.AttenuateBiscuit(originBiscuit, &api.TaskAuthorizationRule{
+ Name: "bq-export",
+ Rules: []*api.TaskRule{{AllowedServices: []string{"egress://bigquery.googleapis.com"}}},
+ })
+ if err != nil {
+ t.Fatalf("attenuate the Member Biscuit: %v", err)
+ }
+
+ const subsystem = "agentmesh-control-plane"
+ prevLevel, err := golog.SubsystemLevelName(subsystem)
+ if err != nil {
+ t.Fatalf("read the %s log level: %v", subsystem, err)
+ }
+ if err := golog.SetLogLevel(subsystem, "info"); err != nil {
+ t.Fatalf("set the %s log level: %v", subsystem, err)
+ }
+ t.Cleanup(func() { _ = golog.SetLogLevel(subsystem, prevLevel) })
+
+ // The pipe sees every logger, including other tests running in parallel,
+ // so the reader keeps only this egress node's /sts/token lines and drops
+ // the rest instead of letting the pipe block a writer.
+ pipe := golog.NewPipeReader()
+ t.Cleanup(func() { _ = pipe.Close() })
+ crossings := make(chan map[string]any, 4)
+ go func() {
+ scanner := bufio.NewScanner(pipe)
+ scanner.Buffer(make([]byte, 0, 64*1024), 1<<20)
+ for scanner.Scan() {
+ var entry map[string]any
+ if json.Unmarshal(scanner.Bytes(), &entry) != nil {
+ continue
+ }
+ if entry["msg"] == "Border Crossing" && entry["endpoint"] == "/sts/token" && entry["actor_node"] == egressPeerID.String() {
+ crossings <- entry
+ }
+ }
+ }()
+
+ postSTS := func(destination, audience string) int {
+ t.Helper()
+ ts := time.Now().UnixMilli()
+ sig, err := egressPriv.Sign(api.STSTokenChallenge(egressPeerID.String(), ts))
+ if err != nil {
+ t.Fatalf("sign the STS challenge: %v", err)
+ }
+ reqBytes, err := proto.Marshal(&api.STSTokenRequest{
+ Biscuit: taskBiscuit,
+ Destination: destination,
+ Audience: audience,
+ ChallengeUnixMs: ts,
+ ChallengeSignature: sig,
+ })
+ if err != nil {
+ t.Fatalf("marshal STSTokenRequest: %v", err)
+ }
+ req, err := http.NewRequest(http.MethodPost, baseURL+"/sts/token", bytes.NewReader(reqBytes))
+ if err != nil {
+ t.Fatalf("build POST /sts/token: %v", err)
+ }
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(egressBiscuit))
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("POST /sts/token for %s: %v", destination, err)
+ }
+ _ = resp.Body.Close()
+ return resp.StatusCode
+ }
+
+ if status := postSTS("bigquery.googleapis.com", "//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/agentmesh/providers/cp"); status != http.StatusOK {
+ t.Fatalf("POST /sts/token for bigquery.googleapis.com returned %d, want 200", status)
+ }
+ if status := postSTS("api.github.com", "https://api.github.com"); status != http.StatusForbidden {
+ t.Fatalf("POST /sts/token for api.github.com returned %d, want 403", status)
+ }
+
+ seen := map[string]bool{}
+ for len(seen) < 2 {
+ select {
+ case entry := <-crossings:
+ decision, _ := entry["decision"].(string)
+ if entry["client_peer_id"] != originPeerID.String() {
+ t.Fatalf("Border Crossing decision=%s logged client_peer_id = %v, want the origin peer %s", decision, entry["client_peer_id"], originPeerID)
+ }
+ seen[decision] = true
+ case <-time.After(5 * time.Second):
+ t.Fatalf("got Border Crossing lines for decisions %v, want allow and deny", seen)
+ }
+ }
+ if !seen["allow"] || !seen["deny"] {
+ t.Fatalf("got Border Crossing lines for decisions %v, want allow and deny", seen)
+ }
+}
diff --git a/site/content/docs/contributing/security-architecture.md b/site/content/docs/contributing/security-architecture.md
index c41e7240..8e1bad3f 100644
--- a/site/content/docs/contributing/security-architecture.md
+++ b/site/content/docs/contributing/security-architecture.md
@@ -307,7 +307,11 @@ minted statelessly on the control plane with zero database writes:
node's libp2p connection (`check if client_peer_id($id), connection_peer_id($id)`),
so an exfiltrated token cannot be replayed from another peer.
- `actor_node("<origin_node_peer_id>")` records the acting node for audit logs
- and `ext_proc` attributes. No policy rule grants permissions on `actor_node`.
+ and `ext_proc` attributes: the node that called `POST /token/exchange`. For
+ `POST /oauth/token` it is the node whose credential came with the request, or
+ else a claimed peer: the `actor_peer_id` the client named (checked only for
+ admission) or the control plane's own peer ID. No policy rule grants
+ permissions on `actor_node`.
- `user(...)`, `email(...)`, `group(...)`, `idp_role(...)`, and `role(...)` are
resolved strictly from the caller's `subject_token`.
- No `node()` fact is minted, which prevents `node:<peer_id>` policy bindings on
@@ -338,7 +342,7 @@ as a standard OIDC issuer:
| :--- | :--- |
| **Signing key** | **ES256** (P-256 ECDSA), rotated with overlap and `kid` (`internal/controlplane/sts.go`). Google Cloud and AWS STS accept RS256 and ES256, not Ed25519; Ed25519 remains the Biscuit authority key. |
| **Discovery & JWKS** | `GET /.well-known/openid-configuration` and `GET /jwks` on the control plane. For air-gapped or private control planes, the JWKS can be uploaded directly to the cloud workload identity pool. |
-| **Mint endpoint** | `POST /sts/token` (protobuf over HTTP): accepts the caller's Biscuit and `destination` name, verifies the Biscuit and the full `tar_block` chain against `egress://<destination>`, writes a `Border Crossing` audit log, and returns a short-lived ES256 JWT. |
+| **Mint endpoint** | `POST /sts/token` (protobuf over HTTP): accepts the caller's Biscuit and `destination` name, verifies the Biscuit and the full `tar_block` chain against `egress://<destination>`, writes a `Border Crossing` audit log, and returns a short-lived ES256 JWT. In the `/sts/token` lines, `actor_node` is the egress node that called it and `client_peer_id` is the origin peer the caller's Biscuit is bound to, on allow and on deny once the Biscuit has verified; in the `/token/exchange` lines, `actor_node` is the exchanging node, which becomes that Biscuit's `client_peer_id`. |
| **JWT claims** | `iss` = control plane URL; `sub` = caller principal (`email` or `issuer#subject`); `act` = `{"sub": "<egress_node_peer_id>"}`; `aud` = destination audience from `EgressDestination` (one audience per destination); `mesh_roles` = authority-block roles (mapped to `google.groups` for `principalSet` bindings); `mesh_task` = innermost `TaskAuthorizationRule.name` (audit-only). |
| **TAR narrows, never selects** | Nothing in a holder-appended `tar_block` can select a broker, role ARN, service account, or audience; every claim that cloud IAM binds on (`sub`, `mesh_roles`, `act.sub`, `aud`) comes from Block 0 or the control plane's `EgressDestination` policy. |
| **Caching** | The egress node caches minted border JWTs and upstream cloud tokens by Biscuit SHA-256 digest, keeping the control plane off the per-request hot path. |