Skip to content

Agent Mesh

Require PoP challenges on read/report endpoints and recover from clock skew via Date header

Antonio Ojea03020b2b1f4f
diff --git a/api/network.go b/api/network.go
index 97ce98a8..62505bb8 100644
--- a/api/network.go
+++ b/api/network.go
@@ -133,6 +133,40 @@ func STSTokenChallenge(peerID string, ts int64) []byte {
return []byte("sam:sts-token:" + peerID + ":" + strconv.FormatInt(ts, 10))
}
+// PoliciesChallenge is the payload an enrolled peer signs with its identity
+// key at GET /policies in the HeaderChallengeTimestamp and
+// HeaderChallengeSignature headers.
+func PoliciesChallenge(peerID string, ts int64) []byte {
+ return []byte("sam:policies:" + peerID + ":" + strconv.FormatInt(ts, 10))
+}
+
+// EgressChallenge is the payload an enrolled peer signs with its identity key
+// at GET /egress in the HeaderChallengeTimestamp and HeaderChallengeSignature
+// headers.
+func EgressChallenge(peerID string, ts int64) []byte {
+ return []byte("sam:egress:" + peerID + ":" + strconv.FormatInt(ts, 10))
+}
+
+// RevocationsChallenge is the payload an enrolled peer signs with its identity
+// key at GET /revocations in the HeaderChallengeTimestamp and
+// HeaderChallengeSignature headers.
+func RevocationsChallenge(peerID string, ts int64) []byte {
+ return []byte("sam:revocations:" + peerID + ":" + strconv.FormatInt(ts, 10))
+}
+
+// NodesCatalogChallenge is the payload an enrolled node signs with its
+// identity key at POST /nodes/catalog in the HeaderChallengeTimestamp and
+// HeaderChallengeSignature headers.
+func NodesCatalogChallenge(peerID string, ts int64) []byte {
+ return []byte("sam:nodes-catalog:" + peerID + ":" + strconv.FormatInt(ts, 10))
+}
+
+// ErrStaleChallengeTimestampMessage is the stable substring returned in a 401
+// response body when a signed challenge timestamp is outside the control
+// plane's freshness window. Clients match this message to recompute
+// challenge_unix_ms from the response's Date header and retry once.
+const ErrStaleChallengeTimestampMessage = "stale or invalid challenge timestamp"
+
// ============================================================================
// OAuth 2.1 & RFC 8693 Token Exchange Constants
// ============================================================================
@@ -171,10 +205,11 @@ const (
HeaderSamBiscuit = "X-Sam-Biscuit"
// HeaderChallengeTimestamp and HeaderChallengeSignature carry the signed
- // freshness challenge on GET /enroll/status: unix milliseconds and an
- // unpadded base64url signature over EnrollStatusChallenge. Headers rather
- // than query parameters, so the signature never lands in access logs,
- // where it would be replayable for its freshness window.
+ // freshness challenge on GET /enroll/status, GET /policies, GET /egress,
+ // GET /revocations, and POST /nodes/catalog: unix milliseconds and an
+ // unpadded base64url signature over the endpoint's challenge payload.
+ // Headers rather than query parameters, so the signature never lands in
+ // access logs, where it would be replayable for its freshness window.
HeaderChallengeTimestamp = "X-Sam-Challenge-Ts"
HeaderChallengeSignature = "X-Sam-Challenge-Sig"
diff --git a/internal/bench/sts.go b/internal/bench/sts.go
index 0e0c4f36..bf95c8b8 100644
--- a/internal/bench/sts.go
+++ b/internal/bench/sts.go
@@ -136,34 +136,20 @@ func RunSTS(ctx context.Context, opts STSOptions) (*STSReport, error) {
}
// Phase 1: Uncached Control Plane POST /token/exchange
- cpClient := cpclient.New(env.cpURL, env.httpClient)
+ cpClient := cpclient.New(env.cpURL, env.httpClient).WithIdentity(env.nodePeerID.String(), env.nodePriv)
report.TokenExchangeUncached = runSTSPhase(ctx, opts.Concurrency, opts.Warmup, opts.Requests, &env.exchangeCalls, func(i int) error {
- nowMs := time.Now().UnixMilli()
- sig, err := env.nodePriv.Sign([]byte(api.TokenExchangeChallenge(env.nodePeerID.String(), nowMs)))
- if err != nil {
- return err
- }
- _, err = cpClient.ExchangeToken(ctx, env.nodeBiscuit, &api.TokenExchangeRequest{
- SubjectToken: baseJWT,
- ChallengeUnixMs: nowMs,
- ChallengeSignature: sig,
+ _, err := cpClient.ExchangeToken(ctx, env.nodeBiscuit, &api.TokenExchangeRequest{
+ SubjectToken: baseJWT,
})
return err
})
// Phase 2: Uncached Control Plane POST /sts/token
report.STSTokenUncached = runSTSPhase(ctx, opts.Concurrency, opts.Warmup, opts.Requests, &env.stsCalls, func(i int) error {
- nowMs := time.Now().UnixMilli()
- sig, err := env.nodePriv.Sign([]byte(api.STSTokenChallenge(env.nodePeerID.String(), nowMs)))
- if err != nil {
- return err
- }
- _, err = cpClient.MintSTSToken(ctx, env.nodeBiscuit, &api.STSTokenRequest{
- Biscuit: taskBiscuit,
- Destination: "bigquery.googleapis.com",
- Audience: "//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/sam/providers/cp",
- ChallengeUnixMs: nowMs,
- ChallengeSignature: sig,
+ _, err := cpClient.MintSTSToken(ctx, env.nodeBiscuit, &api.STSTokenRequest{
+ Biscuit: taskBiscuit,
+ Destination: "bigquery.googleapis.com",
+ Audience: "//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/sam/providers/cp",
})
return err
})
diff --git a/internal/controlplane/catalog.go b/internal/controlplane/catalog.go
index 53e5cb6b..50ee2115 100644
--- a/internal/controlplane/catalog.go
+++ b/internal/controlplane/catalog.go
@@ -15,15 +15,12 @@
package controlplane
import (
- "encoding/base64"
"fmt"
"io"
"net/http"
- "strings"
"time"
"github.com/google/sam/api"
- "github.com/google/sam/internal/identity"
"github.com/google/sam/internal/storage"
"github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/proto"
@@ -128,57 +125,22 @@ func (s *Server) dropCatalogEntry(peerID string) {
// participant or open a P2P connection to every enrolled node itself.
//
// The body is an api.NodeCatalogReport. The reporting peer is the one bound
-// in the presented Biscuit, so a node can only ever describe itself.
+// in the presented Biscuit, verified with a signed challenge in
+// HeaderChallengeTimestamp and HeaderChallengeSignature so a node can only
+// ever describe itself.
//
// This is a live-status cache, not authoritative state: a node that goes
// offline without ever reporting an empty catalog just leaves its last
// report in place until ReportedAt visibly goes stale or its enrollment
-// ends. It is admin-facing display data only and never feeds authorization,
-// which is also why a bare bearer Biscuit (no signed challenge, unlike
-// /refresh) is accepted here: a replayed token can only repaint a table.
+// ends.
func (s *Server) HandleNodeCatalog(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
- authHeader := r.Header.Get("Authorization")
- if !strings.HasPrefix(authHeader, "Bearer ") {
- http.Error(w, "Missing node Biscuit token in Authorization header", http.StatusUnauthorized)
- return
- }
- biscuitBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Bearer "))
- if err != nil {
- http.Error(w, "Malformed base64 token", http.StatusBadRequest)
- return
- }
-
- ctx := r.Context()
- trustedKeys, err := s.store.GetAllValidPublicKeys(ctx)
- if err != nil {
- logger.Errorf("Failed to retrieve valid signing keys: %v", err)
- http.Error(w, "Internal server error", http.StatusInternalServerError)
- return
- }
-
- peerID, err := identity.VerifyAndExtractPeerID(trustedKeys, biscuitBytes, s.config.BiscuitTimeout)
- if err != nil {
- logger.Warnw("Invalid biscuit presented to /nodes/catalog", "error", err)
- http.Error(w, "Invalid biscuit: "+err.Error(), http.StatusUnauthorized)
- return
- }
-
- nodeRecord, err := s.store.GetNode(ctx, peerID.String())
- if err == storage.ErrNotFound || (err == nil && nodeRecord == nil) {
- http.Error(w, "Node not enrolled", http.StatusUnauthorized)
- return
- } else if err != nil {
- logger.Errorf("Failed to retrieve node record: %v", err)
- http.Error(w, "Internal server error", http.StatusInternalServerError)
- return
- }
- if err := nodeRecord.CheckAdmission(time.Now()); err != nil {
- http.Error(w, "Node not admitted: "+err.Error(), http.StatusUnauthorized)
+ nodeRecord := s.admittedNodeWithChallenge(w, r, api.NodesCatalogChallenge)
+ if nodeRecord == nil {
return
}
@@ -200,7 +162,7 @@ func (s *Server) HandleNodeCatalog(w http.ResponseWriter, r *http.Request) {
}
s.catalogMu.Lock()
- s.catalog[peerID.String()] = nodeCatalogEntry{
+ s.catalog[nodeRecord.PeerID] = nodeCatalogEntry{
Services: req.Services,
ReportedAt: time.Now(),
}
diff --git a/internal/controlplane/catalog_test.go b/internal/controlplane/catalog_test.go
index aa28b252..4a1e2c73 100644
--- a/internal/controlplane/catalog_test.go
+++ b/internal/controlplane/catalog_test.go
@@ -21,6 +21,7 @@ import (
"encoding/base64"
"encoding/json"
"net/http"
+ "strconv"
"testing"
"time"
@@ -32,8 +33,9 @@ import (
)
// postCatalog POSTs a raw body to /nodes/catalog under the given
-// Authorization header value and returns the response status.
-func postCatalog(t *testing.T, cpURL, authHeader string, body []byte) int {
+// Authorization header value and optional node key (for signing the PoP
+// challenge) and returns the response status.
+func postCatalog(t *testing.T, cpURL, authHeader string, priv crypto.PrivKey, body []byte) int {
t.Helper()
req, err := http.NewRequest(http.MethodPost, cpURL+"/nodes/catalog", bytes.NewReader(body))
@@ -43,6 +45,19 @@ func postCatalog(t *testing.T, cpURL, authHeader string, body []byte) int {
if authHeader != "" {
req.Header.Set("Authorization", authHeader)
}
+ if priv != nil {
+ pID, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatalf("IDFromPrivateKey: %v", err)
+ }
+ ts := time.Now().UnixMilli()
+ sig, err := priv.Sign(api.NodesCatalogChallenge(pID.String(), ts))
+ if err != nil {
+ t.Fatalf("Sign: %v", err)
+ }
+ req.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(ts, 10))
+ req.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(sig))
+ }
req.Header.Set("Content-Type", "application/x-protobuf")
resp, err := http.DefaultClient.Do(req)
@@ -113,7 +128,7 @@ func TestHandleNodeCatalog(t *testing.T) {
&api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "stvv-compliance-docs", Description: "doc lookup"},
&api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "llama", Description: "local model"},
)
- if got := postCatalog(t, cpURL, bearer(biscuitBytes), body); got != http.StatusNoContent {
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {
t.Fatalf("HandleNodeCatalog: got status %d, want %d", got, http.StatusNoContent)
}
@@ -128,7 +143,7 @@ func TestHandleNodeCatalog(t *testing.T) {
// A second report replaces the first rather than accumulating.
body = catalogBody(t, &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_A2A, Name: "planner"})
- if got := postCatalog(t, cpURL, bearer(biscuitBytes), body); got != http.StatusNoContent {
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {
t.Fatalf("second report: got status %d, want %d", got, http.StatusNoContent)
}
entry = srv.catalogSnapshot()[nodePeer.String()]
@@ -148,7 +163,7 @@ func TestHandleNodeCatalog(t *testing.T) {
}
// An empty report is valid and clears the node's services.
- if got := postCatalog(t, cpURL, bearer(biscuitBytes), catalogBody(t)); got != http.StatusNoContent {
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, catalogBody(t)); got != http.StatusNoContent {
t.Fatalf("empty report: got status %d, want %d", got, http.StatusNoContent)
}
if view := adminNodeCatalog(t, cpURL, srv.config.AdminToken); len(view[nodePeer.String()].Services) != 0 {
@@ -166,7 +181,7 @@ func TestHandleNodeCatalog_Rejections(t *testing.T) {
}()
ctx := context.Background()
- _, biscuitBytes := enrollRefreshTestNode(t, ctx, store)
+ priv, biscuitBytes := enrollRefreshTestNode(t, ctx, store)
cpPriv, _, err := store.GetCurrentKey(ctx)
if err != nil {
t.Fatalf("GetCurrentKey: %v", err)
@@ -204,21 +219,23 @@ func TestHandleNodeCatalog_Rejections(t *testing.T) {
tests := []struct {
name string
auth string
+ priv crypto.PrivKey
body []byte
want int
}{
{name: "missing authorization", auth: "", body: ok, want: http.StatusUnauthorized},
{name: "not a bearer token", auth: "Basic abc", body: ok, want: http.StatusUnauthorized},
- {name: "malformed base64", auth: "Bearer %%%not-base64", body: ok, want: http.StatusBadRequest},
+ {name: "malformed base64", auth: "Bearer %%%not-base64", body: ok, want: http.StatusUnauthorized},
{name: "not a biscuit", auth: bearer([]byte("garbage")), body: ok, want: http.StatusUnauthorized},
- {name: "forged signature", auth: bearer(forged), body: ok, want: http.StatusUnauthorized},
- {name: "unenrolled peer", auth: bearer(unenrolled), body: ok, want: http.StatusUnauthorized},
- {name: "invalid body", auth: bearer(biscuitBytes), body: []byte(`{"services":[]}`), want: http.StatusBadRequest},
- {name: "too many services", auth: bearer(biscuitBytes), body: catalogBody(t, tooMany...), want: http.StatusBadRequest},
+ {name: "forged signature", auth: bearer(forged), priv: strangerPriv, body: ok, want: http.StatusUnauthorized},
+ {name: "unenrolled peer", auth: bearer(unenrolled), priv: strangerPriv, body: ok, want: http.StatusUnauthorized},
+ {name: "missing challenge", auth: bearer(biscuitBytes), body: ok, want: http.StatusUnauthorized},
+ {name: "invalid body", auth: bearer(biscuitBytes), priv: priv, body: []byte(`{"services":[]}`), want: http.StatusBadRequest},
+ {name: "too many services", auth: bearer(biscuitBytes), priv: priv, body: catalogBody(t, tooMany...), want: http.StatusBadRequest},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
- if got := postCatalog(t, cpURL, tc.auth, tc.body); got != tc.want {
+ if got := postCatalog(t, cpURL, tc.auth, tc.priv, tc.body); got != tc.want {
t.Fatalf("got status %d, want %d", got, tc.want)
}
})
@@ -257,7 +274,7 @@ func TestHandleNodeCatalog_Admission(t *testing.T) {
t.Fatalf("IDFromPrivateKey: %v", err)
}
body := catalogBody(t, &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "calc"})
- if got := postCatalog(t, cpURL, bearer(biscuitBytes), body); got != http.StatusNoContent {
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {
t.Fatalf("admitted node: got status %d, want %d", got, http.StatusNoContent)
}
if _, ok := adminNodeCatalog(t, cpURL, srv.config.AdminToken)[nodePeer.String()]; !ok {
@@ -273,7 +290,7 @@ func TestHandleNodeCatalog_Admission(t *testing.T) {
if err := store.EnrollNode(ctx, record); err != nil {
t.Fatalf("EnrollNode: %v", err)
}
- if got := postCatalog(t, cpURL, bearer(biscuitBytes), body); got != http.StatusUnauthorized {
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusUnauthorized {
t.Fatalf("expired session: got status %d, want %d", got, http.StatusUnauthorized)
}
if view := adminNodeCatalog(t, cpURL, srv.config.AdminToken); len(view) != 0 {
@@ -288,7 +305,7 @@ func TestHandleNodeCatalog_Admission(t *testing.T) {
if err := srv.banNode(ctx, record); err != nil {
t.Fatalf("banNode: %v", err)
}
- if got := postCatalog(t, cpURL, bearer(biscuitBytes), body); got != http.StatusUnauthorized {
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusUnauthorized {
t.Fatalf("banned node: got status %d, want %d", got, http.StatusUnauthorized)
}
if snap := srv.catalogSnapshot(); len(snap) != 0 {
@@ -333,7 +350,7 @@ func TestCatalogPeerIDCanonicalization(t *testing.T) {
}
body := catalogBody(t, &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "calc"})
- if got := postCatalog(t, cpURL, bearer(biscuitBytes), body); got != http.StatusNoContent {
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {
t.Fatalf("report: got status %d, want %d", got, http.StatusNoContent)
}
diff --git a/internal/controlplane/client/client.go b/internal/controlplane/client/client.go
index 75a343b7..44ad66e9 100644
--- a/internal/controlplane/client/client.go
+++ b/internal/controlplane/client/client.go
@@ -27,9 +27,11 @@ import (
"fmt"
"io"
"net/http"
+ "strconv"
"strings"
"time"
+ "github.com/libp2p/go-libp2p/core/crypto"
"google.golang.org/protobuf/proto"
"github.com/google/sam/api"
@@ -67,6 +69,54 @@ func ReadBody(r io.Reader) ([]byte, error) {
return body, nil
}
+// SetChallengeHeaders signs payload with priv and sets HeaderChallengeTimestamp
+// and HeaderChallengeSignature on req.
+func SetChallengeHeaders(req *http.Request, priv crypto.PrivKey, payload []byte, ts int64) error {
+ sig, err := priv.Sign(payload)
+ if err != nil {
+ return fmt.Errorf("failed to sign challenge: %w", err)
+ }
+ req.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(ts, 10))
+ req.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(sig))
+ return nil
+}
+
+// DoWithChallenge executes a request built with the local clock's millisecond
+// timestamp. If the control plane answers 401 with
+// api.ErrStaleChallengeTimestampMessage and a valid HTTP Date header, it
+// recomputes the timestamp from Date and retries once. No other 401 is retried.
+func DoWithChallenge(httpClient *http.Client, now func() time.Time, build func(ts int64) (*http.Request, error)) (*http.Response, error) {
+ if now == nil {
+ now = time.Now
+ }
+ req, err := build(now().UnixMilli())
+ if err != nil {
+ return nil, err
+ }
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode != http.StatusUnauthorized {
+ return resp, nil
+ }
+ body, readErr := ReadBody(resp.Body)
+ _ = resp.Body.Close()
+ if readErr == nil && strings.Contains(string(body), api.ErrStaleChallengeTimestampMessage) {
+ if dateHdr := resp.Header.Get("Date"); dateHdr != "" {
+ if serverTime, parseErr := http.ParseTime(dateHdr); parseErr == nil {
+ retryReq, buildErr := build(serverTime.UnixMilli())
+ if buildErr != nil {
+ return nil, buildErr
+ }
+ return httpClient.Do(retryReq)
+ }
+ }
+ }
+ resp.Body = io.NopCloser(bytes.NewReader(body))
+ return resp, nil
+}
+
// transport applies api.ValidateControlPlaneTransport to every request,
// redirects included, so a plaintext hop is refused wherever the URL came
// from. allowInsecure is read per request: the node learns the operator's
@@ -98,6 +148,9 @@ func NewHTTPClient(timeout time.Duration, allowInsecure func() bool, component s
type Client struct {
baseURL string
http *http.Client
+ peerID string
+ priv crypto.PrivKey
+ now func() time.Time
}
// New normalizes baseURL, https:// when no scheme is given and no trailing
@@ -107,14 +160,30 @@ func New(baseURL string, httpClient *http.Client) *Client {
if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
baseURL = "https://" + baseURL
}
- return &Client{baseURL: strings.TrimSuffix(baseURL, "/"), http: httpClient}
+ return &Client{baseURL: strings.TrimSuffix(baseURL, "/"), http: httpClient, now: time.Now}
+}
+
+// WithIdentity configures the peer ID and private key used to sign
+// proof-of-possession challenges on authenticated requests.
+func (c *Client) WithIdentity(peerID string, priv crypto.PrivKey) *Client {
+ c.peerID = peerID
+ c.priv = priv
+ return c
+}
+
+// WithClock overrides the clock used for challenge timestamps.
+func (c *Client) WithClock(now func() time.Time) *Client {
+ if now != nil {
+ c.now = now
+ }
+ return c
}
// FetchInfo is GET /info: the router addresses, the ban set and the OIDC
// details a node needs to enroll.
func (c *Client) FetchInfo(ctx context.Context) (*api.ControlPlaneInfoResponse, error) {
var info api.ControlPlaneInfoResponse
- if err := c.get(ctx, "/info", nil, &info); err != nil {
+ if err := c.get(ctx, "/info", nil, nil, &info); err != nil {
return nil, err
}
return &info, nil
@@ -126,7 +195,7 @@ func (c *Client) FetchInfo(ctx context.Context) (*api.ControlPlaneInfoResponse,
// control plane, not become it.
func (c *Client) FetchKeys(ctx context.Context, trusted []ed25519.PublicKey) ([]ed25519.PublicKey, error) {
var resp api.KeysResponse
- if err := c.get(ctx, "/keys", nil, &resp); err != nil {
+ if err := c.get(ctx, "/keys", nil, nil, &resp); err != nil {
return nil, err
}
keys, err := api.VerifyKeysResponse(&resp, trusted, time.Now())
@@ -136,42 +205,94 @@ func (c *Client) FetchKeys(ctx context.Context, trusted []ed25519.PublicKey) ([]
return keys, nil
}
-// FetchPolicy is GET /policies, authenticated with the caller's biscuit: the
-// mesh policy as the Datalog rules a member adds to its authorizer.
+// FetchPolicy is GET /policies, authenticated with the caller's biscuit and
+// signed challenge: the mesh policy as the Datalog rules a member adds to its
+// authorizer.
func (c *Client) FetchPolicy(ctx context.Context, biscuit []byte) (*api.PolicyConfigGetResponse, error) {
var policy api.PolicyConfigGetResponse
- if err := c.get(ctx, "/policies", biscuit, &policy); err != nil {
+ if err := c.get(ctx, "/policies", biscuit, api.PoliciesChallenge, &policy); err != nil {
return nil, err
}
return &policy, nil
}
-// FetchEgress is GET /egress, authenticated with the caller's biscuit: the
-// egress destinations the control plane assigned to this node.
+// FetchEgress is GET /egress, authenticated with the caller's biscuit and
+// signed challenge: the egress destinations the control plane assigned to this
+// node.
func (c *Client) FetchEgress(ctx context.Context, biscuit []byte) (*api.EgressAssignmentsResponse, error) {
var egress api.EgressAssignmentsResponse
- if err := c.get(ctx, "/egress", biscuit, &egress); err != nil {
+ if err := c.get(ctx, "/egress", biscuit, api.EgressChallenge, &egress); err != nil {
return nil, err
}
return &egress, nil
}
-// FetchRevocations is GET /revocations, authenticated with the caller's biscuit:
-// the revoked Biscuit IDs and banned peer IDs currently tracked by the control plane.
+// FetchRevocations is GET /revocations, authenticated with the caller's
+// biscuit and signed challenge: the revoked Biscuit IDs and banned peer IDs
+// currently tracked by the control plane.
func (c *Client) FetchRevocations(ctx context.Context, biscuit []byte) (*api.RevocationsResponse, error) {
var revocations api.RevocationsResponse
- if err := c.get(ctx, "/revocations", biscuit, &revocations); err != nil {
+ if err := c.get(ctx, "/revocations", biscuit, api.RevocationsChallenge, &revocations); err != nil {
return nil, err
}
return &revocations, nil
}
+// ReportCatalog is POST /nodes/catalog, authenticated with the calling node's
+// biscuit and signed challenge: reports the services currently registered on
+// the node.
+func (c *Client) ReportCatalog(ctx context.Context, biscuit []byte, services []*api.ServiceInfo) error {
+ payload, err := proto.Marshal(&api.NodeCatalogReport{Services: services})
+ if err != nil {
+ return fmt.Errorf("failed to encode catalog report: %w", err)
+ }
+ resp, err := DoWithChallenge(c.http, c.now, func(ts int64) (*http.Request, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/nodes/catalog", bytes.NewReader(payload))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create HTTP request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/x-protobuf")
+ if len(biscuit) > 0 {
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuit))
+ }
+ if c.priv != nil && c.peerID != "" {
+ if err := SetChallengeHeaders(req, c.priv, api.NodesCatalogChallenge(c.peerID, ts), ts); err != nil {
+ return nil, err
+ }
+ }
+ return req, nil
+ })
+ if err != nil {
+ return fmt.Errorf("HTTP request failed: %w", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ if resp.StatusCode != http.StatusNoContent {
+ body, _ := ReadBody(resp.Body)
+ return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))
+ }
+ return nil
+}
+
// ExchangeToken is POST /token/exchange, authenticated with the calling node's
// biscuit: verifies an external OIDC/K8s/SPIFFE JWT and mints a short-lived
// Delegated Session Biscuit bound to the calling node.
func (c *Client) ExchangeToken(ctx context.Context, biscuit []byte, req *api.TokenExchangeRequest) (*api.TokenExchangeResponse, error) {
var resp api.TokenExchangeResponse
- if err := c.post(ctx, "/token/exchange", biscuit, req, &resp); err != nil {
+ err := c.postWithChallenge(ctx, "/token/exchange", biscuit, func(ts int64) (proto.Message, error) {
+ if c.priv == nil || c.peerID == "" {
+ return req, nil
+ }
+ sig, err := c.priv.Sign(api.TokenExchangeChallenge(c.peerID, ts))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign token exchange challenge: %w", err)
+ }
+ cloned := proto.Clone(req).(*api.TokenExchangeRequest)
+ cloned.ChallengeUnixMs = ts
+ cloned.ChallengeSignature = sig
+ return cloned, nil
+ }, &resp)
+ if err != nil {
return nil, err
}
return &resp, nil
@@ -182,21 +303,41 @@ func (c *Client) ExchangeToken(ctx context.Context, biscuit []byte, req *api.Tok
// short-lived ES256 JWT for cloud STS federation.
func (c *Client) MintSTSToken(ctx context.Context, biscuit []byte, req *api.STSTokenRequest) (*api.STSTokenResponse, error) {
var resp api.STSTokenResponse
- if err := c.post(ctx, "/sts/token", biscuit, req, &resp); err != nil {
+ err := c.postWithChallenge(ctx, "/sts/token", biscuit, func(ts int64) (proto.Message, error) {
+ if c.priv == nil || c.peerID == "" {
+ return req, nil
+ }
+ sig, err := c.priv.Sign(api.STSTokenChallenge(c.peerID, ts))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign STS token challenge: %w", err)
+ }
+ cloned := proto.Clone(req).(*api.STSTokenRequest)
+ cloned.ChallengeUnixMs = ts
+ cloned.ChallengeSignature = sig
+ return cloned, nil
+ }, &resp)
+ if err != nil {
return nil, err
}
return &resp, nil
}
-func (c *Client) get(ctx context.Context, path string, biscuit []byte, msg proto.Message) error {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
- if err != nil {
- return fmt.Errorf("failed to create HTTP request: %w", err)
- }
- if len(biscuit) > 0 {
- req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuit))
- }
- resp, err := c.http.Do(req)
+func (c *Client) get(ctx context.Context, path string, biscuit []byte, challengeFn func(peerID string, ts int64) []byte, msg proto.Message) error {
+ resp, err := DoWithChallenge(c.http, c.now, func(ts int64) (*http.Request, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create HTTP request: %w", err)
+ }
+ if len(biscuit) > 0 {
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuit))
+ }
+ if challengeFn != nil && c.priv != nil && c.peerID != "" {
+ if err := SetChallengeHeaders(req, c.priv, challengeFn(c.peerID, ts), ts); err != nil {
+ return nil, err
+ }
+ }
+ return req, nil
+ })
if err != nil {
return fmt.Errorf("HTTP request failed: %w", err)
}
@@ -218,20 +359,26 @@ func (c *Client) get(ctx context.Context, path string, biscuit []byte, msg proto
return nil
}
-func (c *Client) post(ctx context.Context, path string, biscuit []byte, reqMsg, respMsg proto.Message) error {
- payload, err := proto.Marshal(reqMsg)
- if err != nil {
- return fmt.Errorf("failed to marshal %s request: %w", path, err)
- }
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(payload))
- if err != nil {
- return fmt.Errorf("failed to create HTTP request: %w", err)
- }
- req.Header.Set("Content-Type", "application/x-protobuf")
- if len(biscuit) > 0 {
- req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuit))
- }
- resp, err := c.http.Do(req)
+func (c *Client) postWithChallenge(ctx context.Context, path string, biscuit []byte, buildMsg func(ts int64) (proto.Message, error), respMsg proto.Message) error {
+ resp, err := DoWithChallenge(c.http, c.now, func(ts int64) (*http.Request, error) {
+ reqMsg, err := buildMsg(ts)
+ if err != nil {
+ return nil, err
+ }
+ payload, err := proto.Marshal(reqMsg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal %s request: %w", path, err)
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(payload))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create HTTP request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/x-protobuf")
+ if len(biscuit) > 0 {
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuit))
+ }
+ return req, nil
+ })
if err != nil {
return fmt.Errorf("HTTP request failed: %w", err)
}
diff --git a/internal/controlplane/egress_test.go b/internal/controlplane/egress_test.go
index 68ffadc2..70e4b0dc 100644
--- a/internal/controlplane/egress_test.go
+++ b/internal/controlplane/egress_test.go
@@ -20,6 +20,7 @@ import (
"io"
"net/http"
"slices"
+ "strconv"
"strings"
"testing"
"time"
@@ -125,7 +126,12 @@ func TestEgressPolicyIsDistributedToServingNodes(t *testing.T) {
// Two nodes: one holds the pep role, one is a plain node labelled
// site=dc1. Each is selected by exactly one destination.
- enroll := func(t *testing.T, role string, labels map[string]string) []byte {
+ type enrolledMeshNode struct {
+ token []byte
+ priv crypto.PrivKey
+ peerID string
+ }
+ enroll := func(t *testing.T, role string, labels map[string]string) enrolledMeshNode {
t.Helper()
token := createAdminBootstrapToken(t, baseURL, adminToken, role, 1)
priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
@@ -144,15 +150,28 @@ func TestEgressPolicyIsDistributedToServingNodes(t *testing.T) {
if out.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {
t.Fatalf("enroll %s: status %v (%s)", role, out.Status, out.ErrorMessage)
}
- return out.BiscuitToken
+ return enrolledMeshNode{token: out.BiscuitToken, priv: priv, peerID: pID.String()}
}
- pepToken := enroll(t, "pep", nil)
- dc1Token := enroll(t, api.RoleNode, map[string]string{"site": "dc1"})
+ pepNode := enroll(t, "pep", nil)
+ dc1Node := enroll(t, api.RoleNode, map[string]string{"site": "dc1"})
- getMesh := func(t *testing.T, path string, token []byte, out proto.Message) {
+ getMesh := func(t *testing.T, path string, node enrolledMeshNode, out proto.Message) {
t.Helper()
req, _ := http.NewRequest(http.MethodGet, baseURL+path, nil)
- req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(token))
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(node.token))
+ ts := time.Now().UnixMilli()
+ var challenge []byte
+ if path == "/policies" {
+ challenge = api.PoliciesChallenge(node.peerID, ts)
+ } else {
+ challenge = api.EgressChallenge(node.peerID, ts)
+ }
+ sig, err := node.priv.Sign(challenge)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(ts, 10))
+ req.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(sig))
resp, err := client.Do(req)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
@@ -168,7 +187,7 @@ func TestEgressPolicyIsDistributedToServingNodes(t *testing.T) {
}
var rules api.PolicyConfigGetResponse
- getMesh(t, "/policies", pepToken, &rules)
+ getMesh(t, "/policies", pepNode, &rules)
for _, want := range []string{
`granted_service_exact("egress", "api.github.com") <- role("pep")`,
`granted_service_exact("egress", "mam.internal.example.com") <- label("site", "dc1")`,
@@ -192,14 +211,14 @@ func TestEgressPolicyIsDistributedToServingNodes(t *testing.T) {
return out
}
var pepEgress, dc1Egress api.EgressAssignmentsResponse
- getMesh(t, "/egress", pepToken, &pepEgress)
+ getMesh(t, "/egress", pepNode, &pepEgress)
if got := names(&pepEgress); !slices.Equal(got, []string{"api.github.com"}) {
t.Errorf("pep node assigned %v, want [api.github.com]", got)
}
if pepEgress.Egress[0].Credential != "github-eu" {
t.Errorf("assignment lost its credential name: %v", pepEgress.Egress[0])
}
- getMesh(t, "/egress", dc1Token, &dc1Egress)
+ getMesh(t, "/egress", dc1Node, &dc1Egress)
if got := names(&dc1Egress); !slices.Equal(got, []string{"mam.internal.example.com"}) {
t.Errorf("dc1 node assigned %v, want [mam.internal.example.com]", got)
}
diff --git a/internal/controlplane/policies_admission_test.go b/internal/controlplane/policies_admission_test.go
index 2b64678c..7b62bcc6 100644
--- a/internal/controlplane/policies_admission_test.go
+++ b/internal/controlplane/policies_admission_test.go
@@ -21,10 +21,14 @@ import (
"io"
"net/http"
"slices"
+ "strconv"
+ "strings"
+ "sync/atomic"
"testing"
"time"
"github.com/google/sam/api"
+ cpclient "github.com/google/sam/internal/controlplane/client"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/proto"
@@ -101,6 +105,13 @@ func TestPoliciesRequiresAnAdmissibleNode(t *testing.T) {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(enrollResp.BiscuitToken))
+ cTS := time.Now().UnixMilli()
+ cSig, err := privNode.Sign(api.PoliciesChallenge(nodePeer.String(), cTS))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(cTS, 10))
+ req.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(cSig))
resp, err := client.Do(req)
if err != nil {
t.Fatalf("GET /policies failed: %v", err)
@@ -158,3 +169,320 @@ func TestPoliciesRequiresAnAdmissibleNode(t *testing.T) {
t.Errorf("banned node got status %d, want %d", got, http.StatusUnauthorized)
}
}
+
+func TestReadAndReportEndpointsRequireProofOfPossession(t *testing.T) {
+ issuer, mintToken := startCustomMockOIDC(t)
+ srv, store, baseURL := setupTestServer(t, issuer)
+ defer func() {
+ _ = srv.Close()
+ _ = store.Close()
+ }()
+
+ ctx := context.Background()
+ client := &http.Client{Timeout: 5 * time.Second}
+
+ nodeBindings := []*api.PolicyBinding{{Role: api.RoleNode, Members: []string{"group:users"}}}
+ if err := store.SaveMeshPolicy(ctx, nil, nodeBindings); err != nil {
+ t.Fatalf("failed to seed policy: %v", err)
+ }
+
+ privNode, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodePeer, err := peer.IDFromPrivateKey(privNode)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodePubKeyBytes, err := crypto.MarshalPublicKey(privNode.GetPublic())
+ if err != nil {
+ t.Fatal(err)
+ }
+ otherPriv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ts, sig := registerPoP(t, privNode, nodePeer.String())
+ enrollReq := &api.EnrollRequest{
+ Jwt: mintToken(map[string]interface{}{"sub": "node-pop", "groups": []string{"users"}}),
+ PeerId: nodePeer.String(),
+ PublicKey: nodePubKeyBytes,
+ RequestedRole: api.RoleNode,
+ ChallengeUnixMs: ts,
+ ChallengeSignature: sig,
+ }
+ reqData, _ := proto.Marshal(enrollReq)
+ resp, err := client.Post(baseURL+"/register", "application/x-protobuf", bytes.NewReader(reqData))
+ if err != nil {
+ t.Fatalf("node /register failed: %v", err)
+ }
+ body, _ := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("node /register status %s (body: %s)", resp.Status, body)
+ }
+ var enrollResp api.EnrollResponse
+ if err := proto.Unmarshal(body, &enrollResp); err != nil {
+ t.Fatal(err)
+ }
+ authHeader := "Bearer " + base64.StdEncoding.EncodeToString(enrollResp.BiscuitToken)
+ catalogBody, _ := proto.Marshal(&api.NodeCatalogReport{})
+
+ endpoints := []struct {
+ name string
+ method string
+ path string
+ body []byte
+ challenge func(string, int64) []byte
+ wrongDom func(string, int64) []byte
+ wantStatus int
+ }{
+ {
+ name: "GET /policies",
+ method: http.MethodGet,
+ path: "/policies",
+ challenge: api.PoliciesChallenge,
+ wrongDom: api.EgressChallenge,
+ wantStatus: http.StatusOK,
+ },
+ {
+ name: "GET /egress",
+ method: http.MethodGet,
+ path: "/egress",
+ challenge: api.EgressChallenge,
+ wrongDom: api.PoliciesChallenge,
+ wantStatus: http.StatusOK,
+ },
+ {
+ name: "GET /revocations",
+ method: http.MethodGet,
+ path: "/revocations",
+ challenge: api.RevocationsChallenge,
+ wrongDom: api.PoliciesChallenge,
+ wantStatus: http.StatusOK,
+ },
+ {
+ name: "POST /nodes/catalog",
+ method: http.MethodPost,
+ path: "/nodes/catalog",
+ body: catalogBody,
+ challenge: api.NodesCatalogChallenge,
+ wrongDom: api.PoliciesChallenge,
+ wantStatus: http.StatusNoContent,
+ },
+ }
+
+ for _, ep := range endpoints {
+ t.Run(ep.name, func(t *testing.T) {
+ doReq := func(tsHeader string, sig []byte) (*http.Response, string) {
+ var rBody io.Reader
+ if ep.body != nil {
+ rBody = bytes.NewReader(ep.body)
+ }
+ req, err := http.NewRequest(ep.method, baseURL+ep.path, rBody)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Authorization", authHeader)
+ if ep.body != nil {
+ req.Header.Set("Content-Type", "application/x-protobuf")
+ }
+ if tsHeader != "" {
+ req.Header.Set(api.HeaderChallengeTimestamp, tsHeader)
+ }
+ if sig != nil {
+ req.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(sig))
+ }
+ res, err := client.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, _ := io.ReadAll(res.Body)
+ _ = res.Body.Close()
+ return res, string(b)
+ }
+
+ // 1. Bare bearer Biscuit without challenge headers -> 401.
+ if res, _ := doReq("", nil); res.StatusCode != http.StatusUnauthorized {
+ t.Errorf("bare biscuit got %d, want 401", res.StatusCode)
+ }
+
+ // 2. Stale timestamp (+10 min) -> 401 with stale challenge message and Date header.
+ staleTS := time.Now().Add(10 * time.Minute).UnixMilli()
+ staleSig, _ := privNode.Sign(ep.challenge(nodePeer.String(), staleTS))
+ res, msg := doReq(strconv.FormatInt(staleTS, 10), staleSig)
+ if res.StatusCode != http.StatusUnauthorized {
+ t.Errorf("stale challenge got %d, want 401", res.StatusCode)
+ }
+ if !strings.Contains(msg, api.ErrStaleChallengeTimestampMessage) {
+ t.Errorf("stale challenge body = %q, want %q", msg, api.ErrStaleChallengeTimestampMessage)
+ }
+ if res.Header.Get("Date") == "" {
+ t.Errorf("stale challenge 401 missing Date header")
+ }
+
+ // 3. Signature for another endpoint -> 401.
+ nowTS := time.Now().UnixMilli()
+ wrongDomSig, _ := privNode.Sign(ep.wrongDom(nodePeer.String(), nowTS))
+ if res, _ := doReq(strconv.FormatInt(nowTS, 10), wrongDomSig); res.StatusCode != http.StatusUnauthorized {
+ t.Errorf("wrong domain challenge got %d, want 401", res.StatusCode)
+ }
+
+ // 4. Signature by another key -> 401.
+ wrongKeySig, _ := otherPriv.Sign(ep.challenge(nodePeer.String(), nowTS))
+ if res, _ := doReq(strconv.FormatInt(nowTS, 10), wrongKeySig); res.StatusCode != http.StatusUnauthorized {
+ t.Errorf("wrong key challenge got %d, want 401", res.StatusCode)
+ }
+
+ // 5. Fresh valid challenge -> wantStatus.
+ validSig, _ := privNode.Sign(ep.challenge(nodePeer.String(), nowTS))
+ if res, body := doReq(strconv.FormatInt(nowTS, 10), validSig); res.StatusCode != ep.wantStatus {
+ t.Errorf("valid challenge got %d (%s), want %d", res.StatusCode, body, ep.wantStatus)
+ }
+ })
+ }
+}
+
+func TestChallengeClockSkewRecoveryViaDateHeader(t *testing.T) {
+ issuer, mintToken := startCustomMockOIDC(t)
+ srv, store, baseURL := setupTestServer(t, issuer)
+ defer func() {
+ _ = srv.Close()
+ _ = store.Close()
+ }()
+
+ ctx := context.Background()
+ nodeBindings := []*api.PolicyBinding{{Role: api.RoleNode, Members: []string{"group:users"}}}
+ if err := store.SaveMeshPolicy(ctx, nil, nodeBindings); err != nil {
+ t.Fatalf("failed to seed policy: %v", err)
+ }
+
+ privNode, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodePeer, err := peer.IDFromPrivateKey(privNode)
+ if err != nil {
+ t.Fatal(err)
+ }
+ nodePubKeyBytes, err := crypto.MarshalPublicKey(privNode.GetPublic())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify GET /enroll/status checks timestamp freshness before looking up the peer ID,
+ // so it is never a peer-ID existence oracle and skewed callers get 401 (not 404).
+ staleTS := time.Now().Add(10 * time.Minute).UnixMilli()
+ staleSig, _ := privNode.Sign(api.EnrollStatusChallenge(nodePeer.String(), staleTS))
+ statusReq, _ := http.NewRequest(http.MethodGet, baseURL+"/enroll/status?peer_id="+nodePeer.String(), nil)
+ statusReq.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(staleTS, 10))
+ statusReq.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(staleSig))
+ statusResp, err := http.DefaultClient.Do(statusReq)
+ if err != nil {
+ t.Fatal(err)
+ }
+ statusBody, _ := io.ReadAll(statusResp.Body)
+ _ = statusResp.Body.Close()
+ if statusResp.StatusCode != http.StatusUnauthorized || !strings.Contains(string(statusBody), api.ErrStaleChallengeTimestampMessage) {
+ t.Fatalf("GET /enroll/status with stale timestamp got %d (%q), want 401 with %q", statusResp.StatusCode, statusBody, api.ErrStaleChallengeTimestampMessage)
+ }
+
+ // Enroll the node using a client with a +10 min clock via DoWithChallenge.
+ var roundTrips atomic.Int32
+ countingTransport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ roundTrips.Add(1)
+ return http.DefaultTransport.RoundTrip(req)
+ })
+ httpClient := &http.Client{Timeout: 5 * time.Second, Transport: countingTransport}
+ skewedClock := func() time.Time {
+ return time.Now().Add(10 * time.Minute)
+ }
+ skewedCP := cpclient.New(baseURL, httpClient).WithIdentity(nodePeer.String(), privNode).WithClock(skewedClock)
+
+ regResp, err := cpclient.DoWithChallenge(httpClient, skewedClock, func(cTS int64) (*http.Request, error) {
+ cSig, signErr := privNode.Sign(api.RegisterChallenge(nodePeer.String(), cTS))
+ if signErr != nil {
+ return nil, signErr
+ }
+ payload, marshalErr := proto.Marshal(&api.EnrollRequest{
+ Jwt: mintToken(map[string]interface{}{"sub": "node-skewed", "groups": []string{"users"}}),
+ PeerId: nodePeer.String(),
+ PublicKey: nodePubKeyBytes,
+ RequestedRole: api.RoleNode,
+ ChallengeUnixMs: cTS,
+ ChallengeSignature: cSig,
+ })
+ if marshalErr != nil {
+ return nil, marshalErr
+ }
+ req, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/register", bytes.NewReader(payload))
+ if reqErr != nil {
+ return nil, reqErr
+ }
+ req.Header.Set("Content-Type", "application/x-protobuf")
+ return req, nil
+ })
+ if err != nil {
+ t.Fatalf("skewed /register failed: %v", err)
+ }
+ regBody, _ := io.ReadAll(regResp.Body)
+ _ = regResp.Body.Close()
+ if regResp.StatusCode != http.StatusOK {
+ t.Fatalf("skewed /register status %d: %s", regResp.StatusCode, regBody)
+ }
+ var enrollResp api.EnrollResponse
+ if err := proto.Unmarshal(regBody, &enrollResp); err != nil {
+ t.Fatal(err)
+ }
+ if got := roundTrips.Load(); got != 2 {
+ t.Errorf("skewed /register took %d round trips, want 2", got)
+ }
+
+ roundTrips.Store(0)
+ if _, err := skewedCP.FetchPolicy(ctx, enrollResp.BiscuitToken); err != nil {
+ t.Fatalf("skewed FetchPolicy failed: %v", err)
+ }
+ if got := roundTrips.Load(); got != 2 {
+ t.Errorf("skewed FetchPolicy took %d round trips, want 2", got)
+ }
+
+ roundTrips.Store(0)
+ if _, err := skewedCP.FetchEgress(ctx, enrollResp.BiscuitToken); err != nil {
+ t.Fatalf("skewed FetchEgress failed: %v", err)
+ }
+ if got := roundTrips.Load(); got != 2 {
+ t.Errorf("skewed FetchEgress took %d round trips, want 2", got)
+ }
+
+ roundTrips.Store(0)
+ if _, err := skewedCP.FetchRevocations(ctx, enrollResp.BiscuitToken); err != nil {
+ t.Fatalf("skewed FetchRevocations failed: %v", err)
+ }
+ if got := roundTrips.Load(); got != 2 {
+ t.Errorf("skewed FetchRevocations took %d round trips, want 2", got)
+ }
+
+ roundTrips.Store(0)
+ if err := skewedCP.ReportCatalog(ctx, enrollResp.BiscuitToken, nil); err != nil {
+ t.Fatalf("skewed ReportCatalog failed: %v", err)
+ }
+ if got := roundTrips.Load(); got != 2 {
+ t.Errorf("skewed ReportCatalog took %d round trips, want 2", got)
+ }
+
+ // Other 401s (e.g., invalid biscuit) are not retried.
+ roundTrips.Store(0)
+ if _, err := skewedCP.FetchPolicy(ctx, []byte("not-a-valid-biscuit")); err == nil {
+ t.Fatal("expected error for invalid biscuit, got nil")
+ }
+ if got := roundTrips.Load(); got != 1 {
+ t.Errorf("non-stale 401 took %d round trips, want 1 (no retry)", got)
+ }
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+ return f(req)
+}
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index df75ee54..8e2f13f3 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -348,6 +348,7 @@ func noStore(h http.HandlerFunc) http.HandlerFunc {
func meshSurface(h http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
+ w.Header().Set("Access-Control-Expose-Headers", "Date")
if r.Method == http.MethodOptions {
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", strings.Join([]string{"Authorization", "Content-Type", api.HeaderChallengeTimestamp, api.HeaderChallengeSignature}, ", "))
@@ -718,7 +719,7 @@ func (s *Server) HandleRegister(w http.ResponseWriter, r *http.Request) {
}
if err := verifyFreshChallenge(enrolleeKey, api.RegisterChallenge(canonical, req.ChallengeUnixMs), req.ChallengeUnixMs, req.ChallengeSignature); err != nil {
logger.Warnw("Register challenge verification failed", "peer_id", canonical, "error", err)
- http.Error(w, "Invalid registration challenge: "+err.Error(), http.StatusUnauthorized)
+ writeChallengeError(w, "Invalid registration challenge: "+err.Error())
return
}
@@ -964,7 +965,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {
if recovering {
msg = "Invalid biscuit: " + verifyErr.Error()
}
- http.Error(w, msg, http.StatusUnauthorized)
+ writeChallengeError(w, msg)
}
// Fetch node record
@@ -1314,7 +1315,7 @@ func (s *Server) HandleRouterLease(w http.ResponseWriter, r *http.Request) {
}
if err := verifyFreshChallenge(routerKey, api.RouterLeaseChallenge(canonical, req.ChallengeUnixMs), req.ChallengeUnixMs, req.ChallengeSignature); err != nil {
logger.Warnw("Router lease challenge verification failed", "peer_id", canonical, "error", err)
- http.Error(w, "Invalid lease challenge: "+err.Error(), http.StatusUnauthorized)
+ writeChallengeError(w, "Invalid lease challenge: "+err.Error())
return
}
@@ -1371,10 +1372,9 @@ func (s *Server) HandlePolicies(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
// Mesh protocol: an admitted member fetches the policy with its biscuit
- // and receives it as Datalog text only. Operators read the document at
- // GET /admin/policy.
- if !s.isAdmittedNodeRequest(r) {
- http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)
+ // and signed challenge and receives it as Datalog text only. Operators
+ // read the document at GET /admin/policy.
+ if s.admittedNodeWithChallenge(w, r, api.PoliciesChallenge) == nil {
return
}
@@ -1512,12 +1512,34 @@ func (s *Server) unservedEgress(ctx context.Context, policy *api.PolicyConfig) (
return unserved, nil
}
-// isAdmittedNodeRequest reports whether the bearer credential is a biscuit of
-// an enrolled, admitted node. It never falls through to OIDC: running ID token
-// verification on a biscuit logs a failure and would auto-register whoever's
-// ID token lands here.
-func (s *Server) isAdmittedNodeRequest(r *http.Request) bool {
- return s.admittedNode(r) != nil
+// admittedNodeWithChallenge returns the enrolled, admitted node whose biscuit
+// the request bears after verifying the signed freshness challenge in
+// HeaderChallengeTimestamp and HeaderChallengeSignature against the node's
+// stored public key. Every failure writes a uniform 401 Unauthorized.
+func (s *Server) admittedNodeWithChallenge(w http.ResponseWriter, r *http.Request, challengeFn func(peerID string, ts int64) []byte) *storage.EnrolledNode {
+ nodeRecord := s.admittedNode(r)
+ if nodeRecord == nil {
+ writeChallengeError(w, "Unauthorized: node credential required")
+ return nil
+ }
+ ts, tsErr := strconv.ParseInt(r.Header.Get(api.HeaderChallengeTimestamp), 10, 64)
+ sig, sigErr := base64.RawURLEncoding.DecodeString(r.Header.Get(api.HeaderChallengeSignature))
+ if tsErr != nil || sigErr != nil || len(sig) == 0 {
+ writeChallengeError(w, "Missing or invalid challenge headers: signed challenge required")
+ return nil
+ }
+ pubKey, err := crypto.UnmarshalPublicKey(nodeRecord.PublicKey)
+ if err != nil {
+ logger.Errorf("Corrupted public key stored for node %s: %v", nodeRecord.PeerID, err)
+ writeChallengeError(w, "Unauthorized")
+ return nil
+ }
+ if err := verifyFreshChallenge(pubKey, challengeFn(nodeRecord.PeerID, ts), ts, sig); err != nil {
+ logger.Warnw("Node challenge verification failed", "path", r.URL.Path, "peer_id", nodeRecord.PeerID, "error", err)
+ writeChallengeError(w, "Challenge verification failed: "+err.Error())
+ return nil
+ }
+ return nodeRecord
}
// admittedNode returns the enrolled, admitted node whose biscuit the request
@@ -1548,16 +1570,14 @@ func (s *Server) admittedNode(r *http.Request) *storage.EnrolledNode {
// HandleEgress HTTP GET `/egress`: the egress destinations the requesting
// node serves, selected by its roles and labels (see EgressDestination).
-// Mesh protocol, biscuit-authenticated, binary protobuf. A separate endpoint
-// from /policies so a node predating it keeps syncing rules unchanged.
+// Mesh protocol, biscuit-authenticated with signed challenge, binary protobuf.
func (s *Server) HandleEgress(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
- nodeRecord := s.admittedNode(r)
+ nodeRecord := s.admittedNodeWithChallenge(w, r, api.EgressChallenge)
if nodeRecord == nil {
- http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)
return
}
_, bindings, err := s.store.GetMeshPolicy(r.Context())
@@ -1809,6 +1829,10 @@ func (s *Server) HandleEnroll(w http.ResponseWriter, r *http.Request) {
}
if err := verifyFreshChallenge(enrolleeKey, api.EnrollChallenge(canonical, req.ChallengeUnixMs), req.ChallengeUnixMs, req.ChallengeSignature); err != nil {
logger.Warnw("Enroll challenge verification failed", "peer_id", canonical, "error", err)
+ if errors.Is(err, errStaleChallengeTimestamp) {
+ writeChallengeError(w, "Invalid enrollment challenge: "+err.Error())
+ return
+ }
s.writeEnrollError(w, api.EnrollmentStatus_ENROLLMENT_STATUS_REJECTED, "Invalid enrollment challenge: "+err.Error())
return
}
@@ -1976,22 +2000,30 @@ func (s *Server) HandleEnroll(w http.ResponseWriter, r *http.Request) {
}
// challengeMaxAge bounds the freshness window of every signed timestamp
-// challenge on the enrollment surface (/enroll, /enroll/status, /refresh).
+// challenge on the control plane's credentialed endpoints.
const challengeMaxAge = 5 * time.Minute
-// verifyFreshChallenge checks a signed timestamp challenge: ts must be within
-// challengeMaxAge of now and sig must verify over payload with pub. The
-// payload (built by api.EnrollChallenge, api.EnrollStatusChallenge or
-// api.RefreshChallenge) binds the peer and the endpoint, so a signature
-// captured from one request verifies nowhere else.
-func verifyFreshChallenge(pub crypto.PubKey, payload []byte, ts int64, sig []byte) error {
+var errStaleChallengeTimestamp = errors.New(api.ErrStaleChallengeTimestampMessage)
+
+func verifyChallengeTimestamp(ts int64) error {
if ts <= 0 {
return errors.New("missing or invalid challenge timestamp")
}
challengeTime := time.UnixMilli(ts)
now := time.Now()
if now.Sub(challengeTime) > challengeMaxAge || challengeTime.Sub(now) > challengeMaxAge {
- return errors.New("stale or invalid challenge timestamp")
+ return errStaleChallengeTimestamp
+ }
+ return nil
+}
+
+// verifyFreshChallenge checks a signed timestamp challenge: ts must be within
+// challengeMaxAge of now and sig must verify over payload with pub. The
+// payload binds the peer and the endpoint, so a signature captured from one
+// request verifies nowhere else.
+func verifyFreshChallenge(pub crypto.PubKey, payload []byte, ts int64, sig []byte) error {
+ if err := verifyChallengeTimestamp(ts); err != nil {
+ return err
}
ok, err := pub.Verify(payload, sig)
if err != nil || !ok {
@@ -2000,6 +2032,13 @@ func verifyFreshChallenge(pub crypto.PubKey, payload []byte, ts int64, sig []byt
return nil
}
+func writeChallengeError(w http.ResponseWriter, msg string) {
+ if w.Header().Get("Date") == "" {
+ w.Header().Set("Date", time.Now().UTC().Format(http.TimeFormat))
+ }
+ http.Error(w, msg, http.StatusUnauthorized)
+}
+
// HandleEnrollStatus HTTP GET `/enroll/status`
//
// The approved response carries the enrollee's Biscuit, so polling requires
@@ -2029,13 +2068,17 @@ func (s *Server) HandleEnrollStatus(w http.ResponseWriter, r *http.Request) {
ts, tsErr := strconv.ParseInt(r.Header.Get(api.HeaderChallengeTimestamp), 10, 64)
sig, sigErr := base64.RawURLEncoding.DecodeString(r.Header.Get(api.HeaderChallengeSignature))
if tsErr != nil || sigErr != nil || len(sig) == 0 {
- http.Error(w, "Missing or invalid challenge headers: signed challenge required", http.StatusUnauthorized)
+ writeChallengeError(w, "Missing or invalid challenge headers: signed challenge required")
+ return
+ }
+ if err := verifyChallengeTimestamp(ts); err != nil {
+ writeChallengeError(w, "Challenge verification failed: "+err.Error())
return
}
pID, err := peer.Decode(peerID)
if err != nil {
- http.Error(w, "Unauthorized, Invalid Peer ID", http.StatusUnauthorized)
+ writeChallengeError(w, "Unauthorized, Invalid Peer ID")
return
}
canonical := pID.String()
@@ -2043,7 +2086,7 @@ func (s *Server) HandleEnrollStatus(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
enrollReq, err := s.store.GetEnrollmentRequest(ctx, canonical)
if err == storage.ErrNotFound {
- http.Error(w, "Unauthorized", http.StatusUnauthorized)
+ writeChallengeError(w, "Unauthorized")
return
} else if err != nil {
logger.Errorf("Failed to retrieve enrollment status: %v", err)
@@ -2054,12 +2097,12 @@ func (s *Server) HandleEnrollStatus(w http.ResponseWriter, r *http.Request) {
pubKey, err := crypto.UnmarshalPublicKey(enrollReq.PublicKey)
if err != nil {
logger.Errorf("Corrupted public key stored for enrollment %s: %v", canonical, err)
- http.Error(w, "Unauthorized", http.StatusUnauthorized)
+ writeChallengeError(w, "Unauthorized")
return
}
if err := verifyFreshChallenge(pubKey, api.EnrollStatusChallenge(canonical, ts), ts, sig); err != nil {
logger.Warnw("Enroll status challenge verification failed", "peer_id", canonical, "error", err)
- http.Error(w, "Unauthorized", http.StatusUnauthorized)
+ writeChallengeError(w, "Unauthorized")
return
}
diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go
index 62cf91a1..b341a775 100644
--- a/internal/controlplane/sts.go
+++ b/internal/controlplane/sts.go
@@ -383,8 +383,7 @@ func (s *Server) HandleRevocations(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
- if !s.isAdmittedNodeRequest(r) {
- http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)
+ if s.admittedNodeWithChallenge(w, r, api.RevocationsChallenge) == nil {
return
}
ctx := r.Context()
@@ -425,7 +424,7 @@ func (s *Server) HandleTokenExchange(w http.ResponseWriter, r *http.Request) {
}
nodeRecord := s.admittedNode(r)
if nodeRecord == nil {
- http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)
+ writeChallengeError(w, "Unauthorized: node credential required")
return
}
if !s.allowNodeSTSRequest(nodeRecord.PeerID) {
@@ -460,7 +459,7 @@ func (s *Server) HandleTokenExchange(w http.ResponseWriter, r *http.Request) {
challengePayload := api.TokenExchangeChallenge(nodeRecord.PeerID, req.ChallengeUnixMs)
if err := verifyFreshChallenge(nodePubKey, challengePayload, req.ChallengeUnixMs, req.ChallengeSignature); err != nil {
logger.Warnw("Token exchange challenge verification failed", "peer_id", nodeRecord.PeerID, "error", err)
- http.Error(w, "Invalid token exchange challenge: "+err.Error(), http.StatusUnauthorized)
+ writeChallengeError(w, "Invalid token exchange challenge: "+err.Error())
return
}
@@ -617,7 +616,7 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) {
}
nodeRecord := s.admittedNode(r)
if nodeRecord == nil {
- http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)
+ writeChallengeError(w, "Unauthorized: node credential required")
return
}
if !s.allowNodeSTSRequest(nodeRecord.PeerID) {
@@ -661,7 +660,7 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) {
challengePayload := api.STSTokenChallenge(nodeRecord.PeerID, req.ChallengeUnixMs)
if err := verifyFreshChallenge(nodePubKey, challengePayload, req.ChallengeUnixMs, req.ChallengeSignature); err != nil {
logger.Warnw("STS token challenge verification failed", "peer_id", nodeRecord.PeerID, "error", err)
- http.Error(w, "Invalid STS token challenge: "+err.Error(), http.StatusUnauthorized)
+ writeChallengeError(w, "Invalid STS token challenge: "+err.Error())
return
}
diff --git a/internal/controlplane/sts_test.go b/internal/controlplane/sts_test.go
index c895a7c6..f102d167 100644
--- a/internal/controlplane/sts_test.go
+++ b/internal/controlplane/sts_test.go
@@ -28,6 +28,7 @@ import (
"net/http"
"net/url"
"slices"
+ "strconv"
"strings"
"testing"
"time"
@@ -452,6 +453,13 @@ func TestOIDCIssuerJWKSAndSTSToken(t *testing.T) {
httpRevList, _ := http.NewRequest(http.MethodGet, baseURL+"/revocations", nil)
httpRevList.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(nodeBiscuit))
+ revTS := time.Now().UnixMilli()
+ revSig, err := nodePriv.Sign(api.RevocationsChallenge(nodePeerID.String(), revTS))
+ if err != nil {
+ t.Fatalf("sign RevocationsChallenge: %v", err)
+ }
+ httpRevList.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(revTS, 10))
+ httpRevList.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(revSig))
revListResp, err := http.DefaultClient.Do(httpRevList)
if err != nil {
t.Fatalf("GET /revocations failed: %v", err)
diff --git a/internal/node/controlplane.go b/internal/node/controlplane.go
index 99540802..f2ce3248 100644
--- a/internal/node/controlplane.go
+++ b/internal/node/controlplane.go
@@ -18,16 +18,12 @@ import (
"bytes"
"context"
"crypto/ed25519"
- "encoding/base64"
- "fmt"
- "io"
- "net/http"
- "strings"
"time"
"github.com/google/sam/api"
cpclient "github.com/google/sam/internal/controlplane/client"
- "google.golang.org/protobuf/proto"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
)
// maxControlPlaneBodyBytes caps every response body read from the control
@@ -43,6 +39,33 @@ func controlPlaneClient(controlPlaneURL string) *cpclient.Client {
return cpclient.New(controlPlaneURL, controlPlaneHTTPClient(10*time.Second))
}
+// controlPlane returns a control plane client configured with this node's
+// peer ID and private key so authenticated requests carry a signed challenge.
+func (n *SamNode) controlPlane(controlPlaneURL string) *cpclient.Client {
+ c := controlPlaneClient(controlPlaneURL)
+ if n == nil {
+ return c
+ }
+ priv := n.config.PrivKey
+ if priv == nil && n.Store != nil {
+ if kb, err := n.Store.LoadKey(); err == nil && len(kb) > 0 {
+ priv, _ = crypto.UnmarshalPrivateKey(kb)
+ } else {
+ priv = GetOrGenerateKey(n.Store)
+ }
+ }
+ if priv != nil {
+ pid, err := n.localPeerID()
+ if err != nil || pid == "" {
+ pid, _ = peer.IDFromPrivateKey(priv)
+ }
+ if pid != "" {
+ c.WithIdentity(pid.String(), priv)
+ }
+ }
+ return c
+}
+
// FetchControlPlaneInfo retrieves the latest configuration from the control plane's /info endpoint.
func FetchControlPlaneInfo(ctx context.Context, controlPlaneURL string) (*api.ControlPlaneInfoResponse, error) {
return controlPlaneClient(controlPlaneURL).FetchInfo(ctx)
@@ -83,44 +106,10 @@ func publicKeysOf(keys []TrustedKey) []ed25519.PublicKey {
return out
}
-// FetchMeshPolicy retrieves the latest mesh policy from the control plane's /policies endpoint using a biscuit token.
-func FetchMeshPolicy(ctx context.Context, controlPlaneURL string, biscuitToken []byte) (*api.PolicyConfigGetResponse, error) {
- return controlPlaneClient(controlPlaneURL).FetchPolicy(ctx, biscuitToken)
-}
-
// ReportNodeCatalog self-reports this node's locally registered services to
// the control plane's /nodes/catalog endpoint, so an admin can see mesh-wide
// service topology (see catalog.go's HandleNodeCatalog for why this exists
// instead of the control plane discovering it via DHT/P2P itself).
func ReportNodeCatalog(ctx context.Context, controlPlaneURL string, biscuitToken []byte, services []*api.ServiceInfo) error {
- if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
- controlPlaneURL = "https://" + controlPlaneURL
- }
- controlPlaneURL = strings.TrimSuffix(controlPlaneURL, "/")
-
- payload, err := proto.Marshal(&api.NodeCatalogReport{Services: services})
- if err != nil {
- return fmt.Errorf("failed to encode catalog report: %w", err)
- }
-
- urlStr := controlPlaneURL + "/nodes/catalog"
- req, err := http.NewRequestWithContext(ctx, "POST", urlStr, bytes.NewReader(payload))
- if err != nil {
- return fmt.Errorf("failed to create HTTP request: %w", err)
- }
- req.Header.Set("Content-Type", "application/x-protobuf")
- req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuitToken))
-
- client := controlPlaneHTTPClient(10 * time.Second)
- resp, err := client.Do(req)
- if err != nil {
- return fmt.Errorf("HTTP request failed: %w", err)
- }
- defer resp.Body.Close() //nolint:errcheck
-
- if resp.StatusCode != http.StatusNoContent {
- body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
- return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))
- }
- return nil
+ return controlPlaneClient(controlPlaneURL).ReportCatalog(ctx, biscuitToken, services)
}
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.go
index 10daa5bf..ba31be89 100644
--- a/internal/node/controlplane_sync.go
+++ b/internal/node/controlplane_sync.go
@@ -77,7 +77,7 @@ func (n *SamNode) syncRevocations(ctx context.Context, controlPlaneURL string) e
return nil
}
fetchedAt := time.Now()
- resp, err := controlPlaneClient(controlPlaneURL).FetchRevocations(ctx, token)
+ resp, err := n.controlPlane(controlPlaneURL).FetchRevocations(ctx, token)
if errors.Is(err, cpclient.ErrNotFound) {
return nil
}
diff --git a/internal/node/egress.go b/internal/node/egress.go
index f6592d12..2fcb04c8 100644
--- a/internal/node/egress.go
+++ b/internal/node/egress.go
@@ -355,7 +355,7 @@ func (n *SamNode) syncEgressAssignments(ctx context.Context, controlPlaneURL str
if len(token) == 0 {
return errors.New("node has no identity token to fetch egress assignments")
}
- resp, err := controlPlaneClient(controlPlaneURL).FetchEgress(ctx, token)
+ resp, err := n.controlPlane(controlPlaneURL).FetchEgress(ctx, token)
if errors.Is(err, cpclient.ErrNotFound) {
// A control plane predating egress destinations assigns none; the
// node keeps whatever it serves and does not report an error.
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index 0b7e88cc..fad67434 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -18,12 +18,10 @@ import (
"bytes"
"context"
"crypto/ed25519"
- "encoding/base64"
"fmt"
"io"
"net/http"
"net/url"
- "strconv"
"strings"
"time"
@@ -106,25 +104,6 @@ func (n *SamNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt string, p
if err != nil {
return nil, fmt.Errorf("failed to marshal public key: %w", err)
}
- ts := time.Now().UnixMilli()
- sig, err := privKey.Sign(api.RegisterChallenge(peerID.String(), ts))
- if err != nil {
- return nil, fmt.Errorf("failed to sign registration challenge: %w", err)
- }
-
- req := &api.EnrollRequest{
- Jwt: jwt,
- PeerId: peerID.String(),
- PublicKey: pubBytes,
- RequestedRole: n.config.RequiredRole,
- Labels: n.labels(),
- ChallengeUnixMs: ts,
- ChallengeSignature: sig,
- }
- data, err := proto.Marshal(req)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal enroll request: %v", err)
- }
if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
return nil, fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)
@@ -132,14 +111,32 @@ func (n *SamNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt string, p
url := controlPlaneURL + "/register"
logger.Infof("Enrolling via HTTP at %s", url)
- httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(data))
- if err != nil {
- return nil, fmt.Errorf("failed to create HTTP request: %v", err)
- }
- httpReq.Header.Set("Content-Type", "application/x-protobuf")
-
client := controlPlaneHTTPClient(30 * time.Second)
- resp, err := client.Do(httpReq)
+ resp, err := cpclient.DoWithChallenge(client, time.Now, func(ts int64) (*http.Request, error) {
+ sig, err := privKey.Sign(api.RegisterChallenge(peerID.String(), ts))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign registration challenge: %w", err)
+ }
+ req := &api.EnrollRequest{
+ Jwt: jwt,
+ PeerId: peerID.String(),
+ PublicKey: pubBytes,
+ RequestedRole: n.config.RequiredRole,
+ Labels: n.labels(),
+ ChallengeUnixMs: ts,
+ ChallengeSignature: sig,
+ }
+ data, err := proto.Marshal(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal enroll request: %v", err)
+ }
+ httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(data))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create HTTP request: %v", err)
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ return httpReq, nil
+ })
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %v", err)
}
@@ -258,26 +255,6 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
return fmt.Errorf("failed to marshal public key: %w", err)
}
- enrollTS := time.Now().UnixMilli()
- enrollSig, err := n.config.PrivKey.Sign(api.EnrollChallenge(n.Host.ID().String(), enrollTS))
- if err != nil {
- return fmt.Errorf("failed to sign enrollment challenge: %w", err)
- }
-
- req := &api.BootstrapEnrollRequest{
- BootstrapToken: bootstrapToken,
- PeerId: n.Host.ID().String(),
- PublicKey: pubBytes,
- RequestedRole: n.config.RequiredRole,
- Labels: n.labels(),
- ChallengeUnixMs: enrollTS,
- ChallengeSignature: enrollSig,
- }
- data, err := proto.Marshal(req)
- if err != nil {
- return fmt.Errorf("failed to marshal bootstrap enroll request: %w", err)
- }
-
if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
return fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)
}
@@ -285,13 +262,31 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
logger.Infof("Enrolling via Bootstrap token at %s", enrollURL)
client := controlPlaneHTTPClient(30 * time.Second)
- httpReq, err := http.NewRequestWithContext(ctx, "POST", enrollURL, bytes.NewReader(data))
- if err != nil {
- return fmt.Errorf("failed to create HTTP request: %w", err)
- }
- httpReq.Header.Set("Content-Type", "application/x-protobuf")
-
- resp, err := client.Do(httpReq)
+ resp, err := cpclient.DoWithChallenge(client, time.Now, func(enrollTS int64) (*http.Request, error) {
+ enrollSig, err := n.config.PrivKey.Sign(api.EnrollChallenge(n.Host.ID().String(), enrollTS))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign enrollment challenge: %w", err)
+ }
+ req := &api.BootstrapEnrollRequest{
+ BootstrapToken: bootstrapToken,
+ PeerId: n.Host.ID().String(),
+ PublicKey: pubBytes,
+ RequestedRole: n.config.RequiredRole,
+ Labels: n.labels(),
+ ChallengeUnixMs: enrollTS,
+ ChallengeSignature: enrollSig,
+ }
+ data, err := proto.Marshal(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal bootstrap enroll request: %w", err)
+ }
+ httpReq, err := http.NewRequestWithContext(ctx, "POST", enrollURL, bytes.NewReader(data))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create HTTP request: %w", err)
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ return httpReq, nil
+ })
if err != nil {
return fmt.Errorf("HTTP request failed: %w", err)
}
@@ -339,19 +334,16 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
case <-ticker.C:
// Prove possession of the enrollment key on every poll; the
// control plane returns the biscuit only to the enrollee.
- ts := time.Now().UnixMilli()
- sig, err := n.config.PrivKey.Sign(api.EnrollStatusChallenge(n.Host.ID().String(), ts))
- if err != nil {
- return fmt.Errorf("failed to sign enrollment status challenge: %w", err)
- }
- hReq, err := http.NewRequestWithContext(ctx, http.MethodGet, statusURL, nil)
- if err != nil {
- return fmt.Errorf("failed to create status request: %w", err)
- }
- hReq.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(ts, 10))
- hReq.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(sig))
-
- hResp, err := client.Do(hReq)
+ hResp, err := cpclient.DoWithChallenge(client, time.Now, func(ts int64) (*http.Request, error) {
+ hReq, err := http.NewRequestWithContext(ctx, http.MethodGet, statusURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create status request: %w", err)
+ }
+ if err := cpclient.SetChallengeHeaders(hReq, n.config.PrivKey, api.EnrollStatusChallenge(n.Host.ID().String(), ts), ts); err != nil {
+ return nil, fmt.Errorf("failed to sign enrollment status challenge: %w", err)
+ }
+ return hReq, nil
+ })
if err != nil {
logger.Warnf("Failed to check enrollment status: %v", err)
continue
diff --git a/internal/node/node.go b/internal/node/node.go
index 72e69c46..b7a3fc90 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -1244,16 +1244,11 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
return fmt.Errorf("corrupted private key: %w", err)
}
- // 3. Sign the peer-bound refresh challenge
- timestamp := time.Now().UnixMilli()
+ // 3. Derive peer ID from private key
peerID, err := peer.IDFromPrivateKey(privKey)
if err != nil {
return fmt.Errorf("failed to derive peer ID from private key: %w", err)
}
- sig, err := privKey.Sign(api.RefreshChallenge(peerID.String(), timestamp))
- if err != nil {
- return fmt.Errorf("failed to generate signature: %w", err)
- }
// 4. Construct request. peer_id lets the control plane find this node's
// record when the biscuit's signing key has been retired and the biscuit
@@ -1267,16 +1262,6 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
freshJWT = tok
}
}
- req := &api.TokenRefreshRequest{
- ChallengeSignature: sig,
- ChallengeUnixMs: timestamp,
- PeerId: peerID.String(),
- Jwt: freshJWT,
- }
- reqData, err := proto.Marshal(req)
- if err != nil {
- return fmt.Errorf("failed to marshal request: %w", err)
- }
controlPlaneURL, err := n.Store.LoadControlPlaneURL()
if err != nil {
@@ -1288,17 +1273,31 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
}
url := controlPlaneURL + "/refresh"
- httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(reqData))
- if err != nil {
- return fmt.Errorf("failed to create http request: %w", err)
- }
- httpReq.Header.Set("Content-Type", "application/x-protobuf")
- // Set current biscuit in authorization header
b64Biscuit := base64.StdEncoding.EncodeToString(currentBiscuit)
- httpReq.Header.Set("Authorization", "Bearer "+b64Biscuit)
-
client := controlPlaneHTTPClient(30 * time.Second)
- resp, err := client.Do(httpReq)
+ resp, err := cpclient.DoWithChallenge(client, time.Now, func(timestamp int64) (*http.Request, error) {
+ sig, err := privKey.Sign(api.RefreshChallenge(peerID.String(), timestamp))
+ if err != nil {
+ return nil, fmt.Errorf("failed to generate signature: %w", err)
+ }
+ req := &api.TokenRefreshRequest{
+ ChallengeSignature: sig,
+ ChallengeUnixMs: timestamp,
+ PeerId: peerID.String(),
+ Jwt: freshJWT,
+ }
+ reqData, err := proto.Marshal(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+ httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(reqData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create http request: %w", err)
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ httpReq.Header.Set("Authorization", "Bearer "+b64Biscuit)
+ return httpReq, nil
+ })
if err != nil {
return fmt.Errorf("http request failed: %w", err)
}
@@ -2451,7 +2450,7 @@ func (n *SamNode) syncMeshPolicy(ctx context.Context) error {
return fmt.Errorf("node has no identity token to fetch mesh policy")
}
- policyResp, err := FetchMeshPolicy(ctx, controlPlaneURL, token)
+ policyResp, err := n.controlPlane(controlPlaneURL).FetchPolicy(ctx, token)
if err != nil {
return fmt.Errorf("failed to fetch mesh policy: %w", err)
}
@@ -2490,7 +2489,7 @@ func (n *SamNode) reportNodeCatalog(ctx context.Context) error {
}
services := n.ListLocalServices(api.ServiceType_SERVICE_TYPE_UNSPECIFIED)
- if err := ReportNodeCatalog(ctx, controlPlaneURL, token, services); err != nil {
+ if err := n.controlPlane(controlPlaneURL).ReportCatalog(ctx, token, services); err != nil {
return fmt.Errorf("failed to report node catalog: %w", err)
}
return nil
diff --git a/internal/node/sts.go b/internal/node/sts.go
index b0418c19..abd5f778 100644
--- a/internal/node/sts.go
+++ b/internal/node/sts.go
@@ -177,7 +177,7 @@ func (n *SamNode) ExchangeSubjectJWT(ctx context.Context, subjectToken, subjectT
ChallengeUnixMs: challengeMs,
ChallengeSignature: sig,
}
- resp, err := controlPlaneClient(cpURL).ExchangeToken(ctx, nodeBiscuit, req)
+ resp, err := n.controlPlane(cpURL).ExchangeToken(ctx, nodeBiscuit, req)
if err != nil {
return nil, err
}
@@ -249,7 +249,7 @@ func (n *SamNode) MintBorderJWT(ctx context.Context, callerBiscuit []byte, desti
ChallengeUnixMs: challengeMs,
ChallengeSignature: sig,
}
- resp, err := controlPlaneClient(cpURL).MintSTSToken(ctx, nodeBiscuit, req)
+ resp, err := n.controlPlane(cpURL).MintSTSToken(ctx, nodeBiscuit, req)
if err != nil {
return nil, err
}
diff --git a/internal/router/router.go b/internal/router/router.go
index 727caf6e..c87840e8 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -29,7 +29,6 @@ import (
"os"
"path/filepath"
"runtime/debug"
- "strconv"
"strings"
"sync"
"sync/atomic"
@@ -499,27 +498,32 @@ func (r *Router) enroll(peerID peer.ID) error {
if err != nil {
return fmt.Errorf("failed to marshal public key: %w", err)
}
- ts := time.Now().UnixMilli()
- sig, err := r.privKey.Sign(api.RegisterChallenge(peerID.String(), ts))
- if err != nil {
- return fmt.Errorf("failed to sign registration challenge: %w", err)
- }
-
- req := &api.EnrollRequest{
- Jwt: r.config.OIDCToken,
- PeerId: peerID.String(),
- PublicKey: pubBytes,
- RequestedRole: r.config.RequiredRole,
- ChallengeUnixMs: ts,
- ChallengeSignature: sig,
- }
- data, err := proto.Marshal(req)
- if err != nil {
- return err
- }
client := r.controlPlaneClient(30 * time.Second)
- resp, err := client.Post(r.config.ControlPlaneURL+"/register", "application/x-protobuf", bytes.NewReader(data))
+ resp, err := cpclient.DoWithChallenge(client, time.Now, func(ts int64) (*http.Request, error) {
+ sig, err := r.privKey.Sign(api.RegisterChallenge(peerID.String(), ts))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign registration challenge: %w", err)
+ }
+ req := &api.EnrollRequest{
+ Jwt: r.config.OIDCToken,
+ PeerId: peerID.String(),
+ PublicKey: pubBytes,
+ RequestedRole: r.config.RequiredRole,
+ ChallengeUnixMs: ts,
+ ChallengeSignature: sig,
+ }
+ data, err := proto.Marshal(req)
+ if err != nil {
+ return nil, err
+ }
+ httpReq, err := http.NewRequestWithContext(r.ctx, http.MethodPost, r.config.ControlPlaneURL+"/register", bytes.NewReader(data))
+ if err != nil {
+ return nil, err
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ return httpReq, nil
+ })
if err != nil {
return err
}
@@ -559,27 +563,31 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {
return fmt.Errorf("failed to marshal router public key: %w", err)
}
- enrollTS := time.Now().UnixMilli()
- enrollSig, err := r.privKey.Sign(api.EnrollChallenge(peerID.String(), enrollTS))
- if err != nil {
- return fmt.Errorf("failed to sign enrollment challenge: %w", err)
- }
-
- req := &api.BootstrapEnrollRequest{
- BootstrapToken: r.config.BootstrapToken,
- PeerId: peerID.String(),
- PublicKey: pubBytes,
- RequestedRole: r.config.RequiredRole,
- ChallengeUnixMs: enrollTS,
- ChallengeSignature: enrollSig,
- }
- data, err := proto.Marshal(req)
- if err != nil {
- return err
- }
-
client := r.controlPlaneClient(30 * time.Second)
- resp, err := client.Post(r.config.ControlPlaneURL+"/enroll", "application/x-protobuf", bytes.NewReader(data))
+ resp, err := cpclient.DoWithChallenge(client, time.Now, func(enrollTS int64) (*http.Request, error) {
+ enrollSig, err := r.privKey.Sign(api.EnrollChallenge(peerID.String(), enrollTS))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign enrollment challenge: %w", err)
+ }
+ req := &api.BootstrapEnrollRequest{
+ BootstrapToken: r.config.BootstrapToken,
+ PeerId: peerID.String(),
+ PublicKey: pubBytes,
+ RequestedRole: r.config.RequiredRole,
+ ChallengeUnixMs: enrollTS,
+ ChallengeSignature: enrollSig,
+ }
+ data, err := proto.Marshal(req)
+ if err != nil {
+ return nil, err
+ }
+ httpReq, err := http.NewRequestWithContext(r.ctx, http.MethodPost, r.config.ControlPlaneURL+"/enroll", bytes.NewReader(data))
+ if err != nil {
+ return nil, err
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ return httpReq, nil
+ })
if err != nil {
return err
}
@@ -627,18 +635,16 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {
case <-ticker.C:
// Prove possession of the enrollment key on every poll; the
// control plane returns the biscuit only to the enrollee.
- ts := time.Now().UnixMilli()
- sig, err := r.privKey.Sign(api.EnrollStatusChallenge(peerID.String(), ts))
- if err != nil {
- return fmt.Errorf("failed to sign enrollment status challenge: %w", err)
- }
- req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, statusURL, nil)
- if err != nil {
- return fmt.Errorf("failed to create status request: %w", err)
- }
- req.Header.Set(api.HeaderChallengeTimestamp, strconv.FormatInt(ts, 10))
- req.Header.Set(api.HeaderChallengeSignature, base64.RawURLEncoding.EncodeToString(sig))
- statusResp, err := client.Do(req)
+ statusResp, err := cpclient.DoWithChallenge(client, time.Now, func(ts int64) (*http.Request, error) {
+ req, err := http.NewRequestWithContext(r.ctx, http.MethodGet, statusURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create status request: %w", err)
+ }
+ if err := cpclient.SetChallengeHeaders(req, r.privKey, api.EnrollStatusChallenge(peerID.String(), ts), ts); err != nil {
+ return nil, fmt.Errorf("failed to sign enrollment status challenge: %w", err)
+ }
+ return req, nil
+ })
if err != nil {
logger.Warnf("failed to poll enrollment status: %v", err)
continue
@@ -796,7 +802,19 @@ func (r *Router) syncKeys() error {
// controlPlane reads the pull endpoints of the control plane.
func (r *Router) controlPlane(timeout time.Duration) *cpclient.Client {
- return cpclient.New(r.config.ControlPlaneURL, r.controlPlaneClient(timeout))
+ c := cpclient.New(r.config.ControlPlaneURL, r.controlPlaneClient(timeout))
+ if r != nil && r.privKey != nil {
+ var pid peer.ID
+ if r.Host != nil && r.Host.ID() != "" {
+ pid = r.Host.ID()
+ } else {
+ pid, _ = peer.IDFromPrivateKey(r.privKey)
+ }
+ if pid != "" {
+ c.WithIdentity(pid.String(), r.privKey)
+ }
+ }
+ return c
}
// controlPlaneClient is the client for every request to the control plane;
@@ -984,26 +1002,32 @@ func (r *Router) renewLease() {
// The biscuit identifies us; the signature proves it is us (peers we
// authenticate hold a copy of the biscuit).
- ts := time.Now().UnixMilli()
- sig, err := r.privKey.Sign(api.RouterLeaseChallenge(r.Host.ID().String(), ts))
- if err != nil {
- logger.Errorf("Failed to sign lease challenge: %v", err)
- return
- }
-
- req := &api.RouterLeaseRequest{
- PeerId: r.Host.ID().String(),
- Addresses: addrs,
- Biscuit: biscuit,
- ConnectedPeers: connectedPeers,
- DhtSize: dhtSize,
- ChallengeUnixMs: ts,
- ChallengeSignature: sig,
- }
- data, _ := proto.Marshal(req)
-
client := r.controlPlaneClient(10 * time.Second)
- resp, err := client.Post(r.config.ControlPlaneURL+"/routers/lease", "application/x-protobuf", bytes.NewReader(data))
+ resp, err := cpclient.DoWithChallenge(client, time.Now, func(ts int64) (*http.Request, error) {
+ sig, err := r.privKey.Sign(api.RouterLeaseChallenge(r.Host.ID().String(), ts))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign lease challenge: %w", err)
+ }
+ req := &api.RouterLeaseRequest{
+ PeerId: r.Host.ID().String(),
+ Addresses: addrs,
+ Biscuit: biscuit,
+ ConnectedPeers: connectedPeers,
+ DhtSize: dhtSize,
+ ChallengeUnixMs: ts,
+ ChallengeSignature: sig,
+ }
+ data, err := proto.Marshal(req)
+ if err != nil {
+ return nil, err
+ }
+ httpReq, err := http.NewRequestWithContext(r.ctx, http.MethodPost, r.config.ControlPlaneURL+"/routers/lease", bytes.NewReader(data))
+ if err != nil {
+ return nil, err
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ return httpReq, nil
+ })
if err != nil {
logger.Errorf("Failed to renew lease with control plane: %v", err)
leaseRenewalsTotal.WithLabelValues(leaseUnreachable).Inc()
@@ -1458,49 +1482,51 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {
return fmt.Errorf("router not enrolled (no biscuit)")
}
- // 1. Sign the peer-bound refresh challenge
- timestamp := time.Now().UnixMilli()
+ // 1. Derive peer ID from private key
peerID, err := peer.IDFromPrivateKey(r.privKey)
if err != nil {
return fmt.Errorf("failed to derive peer ID from private key: %w", err)
}
- sig, err := r.privKey.Sign(api.RefreshChallenge(peerID.String(), timestamp))
- if err != nil {
- return fmt.Errorf("failed to generate signature: %w", err)
- }
// 2. Construct request. peer_id lets the control plane find this
// router's record when the biscuit's signing key has been retired and
// the biscuit itself can no longer be verified (autonomous recovery,
// opt-in server-side); it is cross-checked against the biscuit otherwise.
- req := &api.TokenRefreshRequest{
- ChallengeSignature: sig,
- ChallengeUnixMs: timestamp,
- PeerId: peerID.String(),
- }
+ var freshJWT string
if src := r.tokenSource(); src != nil {
if jwt, err := src.FetchToken(ctx); err == nil && jwt != "" {
- req.Jwt = jwt
+ freshJWT = jwt
} else if err != nil {
logger.Warnf("Failed to fetch platform JWT for router refresh re-attestation, continuing with PoP-only refresh: %v", err)
}
}
- reqData, err := proto.Marshal(req)
- if err != nil {
- return fmt.Errorf("failed to marshal request: %w", err)
- }
url := r.config.ControlPlaneURL + "/refresh"
- httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(reqData))
- if err != nil {
- return fmt.Errorf("failed to create http request: %w", err)
- }
- httpReq.Header.Set("Content-Type", "application/x-protobuf")
b64Biscuit := base64.StdEncoding.EncodeToString(currentBiscuit)
- httpReq.Header.Set("Authorization", "Bearer "+b64Biscuit)
-
client := r.controlPlaneClient(10 * time.Second)
- resp, err := client.Do(httpReq)
+ resp, err := cpclient.DoWithChallenge(client, time.Now, func(timestamp int64) (*http.Request, error) {
+ sig, err := r.privKey.Sign(api.RefreshChallenge(peerID.String(), timestamp))
+ if err != nil {
+ return nil, fmt.Errorf("failed to generate signature: %w", err)
+ }
+ req := &api.TokenRefreshRequest{
+ ChallengeSignature: sig,
+ ChallengeUnixMs: timestamp,
+ PeerId: peerID.String(),
+ Jwt: freshJWT,
+ }
+ reqData, err := proto.Marshal(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+ httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(reqData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create http request: %w", err)
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ httpReq.Header.Set("Authorization", "Bearer "+b64Biscuit)
+ return httpReq, nil
+ })
if err != nil {
return fmt.Errorf("http request failed: %w", err)
}
diff --git a/sdk/README.md b/sdk/README.md
index d5ae0648..399778fc 100644
--- a/sdk/README.md
+++ b/sdk/README.md
@@ -224,16 +224,22 @@ messages in `api/sam.proto`. Bodies are capped at 1 MiB on both sides.
| `POST /register` | `EnrollRequest` | `EnrollResponse` | sign `sam:register:<peer_id>:<ts>` |
| `POST /refresh` | `TokenRefreshRequest`, header `Authorization: Bearer <base64 biscuit>` | `TokenRefreshResponse` | sign `sam:refresh:<peer_id>:<ts>` |
| `GET /keys` | — | `KeysResponse` | none; see below |
-| `GET /policies` | header `Authorization: Bearer <base64 biscuit>` | `PolicyConfigGetResponse{datalog_rules}` | none; the biscuit must belong to an admitted node |
-| `POST /nodes/catalog` | `NodeCatalogReport`, header `Authorization: Bearer <base64 biscuit>` | `204` | none; the reporting peer is read from the biscuit. `sam-node` reports what it publishes; an SDK member publishes nothing and does not call it |
+| `GET /policies` | headers `Authorization: Bearer <base64 biscuit>`, `X-Sam-Challenge-Ts`, `X-Sam-Challenge-Sig` | `PolicyConfigGetResponse{datalog_rules}` | sign `sam:policies:<peer_id>:<ts>` |
+| `GET /egress` | headers `Authorization: Bearer <base64 biscuit>`, `X-Sam-Challenge-Ts`, `X-Sam-Challenge-Sig` | `EgressAssignmentsResponse` | sign `sam:egress:<peer_id>:<ts>` (`sam-node` only) |
+| `GET /revocations` | headers `Authorization: Bearer <base64 biscuit>`, `X-Sam-Challenge-Ts`, `X-Sam-Challenge-Sig` | `RevocationsResponse` | sign `sam:revocations:<peer_id>:<ts>` (`sam-node` only) |
+| `POST /nodes/catalog` | `NodeCatalogReport`, headers `Authorization: Bearer <base64 biscuit>`, `X-Sam-Challenge-Ts`, `X-Sam-Challenge-Sig` | `204` | sign `sam:nodes-catalog:<peer_id>:<ts>`; `sam-node` reports what it publishes, while an SDK member publishes nothing and does not call it |
- `<ts>` is the request's `challenge_unix_ms`, unix milliseconds, and must
- be within 5 minutes of the control plane's clock (`challengeMaxAge`).
+ be within 5 minutes of the control plane's clock (`challengeMaxAge`). When
+ the control plane answers `401` with `"stale or invalid challenge timestamp"`,
+ a client reads the response's `Date` header (`Access-Control-Expose-Headers: Date`),
+ recomputes `<ts>` from it, and retries once; other `401`s are not retried.
- The endpoints above answer a CORS preflight and mark their responses for
any origin (`Access-Control-Allow-Origin: *`), so a page on another origin
can call them. They authenticate by what the request carries, a token in
- the body or a biscuit as a bearer, never by a cookie. The operator plane
- (`/admin/*`, `/user/*`) and `/routers/lease` do not.
+ the body or a biscuit as a bearer plus the signed challenge, never by a
+ cookie. The operator plane (`/admin/*`, `/user/*`) and `/routers/lease` do
+ not.
Challenges are defined in `api/network.go`. It is the one instant on the
wire that is an `int64`: it is the number in the signed text. Every other
instant (`expire_time`, `sign_time`, `event_time`, `announce_time`) is a
diff --git a/sdk/js/src/challenges.ts b/sdk/js/src/challenges.ts
index becccd32..72768ce1 100644
--- a/sdk/js/src/challenges.ts
+++ b/sdk/js/src/challenges.ts
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-// Proof-of-possession challenges on the control plane's enrollment surface.
+// Proof-of-possession challenges on the control plane's mesh surface.
// Each payload names the peer and the endpoint, so a captured signature
// verifies nowhere else. Mirrors api/network.go; ts is unix milliseconds.
@@ -44,3 +44,24 @@ export function registerChallenge(peerId: string, ts: number): Uint8Array {
export function refreshChallenge(peerId: string, ts: number): Uint8Array {
return challenge("refresh", peerId, ts);
}
+
+/** Signed at GET /policies. */
+export function policiesChallenge(peerId: string, ts: number): Uint8Array {
+ return challenge("policies", peerId, ts);
+}
+
+/** Signed at GET /egress. */
+export function egressChallenge(peerId: string, ts: number): Uint8Array {
+ return challenge("egress", peerId, ts);
+}
+
+/** Signed at GET /revocations. */
+export function revocationsChallenge(peerId: string, ts: number): Uint8Array {
+ return challenge("revocations", peerId, ts);
+}
+
+/** Signed at POST /nodes/catalog. */
+export function nodesCatalogChallenge(peerId: string, ts: number): Uint8Array {
+ return challenge("nodes-catalog", peerId, ts);
+}
+
diff --git a/sdk/js/src/controlplane.test.ts b/sdk/js/src/controlplane.test.ts
index 06408150..06a68c98 100644
--- a/sdk/js/src/controlplane.test.ts
+++ b/sdk/js/src/controlplane.test.ts
@@ -21,7 +21,16 @@ import { create, fromBinary, toBinary } from "@bufbuild/protobuf";
import { timestampFromMs } from "@bufbuild/protobuf/wkt";
import assert from "node:assert/strict";
import { test } from "node:test";
-import { enrollChallenge, enrollStatusChallenge, refreshChallenge, registerChallenge } from "./challenges.ts";
+import {
+ egressChallenge,
+ enrollChallenge,
+ enrollStatusChallenge,
+ nodesCatalogChallenge,
+ policiesChallenge,
+ refreshChallenge,
+ registerChallenge,
+ revocationsChallenge,
+} from "./challenges.ts";
import {
ControlPlaneClient,
ControlPlaneError,
@@ -31,6 +40,7 @@ import {
InsecureControlPlaneURLError,
KEYS_RESPONSE_FRESHNESS_MS,
ROLE_NODE,
+ STALE_CHALLENGE_TIMESTAMP_MESSAGE,
validateControlPlaneURL,
verifyKeysResponse,
} from "./controlplane.ts";
@@ -41,6 +51,7 @@ import {
EnrollResponseSchema,
EnrollmentStatus,
KeysResponseSchema,
+ PolicyConfigGetResponseSchema,
TokenRefreshRequestSchema,
TokenRefreshResponseSchema,
type KeysResponse,
@@ -230,6 +241,93 @@ test("refresh presents the biscuit as a bearer and signs the refresh challenge",
await assert.rejects(replayed.refresh({ identity: id, biscuit }), (err: unknown) => err instanceof ControlPlaneError && err.status === 401);
});
+test("policyRules presents the biscuit and signs the policies challenge", async () => {
+ const id = Identity.generate();
+ const rules = ['granted_service("mcp", "echo") <- role("sam:role:node")'];
+ const client = new ControlPlaneClient({
+ url: "http://127.0.0.1:1",
+ fetch: fakeFetch({
+ "GET /policies": (req) => {
+ assert.equal(req.headers.get("authorization"), `Bearer ${Buffer.from(biscuit).toString("base64")}`);
+ const ts = Number(req.headers.get(HEADER_CHALLENGE_TIMESTAMP));
+ const sig = new Uint8Array(Buffer.from(req.headers.get(HEADER_CHALLENGE_SIGNATURE) ?? "", "base64url"));
+ assert.ok(verifyEd25519(id.publicKeyRaw, policiesChallenge(id.peerId, ts), sig));
+ return proto(toBinary(PolicyConfigGetResponseSchema, create(PolicyConfigGetResponseSchema, { datalogRules: rules })));
+ },
+ }),
+ });
+ assert.deepEqual(await client.policyRules(id, biscuit), rules);
+ assert.deepEqual(await client.policyRules({ identity: id, biscuit }), rules);
+
+ // Domain-separated challenge payloads match api/network.go.
+ const dec = new TextDecoder();
+ assert.equal(dec.decode(policiesChallenge("peer1", 42)), "sam:policies:peer1:42");
+ assert.equal(dec.decode(egressChallenge("peer1", 42)), "sam:egress:peer1:42");
+ assert.equal(dec.decode(revocationsChallenge("peer1", 42)), "sam:revocations:peer1:42");
+ assert.equal(dec.decode(nodesCatalogChallenge("peer1", 42)), "sam:nodes-catalog:peer1:42");
+});
+
+test("clock skew: +10 min client clock retries once using response Date header; other 401s are not retried", async () => {
+ const id = Identity.generate();
+ const serverNowSec = Math.floor(Date.now() / 1000);
+ const serverNowMs = serverNowSec * 1000;
+ const skewedClientNowMs = serverNowMs + 10 * 60 * 1000;
+ const dateHeader = new Date(serverNowMs).toUTCString();
+ const fresh = new TextEncoder().encode("fresher-biscuit");
+ const rules = ['granted_service("mcp", "echo") <- true'];
+
+ let refreshCalls = 0;
+ let policyCalls = 0;
+ const skewedClient = new ControlPlaneClient({
+ url: "http://127.0.0.1:1",
+ now: () => skewedClientNowMs,
+ fetch: fakeFetch({
+ "POST /refresh": (_req, body) => {
+ refreshCalls++;
+ const r = fromBinary(TokenRefreshRequestSchema, body);
+ const ts = Number(r.challengeUnixMs);
+ assert.ok(verifyEd25519(id.publicKeyRaw, refreshChallenge(id.peerId, ts), r.challengeSignature));
+ if (Math.abs(serverNowMs - ts) > 5 * 60 * 1000) {
+ return new Response(STALE_CHALLENGE_TIMESTAMP_MESSAGE, { status: 401, headers: { Date: dateHeader } });
+ }
+ return proto(toBinary(TokenRefreshResponseSchema, create(TokenRefreshResponseSchema, { biscuitToken: fresh, expireTime: timestampFromMs(99_000) })));
+ },
+ "GET /policies": (req) => {
+ policyCalls++;
+ const ts = Number(req.headers.get(HEADER_CHALLENGE_TIMESTAMP));
+ const sig = new Uint8Array(Buffer.from(req.headers.get(HEADER_CHALLENGE_SIGNATURE) ?? "", "base64url"));
+ assert.ok(verifyEd25519(id.publicKeyRaw, policiesChallenge(id.peerId, ts), sig));
+ if (Math.abs(serverNowMs - ts) > 5 * 60 * 1000) {
+ return new Response(STALE_CHALLENGE_TIMESTAMP_MESSAGE, { status: 401, headers: { Date: dateHeader } });
+ }
+ return proto(toBinary(PolicyConfigGetResponseSchema, create(PolicyConfigGetResponseSchema, { datalogRules: rules })));
+ },
+ }),
+ });
+
+ const refreshed = await skewedClient.refresh({ identity: id, biscuit });
+ assert.deepEqual(refreshed.biscuit, fresh);
+ assert.equal(refreshCalls, 2);
+
+ assert.deepEqual(await skewedClient.policyRules(id, biscuit), rules);
+ assert.equal(policyCalls, 2);
+
+ // Other 401s (even with a Date header) are not retried.
+ let nonStaleCalls = 0;
+ const nonStaleClient = new ControlPlaneClient({
+ url: "http://127.0.0.1:1",
+ now: () => skewedClientNowMs,
+ fetch: fakeFetch({
+ "GET /policies": () => {
+ nonStaleCalls++;
+ return new Response("invalid challenge signature", { status: 401, headers: { Date: dateHeader } });
+ },
+ }),
+ });
+ await assert.rejects(nonStaleClient.policyRules(id, biscuit), (err: unknown) => err instanceof ControlPlaneError && err.status === 401);
+ assert.equal(nonStaleCalls, 1);
+});
+
test("verifyKeysResponse accepts a set vouched for by a trusted key and nothing else", () => {
const retiring = Identity.generate();
const resp = signedKeys([cpKey, retiring]);
@@ -269,3 +367,4 @@ test("an injected fetch is called unbound, as a browser's window.fetch requires"
await client.keys([cpKey.publicKeyRaw]);
assert.equal(receiver, undefined);
});
+
diff --git a/sdk/js/src/controlplane.ts b/sdk/js/src/controlplane.ts
index 8cbf51b7..36cb8826 100644
--- a/sdk/js/src/controlplane.ts
+++ b/sdk/js/src/controlplane.ts
@@ -17,7 +17,7 @@
import { create, fromBinary, toBinary } from "@bufbuild/protobuf";
import { timestampMs, type Timestamp } from "@bufbuild/protobuf/wkt";
-import { enrollChallenge, enrollStatusChallenge, refreshChallenge, registerChallenge } from "./challenges.ts";
+import { enrollChallenge, enrollStatusChallenge, policiesChallenge, refreshChallenge, registerChallenge } from "./challenges.ts";
import {
BootstrapEnrollRequestSchema,
BootstrapEnrollResponseSchema,
@@ -40,6 +40,7 @@ import { verifyEd25519 } from "./identity.ts";
export const PROTOBUF_CONTENT_TYPE = "application/x-protobuf";
export const HEADER_CHALLENGE_TIMESTAMP = "X-Sam-Challenge-Ts";
export const HEADER_CHALLENGE_SIGNATURE = "X-Sam-Challenge-Sig";
+export const STALE_CHALLENGE_TIMESTAMP_MESSAGE = "stale or invalid challenge timestamp";
/** The role a plain mesh member enrolls with (api.RoleNode). */
export const ROLE_NODE = "sam:role:node";
@@ -100,6 +101,8 @@ export interface ControlPlaneClientOptions {
timeoutMs?: number;
/** Injection point for tests. Defaults to the global fetch. */
fetch?: typeof fetch;
+ /** Injection point for clock-skew tests; returns unix milliseconds. Defaults to Date.now. */
+ now?: () => number;
}
/** What an approved enrollment hands the caller. */
@@ -138,6 +141,11 @@ export interface RefreshParams {
jwt?: string | undefined;
}
+export interface PolicyRulesParams {
+ identity: Identity;
+ biscuit: Uint8Array;
+}
+
export interface RefreshResult {
biscuit: Uint8Array;
/** Unix seconds. */
@@ -237,6 +245,7 @@ export class ControlPlaneClient {
readonly url: URL;
readonly #fetch: typeof fetch;
readonly #timeoutMs: number;
+ readonly #now: () => number;
constructor(options: ControlPlaneClientOptions) {
this.url = validateControlPlaneURL(options.url, options.allowInsecure ?? false);
@@ -244,6 +253,7 @@ export class ControlPlaneClient {
const f = options.fetch ?? globalThis.fetch;
this.#fetch = (input, init) => f(input, init);
this.#timeoutMs = options.timeoutMs ?? 30_000;
+ this.#now = options.now ?? (() => Date.now());
}
/** GET /info: OIDC settings, router addresses and the ban list. Unauthenticated. */
@@ -268,17 +278,19 @@ export class ControlPlaneClient {
async enrollBootstrap(params: EnrollBootstrapParams): Promise<Enrollment> {
const { identity, signal } = params;
const role = params.role ?? ROLE_NODE;
- const ts = Date.now();
- const req = create(BootstrapEnrollRequestSchema, {
- bootstrapToken: params.bootstrapToken,
- peerId: identity.peerId,
- publicKey: identity.libp2pPublicKey,
- requestedRole: role,
- labels: params.labels ?? {},
- challengeUnixMs: BigInt(ts),
- challengeSignature: identity.sign(enrollChallenge(identity.peerId, ts)),
+ const raw = await this.#requestWithChallenge("POST", "/enroll", (ts) => {
+ const req = create(BootstrapEnrollRequestSchema, {
+ bootstrapToken: params.bootstrapToken,
+ peerId: identity.peerId,
+ publicKey: identity.libp2pPublicKey,
+ requestedRole: role,
+ labels: params.labels ?? {},
+ challengeUnixMs: BigInt(ts),
+ challengeSignature: identity.sign(enrollChallenge(identity.peerId, ts)),
+ });
+ return { body: toBinary(BootstrapEnrollRequestSchema, req) };
});
- let resp = fromBinary(BootstrapEnrollResponseSchema, await this.#request("POST", "/enroll", toBinary(BootstrapEnrollRequestSchema, req)));
+ let resp = fromBinary(BootstrapEnrollResponseSchema, raw);
while (resp.status === EnrollmentStatus.PENDING) {
const waitMs = params.pollIntervalMs ?? Math.max(1, resp.pollIntervalSeconds) * 1000;
@@ -289,29 +301,38 @@ export class ControlPlaneClient {
}
async #enrollStatus(identity: Identity): Promise<BootstrapEnrollResponse> {
- const ts = Date.now();
- const sig = identity.sign(enrollStatusChallenge(identity.peerId, ts));
- const body = await this.#request("GET", `/enroll/status?peer_id=${encodeURIComponent(identity.peerId)}`, undefined, {
- [HEADER_CHALLENGE_TIMESTAMP]: String(ts),
- [HEADER_CHALLENGE_SIGNATURE]: toBase64Url(sig),
- });
+ const body = await this.#requestWithChallenge(
+ "GET",
+ `/enroll/status?peer_id=${encodeURIComponent(identity.peerId)}`,
+ (ts) => {
+ const sig = identity.sign(enrollStatusChallenge(identity.peerId, ts));
+ return {
+ headers: {
+ [HEADER_CHALLENGE_TIMESTAMP]: String(ts),
+ [HEADER_CHALLENGE_SIGNATURE]: toBase64Url(sig),
+ },
+ };
+ },
+ );
return fromBinary(BootstrapEnrollResponseSchema, body);
}
/** POST /register with an OIDC ID token. */
async register(params: RegisterParams): Promise<Enrollment> {
const { identity } = params;
- const ts = Date.now();
- const req = create(EnrollRequestSchema, {
- jwt: params.jwt,
- peerId: identity.peerId,
- publicKey: identity.libp2pPublicKey,
- requestedRole: params.role ?? ROLE_NODE,
- labels: params.labels ?? {},
- challengeUnixMs: BigInt(ts),
- challengeSignature: identity.sign(registerChallenge(identity.peerId, ts)),
+ const raw = await this.#requestWithChallenge("POST", "/register", (ts) => {
+ const req = create(EnrollRequestSchema, {
+ jwt: params.jwt,
+ peerId: identity.peerId,
+ publicKey: identity.libp2pPublicKey,
+ requestedRole: params.role ?? ROLE_NODE,
+ labels: params.labels ?? {},
+ challengeUnixMs: BigInt(ts),
+ challengeSignature: identity.sign(registerChallenge(identity.peerId, ts)),
+ });
+ return { body: toBinary(EnrollRequestSchema, req) };
});
- const resp = fromBinary(EnrollResponseSchema, await this.#request("POST", "/register", toBinary(EnrollRequestSchema, req)));
+ const resp = fromBinary(EnrollResponseSchema, raw);
if (resp.errorMessage) {
throw new EnrollmentRejectedError(`enrollment failed: ${resp.errorMessage}`);
}
@@ -328,15 +349,18 @@ export class ControlPlaneClient {
*/
async refresh(params: RefreshParams): Promise<RefreshResult> {
const { identity } = params;
- const ts = Date.now();
- const req = create(TokenRefreshRequestSchema, {
- challengeUnixMs: BigInt(ts),
- challengeSignature: identity.sign(refreshChallenge(identity.peerId, ts)),
- peerId: identity.peerId,
- ...(params.jwt ? { jwt: params.jwt } : {}),
- });
- const body = await this.#request("POST", "/refresh", toBinary(TokenRefreshRequestSchema, req), {
- Authorization: `Bearer ${toBase64(params.biscuit)}`,
+ const authHeader = `Bearer ${toBase64(params.biscuit)}`;
+ const body = await this.#requestWithChallenge("POST", "/refresh", (ts) => {
+ const req = create(TokenRefreshRequestSchema, {
+ challengeUnixMs: BigInt(ts),
+ challengeSignature: identity.sign(refreshChallenge(identity.peerId, ts)),
+ peerId: identity.peerId,
+ ...(params.jwt ? { jwt: params.jwt } : {}),
+ });
+ return {
+ body: toBinary(TokenRefreshRequestSchema, req),
+ headers: { Authorization: authHeader },
+ };
});
const resp = fromBinary(TokenRefreshResponseSchema, body);
if (resp.errorMessage) {
@@ -353,9 +377,21 @@ export class ControlPlaneClient {
* its authorizer, one per entry, rendered by the control plane. The text
* is the contract; nothing here derives rules from roles and bindings.
*/
- async policyRules(biscuit: Uint8Array): Promise<string[]> {
- const body = await this.#request("GET", "/policies", undefined, {
- Authorization: `Bearer ${toBase64(biscuit)}`,
+ async policyRules(identity: Identity, biscuit: Uint8Array): Promise<string[]>;
+ async policyRules(params: PolicyRulesParams): Promise<string[]>;
+ async policyRules(identityOrParams: Identity | PolicyRulesParams, maybeBiscuit?: Uint8Array): Promise<string[]> {
+ const identity = "identity" in identityOrParams ? identityOrParams.identity : identityOrParams;
+ const biscuit = "identity" in identityOrParams ? identityOrParams.biscuit : (maybeBiscuit as Uint8Array);
+ const authHeader = `Bearer ${toBase64(biscuit)}`;
+ const body = await this.#requestWithChallenge("GET", "/policies", (ts) => {
+ const sig = identity.sign(policiesChallenge(identity.peerId, ts));
+ return {
+ headers: {
+ Authorization: authHeader,
+ [HEADER_CHALLENGE_TIMESTAMP]: String(ts),
+ [HEADER_CHALLENGE_SIGNATURE]: toBase64Url(sig),
+ },
+ };
});
const resp = fromBinary(PolicyConfigGetResponseSchema, body);
if (resp.$unknown !== undefined && resp.$unknown.length > 0) {
@@ -364,7 +400,33 @@ export class ControlPlaneClient {
return resp.datalogRules;
}
- async #request(method: "GET" | "POST", path: string, body?: Uint8Array, headers: Record<string, string> = {}): Promise<Uint8Array> {
+ async #requestWithChallenge(
+ method: "GET" | "POST",
+ path: string,
+ build: (ts: number) => { body?: Uint8Array; headers?: Record<string, string> },
+ ): Promise<Uint8Array> {
+ const first = build(this.#now());
+ const outcome = await this.#fetchOnce(method, path, first.body, first.headers);
+ if (outcome.ok) {
+ return outcome.buf;
+ }
+ const text = new TextDecoder().decode(outcome.buf);
+ if (outcome.status === 401 && text.includes(STALE_CHALLENGE_TIMESTAMP_MESSAGE) && outcome.dateHeader) {
+ const serverMs = Date.parse(outcome.dateHeader);
+ if (Number.isFinite(serverMs) && serverMs > 0) {
+ const retry = build(serverMs);
+ return this.#request(method, path, retry.body, retry.headers);
+ }
+ }
+ throw new ControlPlaneError(path, outcome.status, text);
+ }
+
+ async #fetchOnce(
+ method: "GET" | "POST",
+ path: string,
+ body?: Uint8Array,
+ headers: Record<string, string> = {},
+ ): Promise<{ ok: boolean; status: number; buf: Uint8Array; dateHeader: string | null }> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(new Error(`control plane ${path}: timed out after ${this.#timeoutMs}ms`)), this.#timeoutMs);
try {
@@ -382,14 +444,19 @@ export class ControlPlaneClient {
if (buf.length > MAX_RESPONSE_BYTES) {
throw new Error(`control plane ${path}: response of ${buf.length} bytes exceeds the ${MAX_RESPONSE_BYTES} byte limit`);
}
- if (!resp.ok) {
- throw new ControlPlaneError(path, resp.status, new TextDecoder().decode(buf));
- }
- return buf;
+ return { ok: resp.ok, status: resp.status, buf, dateHeader: resp.headers.get("date") };
} finally {
clearTimeout(timer);
}
}
+
+ async #request(method: "GET" | "POST", path: string, body?: Uint8Array, headers: Record<string, string> = {}): Promise<Uint8Array> {
+ const outcome = await this.#fetchOnce(method, path, body, headers);
+ if (!outcome.ok) {
+ throw new ControlPlaneError(path, outcome.status, new TextDecoder().decode(outcome.buf));
+ }
+ return outcome.buf;
+ }
}
function enrollmentFromBootstrapResponse(resp: BootstrapEnrollResponse): Enrollment {
diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts
index 4ff81249..ebb306d6 100644
--- a/sdk/js/src/index.ts
+++ b/sdk/js/src/index.ts
@@ -26,6 +26,7 @@ export {
type ControlPlaneClientOptions,
type Enrollment,
type EnrollBootstrapParams,
+ type PolicyRulesParams,
type RegisterParams,
type RefreshParams,
type RefreshResult,
@@ -41,7 +42,16 @@ export {
withCredentialMethods,
type MeshCredential,
} from "./credential.ts";
-export { enrollChallenge, enrollStatusChallenge, refreshChallenge, registerChallenge } from "./challenges.ts";
+export {
+ egressChallenge,
+ enrollChallenge,
+ enrollStatusChallenge,
+ nodesCatalogChallenge,
+ policiesChallenge,
+ refreshChallenge,
+ registerChallenge,
+ revocationsChallenge,
+} from "./challenges.ts";
export { MeshSession, type AdmittedRouter, type DiscoveredProvider, type JoinOptions, type Peer, type ToolCallResult } from "./session.ts";
export {
BiscuitVerificationError,
diff --git a/sdk/js/src/session.ts b/sdk/js/src/session.ts
index 3a2e8662..d59db2a3 100644
--- a/sdk/js/src/session.ts
+++ b/sdk/js/src/session.ts
@@ -685,7 +685,7 @@ export class MeshSession {
/** Re-reads the mesh policy from the control plane. */
async syncPolicy(): Promise<void> {
- this.#policyRules = await this.mesh.controlPlane.policyRules(this.mesh.credential.biscuit);
+ this.#policyRules = await this.mesh.controlPlane.policyRules(this.mesh.identity, this.mesh.credential.biscuit);
}
/**
diff --git a/sdk/python/src/agent_mesh/__init__.py b/sdk/python/src/agent_mesh/__init__.py
index 2e78b04c..6c8cb371 100644
--- a/sdk/python/src/agent_mesh/__init__.py
+++ b/sdk/python/src/agent_mesh/__init__.py
@@ -17,7 +17,16 @@
from .auth import AUTH_PROTOCOL, MCP_PROTOCOL, AuthRejectedError, auth_stream_handler, authenticate_with_peer
from .authorizer import BASELINE_DATALOG, AuthorizationError, AuthorizeRequest, ProviderAuthorizerOptions, authorize_caller
from .biscuit import ROLE_ROUTER, BiscuitVerificationError, VerifiedBiscuit, attenuate_biscuit, require_role, seal_biscuit, verify_peer_biscuit
-from .challenges import enroll_challenge, enroll_status_challenge, refresh_challenge, register_challenge
+from .challenges import (
+ egress_challenge,
+ enroll_challenge,
+ enroll_status_challenge,
+ nodes_catalog_challenge,
+ policies_challenge,
+ refresh_challenge,
+ register_challenge,
+ revocations_challenge,
+)
from .controlplane import (
ROLE_NODE,
ControlPlaneClient,
@@ -131,6 +140,7 @@ __all__ = [
"decode_tar_block_payload",
"dial_through_relay",
"effective_tar_expiration",
+ "egress_challenge",
"encode_auth_frame",
"encode_tar_block_fact",
"encode_tar_block_payload",
@@ -146,17 +156,20 @@ __all__ = [
"match_task_rule",
"mesh_http_target",
"mesh_url",
+ "nodes_catalog_challenge",
"open_http_request",
"open_mcp_session",
"parse_service_target",
"parse_tar_block_source",
"peer_id_from_public_key",
+ "policies_challenge",
"refresh_challenge",
"register_challenge",
"require_egress_labels",
"require_labels",
"require_role",
"reserve_relay",
+ "revocations_challenge",
"seal_biscuit",
"service_key",
"rewrite_agent_card",
@@ -171,3 +184,4 @@ __all__ = [
"verify_mesh_event",
"verify_peer_biscuit",
]
+
diff --git a/sdk/python/src/agent_mesh/challenges.py b/sdk/python/src/agent_mesh/challenges.py
index ff03477f..fc1b38c2 100644
--- a/sdk/python/src/agent_mesh/challenges.py
+++ b/sdk/python/src/agent_mesh/challenges.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Proof-of-possession challenges on the control plane's enrollment surface.
+"""Proof-of-possession challenges on the control plane's mesh surface.
Each payload names the peer and the endpoint, so a captured signature
verifies nowhere else. Mirrors api/network.go; ts is unix milliseconds."""
@@ -41,3 +41,24 @@ def register_challenge(peer_id: str, ts: int) -> bytes:
def refresh_challenge(peer_id: str, ts: int) -> bytes:
"""Signed at POST /refresh."""
return _challenge("refresh", peer_id, ts)
+
+
+def policies_challenge(peer_id: str, ts: int) -> bytes:
+ """Signed at GET /policies."""
+ return _challenge("policies", peer_id, ts)
+
+
+def egress_challenge(peer_id: str, ts: int) -> bytes:
+ """Signed at GET /egress."""
+ return _challenge("egress", peer_id, ts)
+
+
+def revocations_challenge(peer_id: str, ts: int) -> bytes:
+ """Signed at GET /revocations."""
+ return _challenge("revocations", peer_id, ts)
+
+
+def nodes_catalog_challenge(peer_id: str, ts: int) -> bytes:
+ """Signed at POST /nodes/catalog."""
+ return _challenge("nodes-catalog", peer_id, ts)
+
diff --git a/sdk/python/src/agent_mesh/controlplane.py b/sdk/python/src/agent_mesh/controlplane.py
index 01ec5aa8..76f9b87a 100644
--- a/sdk/python/src/agent_mesh/controlplane.py
+++ b/sdk/python/src/agent_mesh/controlplane.py
@@ -18,6 +18,7 @@
from __future__ import annotations
import base64
+import email.utils
import ipaddress
import threading
import time
@@ -25,7 +26,7 @@ import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
-from typing import Callable, Mapping, Optional, Sequence
+from typing import Callable, Mapping, Optional, Sequence, Union
from google.protobuf.timestamp_pb2 import Timestamp
from google.protobuf.unknown_fields import UnknownFieldSet
@@ -37,6 +38,7 @@ from .identity import PUBLIC_KEY_SIZE, Identity, verify_ed25519
PROTOBUF_CONTENT_TYPE = "application/x-protobuf"
HEADER_CHALLENGE_TIMESTAMP = "X-Sam-Challenge-Ts"
HEADER_CHALLENGE_SIGNATURE = "X-Sam-Challenge-Sig"
+STALE_CHALLENGE_TIMESTAMP_MESSAGE = "stale or invalid challenge timestamp"
# The role a plain mesh member enrolls with (api.RoleNode).
ROLE_NODE = "sam:role:node"
@@ -46,8 +48,9 @@ KEYS_RESPONSE_FRESHNESS_MS = 5 * 60 * 1000
_MAX_RESPONSE_BYTES = 1024 * 1024
-# (method, url, headers, body) -> (status, body). Injection point for tests.
-Transport = Callable[[str, str, Mapping[str, str], Optional[bytes]], "tuple[int, bytes]"]
+# (method, url, headers, body) -> (status, body[, response_headers]). Injection point for tests.
+TransportResult = Union[tuple[int, bytes], tuple[int, bytes, Mapping[str, str]]]
+Transport = Callable[[str, str, Mapping[str, str], Optional[bytes]], TransportResult]
class ControlPlaneError(Exception):
@@ -152,17 +155,34 @@ def verify_keys_response(resp: pb.KeysResponse, trusted: Sequence[bytes], now_ms
def _urllib_transport(timeout: float) -> Transport:
- def send(method: str, url: str, headers: Mapping[str, str], body: Optional[bytes]) -> tuple[int, bytes]:
+ def send(method: str, url: str, headers: Mapping[str, str], body: Optional[bytes]) -> tuple[int, bytes, Mapping[str, str]]:
req = urllib.request.Request(url, data=body, method=method, headers=dict(headers))
try:
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 - scheme validated by the client
- return resp.status, resp.read(_MAX_RESPONSE_BYTES + 1)
+ return resp.status, resp.read(_MAX_RESPONSE_BYTES + 1), dict(resp.headers)
except urllib.error.HTTPError as err:
- return err.code, err.read(_MAX_RESPONSE_BYTES + 1)
+ resp_headers = dict(err.headers) if err.headers is not None else {}
+ return err.code, err.read(_MAX_RESPONSE_BYTES + 1), resp_headers
return send
+def _parse_http_date_ms(headers: Mapping[str, str]) -> Optional[int]:
+ date_value: Optional[str] = None
+ for k, v in headers.items():
+ if k.lower() == "date":
+ date_value = v
+ break
+ if not date_value:
+ return None
+ try:
+ dt = email.utils.parsedate_to_datetime(date_value)
+ except (TypeError, ValueError):
+ return None
+ ms = int(dt.timestamp() * 1000)
+ return ms if ms > 0 else None
+
+
class ControlPlaneClient:
def __init__(
self,
@@ -171,9 +191,11 @@ class ControlPlaneClient:
allow_insecure: bool = False,
timeout: float = 30.0,
transport: Optional[Transport] = None,
+ now_ms: Optional[Callable[[], int]] = None,
):
self.url = validate_control_plane_url(url, allow_insecure)
self._transport = transport or _urllib_transport(timeout)
+ self._now_ms = now_ms or _now_ms
def info(self) -> pb.ControlPlaneInfoResponse:
"""GET /info: OIDC settings, router addresses and the ban list. Unauthenticated."""
@@ -197,17 +219,23 @@ class ControlPlaneClient:
"""POST /enroll with a bootstrap token, then GET /enroll/status until an
operator approves the request if the mesh is not on auto-approve.
`cancel` bounds that wait; set it to stop polling."""
- ts = _now_ms()
- req = pb.BootstrapEnrollRequest(
- bootstrap_token=bootstrap_token,
- peer_id=identity.peer_id,
- public_key=identity.libp2p_public_key,
- requested_role=role,
- labels=dict(labels or {}),
- challenge_unix_ms=ts,
- challenge_signature=identity.sign(challenges.enroll_challenge(identity.peer_id, ts)),
+ raw = self._request_with_challenge(
+ "POST",
+ "/enroll",
+ lambda ts: (
+ pb.BootstrapEnrollRequest(
+ bootstrap_token=bootstrap_token,
+ peer_id=identity.peer_id,
+ public_key=identity.libp2p_public_key,
+ requested_role=role,
+ labels=dict(labels or {}),
+ challenge_unix_ms=ts,
+ challenge_signature=identity.sign(challenges.enroll_challenge(identity.peer_id, ts)),
+ ).SerializeToString(),
+ None,
+ ),
)
- resp = pb.BootstrapEnrollResponse.FromString(self._request("POST", "/enroll", req.SerializeToString()))
+ resp = pb.BootstrapEnrollResponse.FromString(raw)
while resp.status == pb.ENROLLMENT_STATUS_PENDING:
wait = poll_interval if poll_interval is not None else max(1, resp.poll_interval_seconds)
if cancel is not None:
@@ -219,15 +247,21 @@ class ControlPlaneClient:
return _enrollment_from_bootstrap_response(resp)
def _enroll_status(self, identity: Identity) -> pb.BootstrapEnrollResponse:
- ts = _now_ms()
- sig = identity.sign(challenges.enroll_status_challenge(identity.peer_id, ts))
- body = self._request(
+ path = "/enroll/status?" + urllib.parse.urlencode({"peer_id": identity.peer_id})
+ body = self._request_with_challenge(
"GET",
- "/enroll/status?" + urllib.parse.urlencode({"peer_id": identity.peer_id}),
- headers={
- HEADER_CHALLENGE_TIMESTAMP: str(ts),
- HEADER_CHALLENGE_SIGNATURE: base64.urlsafe_b64encode(sig).rstrip(b"=").decode(),
- },
+ path,
+ lambda ts: (
+ None,
+ {
+ HEADER_CHALLENGE_TIMESTAMP: str(ts),
+ HEADER_CHALLENGE_SIGNATURE: base64.urlsafe_b64encode(
+ identity.sign(challenges.enroll_status_challenge(identity.peer_id, ts))
+ )
+ .rstrip(b"=")
+ .decode(),
+ },
+ ),
)
return pb.BootstrapEnrollResponse.FromString(body)
@@ -240,17 +274,23 @@ class ControlPlaneClient:
labels: Optional[Mapping[str, str]] = None,
) -> Enrollment:
"""POST /register with an OIDC ID token."""
- ts = _now_ms()
- req = pb.EnrollRequest(
- jwt=jwt,
- peer_id=identity.peer_id,
- public_key=identity.libp2p_public_key,
- requested_role=role,
- labels=dict(labels or {}),
- challenge_unix_ms=ts,
- challenge_signature=identity.sign(challenges.register_challenge(identity.peer_id, ts)),
+ raw = self._request_with_challenge(
+ "POST",
+ "/register",
+ lambda ts: (
+ pb.EnrollRequest(
+ jwt=jwt,
+ peer_id=identity.peer_id,
+ public_key=identity.libp2p_public_key,
+ requested_role=role,
+ labels=dict(labels or {}),
+ challenge_unix_ms=ts,
+ challenge_signature=identity.sign(challenges.register_challenge(identity.peer_id, ts)),
+ ).SerializeToString(),
+ None,
+ ),
)
- resp = pb.EnrollResponse.FromString(self._request("POST", "/register", req.SerializeToString()))
+ resp = pb.EnrollResponse.FromString(raw)
if resp.error_message:
raise EnrollmentRejectedError(f"enrollment failed: {resp.error_message}")
return _checked_enrollment(
@@ -263,18 +303,19 @@ class ControlPlaneClient:
def refresh(self, identity: Identity, biscuit: bytes, *, jwt: Optional[str] = None) -> RefreshResult:
"""POST /refresh: trades the biscuit for a fresh one. The old one is
spent by this call; callers must persist the result before using it."""
- ts = _now_ms()
- req = pb.TokenRefreshRequest(
- challenge_unix_ms=ts,
- challenge_signature=identity.sign(challenges.refresh_challenge(identity.peer_id, ts)),
- peer_id=identity.peer_id,
- jwt=jwt or "",
- )
- body = self._request(
+ auth_header = "Bearer " + base64.b64encode(biscuit).decode()
+ body = self._request_with_challenge(
"POST",
"/refresh",
- req.SerializeToString(),
- headers={"Authorization": "Bearer " + base64.b64encode(biscuit).decode()},
+ lambda ts: (
+ pb.TokenRefreshRequest(
+ challenge_unix_ms=ts,
+ challenge_signature=identity.sign(challenges.refresh_challenge(identity.peer_id, ts)),
+ peer_id=identity.peer_id,
+ jwt=jwt or "",
+ ).SerializeToString(),
+ {"Authorization": auth_header},
+ ),
)
resp = pb.TokenRefreshResponse.FromString(body)
if resp.error_message:
@@ -285,23 +326,72 @@ class ControlPlaneClient:
raise ValueError("refresh response carries no expire_time")
return RefreshResult(biscuit=resp.biscuit_token, expiration=resp.expire_time.ToSeconds())
- def policy_rules(self, biscuit: bytes) -> list[str]:
+ def policy_rules(self, identity: Identity, biscuit: bytes) -> list[str]:
"""GET /policies: the mesh policy as the Datalog rules a provider adds
to its authorizer, one per entry, rendered by the control plane. The
text is the contract; nothing here derives rules from roles and bindings."""
- body = self._request("GET", "/policies", headers={"Authorization": "Bearer " + base64.b64encode(biscuit).decode()})
+ auth_header = "Bearer " + base64.b64encode(biscuit).decode()
+ body = self._request_with_challenge(
+ "GET",
+ "/policies",
+ lambda ts: (
+ None,
+ {
+ "Authorization": auth_header,
+ HEADER_CHALLENGE_TIMESTAMP: str(ts),
+ HEADER_CHALLENGE_SIGNATURE: base64.urlsafe_b64encode(
+ identity.sign(challenges.policies_challenge(identity.peer_id, ts))
+ )
+ .rstrip(b"=")
+ .decode(),
+ },
+ ),
+ )
resp = pb.PolicyConfigGetResponse.FromString(body)
if len(UnknownFieldSet(resp)) > 0:
raise ValueError("control plane predates datalog_rules in its policy response; upgrade the control plane")
return list(resp.datalog_rules)
- def _request(self, method: str, path: str, body: Optional[bytes] = None, headers: Optional[Mapping[str, str]] = None) -> bytes:
+ def _request_with_challenge(
+ self,
+ method: str,
+ path: str,
+ build: Callable[[int], tuple[Optional[bytes], Optional[Mapping[str, str]]]],
+ ) -> bytes:
+ first_body, first_headers = build(self._now_ms())
+ status, data, resp_headers = self._fetch_once(method, path, first_body, first_headers)
+ if 200 <= status < 300:
+ return data
+ text = data.decode("utf-8", "replace")
+ if status == 401 and STALE_CHALLENGE_TIMESTAMP_MESSAGE in text:
+ server_ms = _parse_http_date_ms(resp_headers)
+ if server_ms is not None:
+ retry_body, retry_headers = build(server_ms)
+ return self._request(method, path, retry_body, retry_headers)
+ raise ControlPlaneError(path, status, data)
+
+ def _fetch_once(
+ self,
+ method: str,
+ path: str,
+ body: Optional[bytes] = None,
+ headers: Optional[Mapping[str, str]] = None,
+ ) -> tuple[int, bytes, Mapping[str, str]]:
all_headers = {"Accept": PROTOBUF_CONTENT_TYPE, **(headers or {})}
if body is not None:
all_headers["Content-Type"] = PROTOBUF_CONTENT_TYPE
- status, data = self._transport(method, self.url + path, all_headers, body)
+ res = self._transport(method, self.url + path, all_headers, body)
+ if len(res) == 3:
+ status, data, resp_headers = res
+ else:
+ status, data = res
+ resp_headers = {}
if len(data) > _MAX_RESPONSE_BYTES:
raise ValueError(f"control plane {path}: response of {len(data)} bytes exceeds the {_MAX_RESPONSE_BYTES} byte limit")
+ return status, data, resp_headers
+
+ def _request(self, method: str, path: str, body: Optional[bytes] = None, headers: Optional[Mapping[str, str]] = None) -> bytes:
+ status, data, _ = self._fetch_once(method, path, body, headers)
if not 200 <= status < 300:
raise ControlPlaneError(path, status, data)
return data
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index af6b4221..14411a1a 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -434,7 +434,7 @@ class MeshSession:
async def sync_policy(self) -> None:
"""Re-reads the mesh policy from the control plane."""
- self._policy_rules = await trio.to_thread.run_sync(self.mesh.control_plane.policy_rules, self.mesh.credential.biscuit)
+ self._policy_rules = await trio.to_thread.run_sync(self.mesh.control_plane.policy_rules, self.mesh.identity, self.mesh.credential.biscuit)
async def sync(self) -> "ControlPlaneSync":
"""Pulls keys, bans and router addresses from the control plane now, and
diff --git a/sdk/python/tests/test_controlplane.py b/sdk/python/tests/test_controlplane.py
index d38e7eea..11a2f6a2 100644
--- a/sdk/python/tests/test_controlplane.py
+++ b/sdk/python/tests/test_controlplane.py
@@ -18,6 +18,7 @@ and how it treats every answer. The real one is exercised by
tests/integration/sdk_enroll_test.go."""
import base64
+import email.utils
import threading
import time
import urllib.parse
@@ -31,6 +32,7 @@ from agent_mesh.controlplane import (
HEADER_CHALLENGE_TIMESTAMP,
KEYS_RESPONSE_FRESHNESS_MS,
ROLE_NODE,
+ STALE_CHALLENGE_TIMESTAMP_MESSAGE,
ControlPlaneClient,
ControlPlaneError,
EnrollmentRejectedError,
@@ -58,7 +60,7 @@ BISCUIT = b"not-really-a-biscuit"
def fake_transport(routes):
- """routes: {"METHOD /path": handler(url_parts, headers, body) -> (status, bytes)}"""
+ """routes: {"METHOD /path": handler(url_parts, headers, body) -> (status, bytes[, resp_headers])}"""
def send(method, url, headers, body):
parts = urllib.parse.urlsplit(url)
@@ -214,6 +216,83 @@ def test_refresh_presents_bearer_biscuit_and_signs_refresh_challenge():
assert excinfo.value.status == 401
+def test_policy_rules_presents_biscuit_and_signs_policies_challenge():
+ identity = Identity.generate()
+ rules = ['granted_service("mcp", "echo") <- role("sam:role:node")']
+
+ def policies(parts, headers, body):
+ assert headers["Authorization"] == "Bearer " + base64.b64encode(BISCUIT).decode()
+ ts = int(headers[HEADER_CHALLENGE_TIMESTAMP])
+ sig = base64.urlsafe_b64decode(headers[HEADER_CHALLENGE_SIGNATURE] + "==")
+ assert verify_ed25519(identity.public_key_raw, challenges.policies_challenge(identity.peer_id, ts), sig)
+ return 200, pb.PolicyConfigGetResponse(datalog_rules=rules).SerializeToString()
+
+ client = ControlPlaneClient("http://127.0.0.1:1", transport=fake_transport({"GET /policies": policies}))
+ assert client.policy_rules(identity, BISCUIT) == rules
+
+ assert challenges.policies_challenge("peer1", 42) == b"sam:policies:peer1:42"
+ assert challenges.egress_challenge("peer1", 42) == b"sam:egress:peer1:42"
+ assert challenges.revocations_challenge("peer1", 42) == b"sam:revocations:peer1:42"
+ assert challenges.nodes_catalog_challenge("peer1", 42) == b"sam:nodes-catalog:peer1:42"
+
+
+def test_clock_skew_retries_once_using_response_date_header_and_does_not_retry_other_401s():
+ identity = Identity.generate()
+ server_now_sec = int(time.time())
+ server_now_ms = server_now_sec * 1000
+ skewed_client_now_ms = server_now_ms + 10 * 60 * 1000
+ date_header = email.utils.formatdate(server_now_sec, usegmt=True)
+ rules = ['granted_service("mcp", "echo") <- true']
+
+ refresh_calls = []
+ policy_calls = []
+
+ def refresh(parts, headers, body):
+ refresh_calls.append(1)
+ r = pb.TokenRefreshRequest.FromString(body)
+ assert verify_ed25519(identity.public_key_raw, challenges.refresh_challenge(identity.peer_id, r.challenge_unix_ms), r.challenge_signature)
+ if abs(server_now_ms - r.challenge_unix_ms) > 5 * 60 * 1000:
+ return 401, STALE_CHALLENGE_TIMESTAMP_MESSAGE.encode(), {"Date": date_header}
+ return 200, pb.TokenRefreshResponse(biscuit_token=b"fresher-biscuit", expire_time=_ts_s(99)).SerializeToString()
+
+ def policies(parts, headers, body):
+ policy_calls.append(1)
+ ts = int(headers[HEADER_CHALLENGE_TIMESTAMP])
+ sig = base64.urlsafe_b64decode(headers[HEADER_CHALLENGE_SIGNATURE] + "==")
+ assert verify_ed25519(identity.public_key_raw, challenges.policies_challenge(identity.peer_id, ts), sig)
+ if abs(server_now_ms - ts) > 5 * 60 * 1000:
+ return 401, STALE_CHALLENGE_TIMESTAMP_MESSAGE.encode(), {"Date": date_header}
+ return 200, pb.PolicyConfigGetResponse(datalog_rules=rules).SerializeToString()
+
+ skewed_client = ControlPlaneClient(
+ "http://127.0.0.1:1",
+ transport=fake_transport({"POST /refresh": refresh, "GET /policies": policies}),
+ now_ms=lambda: skewed_client_now_ms,
+ )
+ refreshed = skewed_client.refresh(identity, BISCUIT)
+ assert refreshed.biscuit == b"fresher-biscuit"
+ assert len(refresh_calls) == 2
+
+ assert skewed_client.policy_rules(identity, BISCUIT) == rules
+ assert len(policy_calls) == 2
+
+ non_stale_calls = []
+
+ def bad_sig(parts, headers, body):
+ non_stale_calls.append(1)
+ return 401, b"invalid challenge signature", {"Date": date_header}
+
+ non_stale_client = ControlPlaneClient(
+ "http://127.0.0.1:1",
+ transport=fake_transport({"GET /policies": bad_sig}),
+ now_ms=lambda: skewed_client_now_ms,
+ )
+ with pytest.raises(ControlPlaneError) as excinfo:
+ non_stale_client.policy_rules(identity, BISCUIT)
+ assert excinfo.value.status == 401
+ assert len(non_stale_calls) == 1
+
+
def test_verify_keys_response_accepts_only_a_set_vouched_for_by_a_trusted_key():
retiring = Identity.generate()
resp = signed_keys([CP_KEY, retiring])
@@ -244,3 +323,4 @@ def test_keys_fetches_and_verifies_against_the_enrollment_key():
assert client.keys([CP_KEY.public_key_raw]) == [CP_KEY.public_key_raw]
with pytest.raises(ValueError, match="not signed by any trusted"):
client.keys([Identity.generate().public_key_raw])
+
diff --git a/site/content/docs/contributing/security-architecture.md b/site/content/docs/contributing/security-architecture.md
index f18c6c67..7362e733 100644
--- a/site/content/docs/contributing/security-architecture.md
+++ b/site/content/docs/contributing/security-architecture.md
@@ -351,8 +351,8 @@ as a standard OIDC issuer:
| Surface | Mechanism | What it binds | Replay defense |
| :--- | :--- | :--- | :--- |
| **libp2p data path** (node-to-node, node-to-router) | Channel binding: Block 0 `client_peer_id` must equal the `connection_peer_id` authenticated by the libp2p Noise or TLS handshake. | The transport private key | None needed; the token is unusable on any other connection. |
-| **Control plane mesh protocol** (`/enroll`, `/enroll/status`, `/register`, `/refresh`, `/routers/lease`, `/token/exchange`, `/sts/token`) | Ed25519 signature over `sam:<endpoint>:<peer_id>:<challenge_unix_ms>`, carried in protobuf fields (`challenge_unix_ms`, `challenge_signature`) or `X-Sam-Challenge-Ts` / `X-Sam-Challenge-Sig` on `GET /enroll/status`. | Endpoint name, peer ID, timestamp | 5-minute freshness window (`challengeMaxAge`); `/refresh` additionally redeems only the last Biscuit issued to the peer. |
-| **Control plane read & report endpoints** (`/policies`, `/egress`, `/revocations`, `/nodes/catalog`) | Bearer Member Biscuit of an admitted node (`admittedNode`). | Enrolled node credential | Bounded by Biscuit TTL and ban/revocation list. |
+| **Control plane mesh protocol** (`/enroll`, `/enroll/status`, `/register`, `/refresh`, `/routers/lease`, `/token/exchange`, `/sts/token`) | Ed25519 signature over `sam:<endpoint>:<peer_id>:<challenge_unix_ms>`, carried in protobuf fields (`challenge_unix_ms`, `challenge_signature`) or `X-Sam-Challenge-Ts` / `X-Sam-Challenge-Sig` on `GET /enroll/status`. | Endpoint name, peer ID, timestamp | 5-minute freshness window (`challengeMaxAge`) with one-shot `Date`-header clock-skew recovery on `401`; `/refresh` additionally redeems only the last Biscuit issued to the peer. |
+| **Control plane read & report endpoints** (`GET /policies`, `GET /egress`, `GET /revocations`, `POST /nodes/catalog`) | Bearer Member Biscuit of an admitted node (`admittedNodeWithChallenge`) plus `X-Sam-Challenge-Ts` / `X-Sam-Challenge-Sig` signed over `sam:<endpoint>:<peer_id>:<ts>` (`policies`, `egress`, `revocations`, `nodes-catalog`), verified against the node's stored Ed25519 public key via `verifyFreshChallenge`. | Enrolled node credential, endpoint name, peer ID, timestamp | 5-minute freshness window (`challengeMaxAge`) with one-shot `Date`-header clock-skew recovery on `401`, bounded by Biscuit TTL and ban/revocation list. |
| **OAuth 2.1 external client surface** (`/oauth/authorize`, `/oauth/token`, `/mcp` over HTTPS) | Authorization Code + PKCE (`S256`) on the code grant. The `client_peer_id` of the minted Biscuit is the calling node's when a node calls, otherwise the `actor_peer_id` form parameter or the control plane's own peer ID (`resolveDefaultActorPeer`). | A claimed peer ID (the Biscuit is a bearer token on the HTTPS `/mcp` hop unless sender-constrained with DPoP; see §5.2) | Single-use authorization codes with PKCE verifier check on the code exchange; none on the resulting bearer Biscuit. |
---
@@ -498,19 +498,23 @@ sequenceDiagram
### 5.1 Nonce and body-hash binding on control-plane HTTP requests
-`POST /enroll`, `GET /enroll/status`, `POST /register`, `POST /refresh`,
-`POST /token/exchange`, and `POST /sts/token` verify an Ed25519 signature over
-`sam:<endpoint>:<peer_id>:<challenge_unix_ms>` within a 5-minute clock window
-(`challengeMaxAge`), while `/policies`, `/egress`, `/revocations`, and
-`/nodes/catalog` authenticate the caller via its bearer Member Biscuit.
-
-Because the signed text does not yet carry a server-tracked random nonce or
-request-body SHA-256 hash, TLS (`https://`) is mandatory for every remote
-control plane (`AllowInsecureControlPlane` is restricted to loopback or
-explicitly trusted networks). Extending the proof message with a random nonce,
-an optional server nonce for skewed clocks, and a request-body SHA-256 hash across
-all credentialed control-plane endpoints is a self-contained future hardening
-step.
+Every authenticated mesh-protocol endpoint on the control plane (`POST /enroll`,
+`GET /enroll/status`, `POST /register`, `POST /refresh`, `POST /routers/lease`,
+`POST /token/exchange`, `POST /sts/token`, `GET /policies`, `GET /egress`,
+`GET /revocations`, and `POST /nodes/catalog`) verifies an Ed25519 signature
+over `sam:<endpoint>:<peer_id>:<challenge_unix_ms>` within a 5-minute clock
+window (`challengeMaxAge`). When a client's clock has drifted outside that
+window, the control plane answers `401` (`"stale or invalid challenge timestamp"`)
+with the standard HTTP `Date` header (`Access-Control-Expose-Headers: Date` on
+CORS surfaces), and clients in Go, TypeScript, and Python recompute
+`challenge_unix_ms` from `Date` and retry once.
+
+Because the signed text does not carry a server-tracked random nonce or a
+request-body SHA-256 hash, TLS end-to-end to the control plane (`https://`, with
+`AllowInsecureControlPlane` restricted to loopback or explicitly trusted
+networks) is the transport boundary, and replay within the 5-minute freshness
+window is accepted on that basis. `/refresh` additionally enforces single-use
+redemption of the last Biscuit issued to the peer.
### 5.2 Optional DPoP (RFC 9449) at the OAuth 2.1 border