Skip to content

Agent Mesh

client: an oversized control plane answer is an error, never a prefix

Antonio Ojea7296a5c054d1
diff --git a/internal/controlplane/client/client.go b/internal/controlplane/client/client.go
index 0479f202..6f61260b 100644
--- a/internal/controlplane/client/client.go
+++ b/internal/controlplane/client/client.go
@@ -21,6 +21,8 @@ package client
import (
"context"
"crypto/ed25519"
+ "encoding/base64"
+ "errors"
"fmt"
"io"
"net/http"
@@ -34,8 +36,29 @@ import (
// MaxBodyBytes caps every response body read from a control plane: a
// misbehaving or impersonated server must not be able to make a client
-// buffer arbitrary amounts of memory.
-const MaxBodyBytes = 1 << 20
+// buffer arbitrary amounts of memory. It is sized for the largest legitimate
+// answer, the ban set in /info at roughly 55 bytes per peer ID, so about
+// 150k banned peers fit; the policy is bounded by the control plane's own
+// 1 MiB cap on POST /policies, and /keys is a few hundred bytes.
+const MaxBodyBytes = 8 << 20
+
+// ErrBodyTooLarge marks an answer over MaxBodyBytes. It is an error, never a
+// prefix: a protobuf message cut at a field boundary still decodes, so a
+// truncated ban set or router list would be read as a smaller, valid one.
+var ErrBodyTooLarge = errors.New("control plane answer exceeds the body cap")
+
+// ReadBody reads a control plane response body of at most MaxBodyBytes and
+// reports ErrBodyTooLarge for anything larger.
+func ReadBody(r io.Reader) ([]byte, error) {
+ body, err := io.ReadAll(io.LimitReader(r, MaxBodyBytes+1))
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+ if len(body) > MaxBodyBytes {
+ return nil, fmt.Errorf("%w (%d bytes)", ErrBodyTooLarge, MaxBodyBytes)
+ }
+ return body, nil
+}
// transport applies api.ValidateControlPlaneTransport to every request,
// redirects included, so a plaintext hop is refused wherever the URL came
@@ -81,7 +104,7 @@ func New(baseURL string, httpClient *http.Client) *Client {
// details a node needs to enroll.
func (c *Client) FetchInfo(ctx context.Context) (*api.ControlPlaneInfoResponse, error) {
var info api.ControlPlaneInfoResponse
- if err := c.get(ctx, "/info", &info); err != nil {
+ if err := c.get(ctx, "/info", nil, &info); err != nil {
return nil, err
}
return &info, nil
@@ -93,7 +116,7 @@ func (c *Client) FetchInfo(ctx context.Context) (*api.ControlPlaneInfoResponse,
// control plane, not become it.
func (c *Client) FetchKeys(ctx context.Context, trusted []ed25519.PublicKey) ([]ed25519.PublicKey, error) {
var resp api.KeysResponse
- if err := c.get(ctx, "/keys", &resp); err != nil {
+ if err := c.get(ctx, "/keys", nil, &resp); err != nil {
return nil, err
}
keys, err := api.VerifyKeysResponse(&resp, trusted, time.Now())
@@ -103,20 +126,33 @@ func (c *Client) FetchKeys(ctx context.Context, trusted []ed25519.PublicKey) ([]
return keys, nil
}
-func (c *Client) get(ctx context.Context, path string, msg proto.Message) error {
+// FetchPolicy is GET /policies, authenticated with the caller's biscuit: the
+// roles and bindings a node compiles into its authorization rules.
+func (c *Client) FetchPolicy(ctx context.Context, biscuit []byte) (*api.PolicyConfigGetResponse, error) {
+ var policy api.PolicyConfigGetResponse
+ if err := c.get(ctx, "/policies", biscuit, &policy); err != nil {
+ return nil, err
+ }
+ return &policy, nil
+}
+
+func (c *Client) get(ctx context.Context, path string, biscuit []byte, msg proto.Message) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
if err != nil {
return fmt.Errorf("failed to create HTTP request: %w", err)
}
+ if len(biscuit) > 0 {
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuit))
+ }
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("HTTP request failed: %w", err)
}
defer func() { _ = resp.Body.Close() }()
- body, err := io.ReadAll(io.LimitReader(resp.Body, MaxBodyBytes))
+ body, err := ReadBody(resp.Body)
if err != nil {
- return fmt.Errorf("failed to read response body: %w", err)
+ return fmt.Errorf("%s: %w", path, err)
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))
diff --git a/internal/controlplane/client/client_test.go b/internal/controlplane/client/client_test.go
index e8e46a6c..ef2689e0 100644
--- a/internal/controlplane/client/client_test.go
+++ b/internal/controlplane/client/client_test.go
@@ -18,6 +18,7 @@ import (
"context"
"crypto/ed25519"
"crypto/rand"
+ "encoding/base64"
"errors"
"net/http"
"net/http/httptest"
@@ -156,20 +157,107 @@ func TestErrors(t *testing.T) {
}
})
- t.Run("an oversized body is cut at the cap, not buffered", func(t *testing.T) {
+ t.Run("an oversized body is an error, not a shorter message", func(t *testing.T) {
+ // A ban set cut at an entry boundary would decode as a valid, smaller
+ // ban set; the client must refuse the answer instead.
+ big := &api.ControlPlaneInfoResponse{}
+ for len(big.BannedPeerIds) < 200_000 {
+ big.BannedPeerIds = append(big.BannedPeerIds, "12D3KooWL7xNnc7bdzPGobhPxvunLC1uNhHfSCK8ZTXjR1YSTG9T")
+ }
+ if n := proto.Size(big); n <= MaxBodyBytes {
+ t.Fatalf("test answer is %d bytes, need more than %d", n, MaxBodyBytes)
+ }
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if _, err := w.Write(make([]byte, MaxBodyBytes+1)); err != nil {
- t.Errorf("write: %v", err)
- }
+ writeProto(t, w, big)
}))
defer srv.Close()
- c := New(srv.URL, NewHTTPClient(time.Second, nil))
- if _, err := c.FetchInfo(context.Background()); err == nil {
- t.Fatal("a body over the cap must not decode into a valid answer")
+ c := New(srv.URL, NewHTTPClient(5*time.Second, nil))
+ info, err := c.FetchInfo(context.Background())
+ if !errors.Is(err, ErrBodyTooLarge) {
+ t.Fatalf("FetchInfo = (%d bans, %v), want ErrBodyTooLarge", len(info.GetBannedPeerIds()), err)
+ }
+ })
+}
+
+// The cap must never cut a legitimate answer: a large ban set well inside it
+// arrives whole, and a body of exactly the cap still decodes.
+func TestLargeAnswersArriveWhole(t *testing.T) {
+ const bans = 100_000
+ big := &api.ControlPlaneInfoResponse{RouterAddresses: []string{"/ip4/10.0.0.1/tcp/4501"}}
+ for i := 0; i < bans; i++ {
+ big.BannedPeerIds = append(big.BannedPeerIds, "12D3KooWL7xNnc7bdzPGobhPxvunLC1uNhHfSCK8ZTXjR1YSTG9T")
+ }
+ if n := proto.Size(big); n >= MaxBodyBytes {
+ t.Fatalf("%d bans is %d bytes, over the %d cap: the cap is too small for a mesh this size", bans, n, MaxBodyBytes)
+ }
+
+ t.Run("large ban set", func(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ writeProto(t, w, big)
+ }))
+ defer srv.Close()
+ info, err := New(srv.URL, NewHTTPClient(5*time.Second, nil)).FetchInfo(context.Background())
+ if err != nil {
+ t.Fatalf("FetchInfo: %v", err)
+ }
+ if got := len(info.BannedPeerIds); got != bans {
+ t.Fatalf("FetchInfo returned %d bans, want %d: the answer was cut", got, bans)
+ }
+ })
+
+ t.Run("body of exactly the cap", func(t *testing.T) {
+ // Pad the router address so the encoded message is exactly MaxBodyBytes.
+ exact := &api.ControlPlaneInfoResponse{RouterAddresses: []string{""}}
+ pad := MaxBodyBytes - proto.Size(exact) - 3 // 3: the length prefix grows to three bytes
+ exact.RouterAddresses[0] = strings.Repeat("a", pad)
+ for proto.Size(exact) < MaxBodyBytes {
+ exact.RouterAddresses[0] += "a"
+ }
+ if n := proto.Size(exact); n != MaxBodyBytes {
+ t.Fatalf("test body is %d bytes, want exactly %d", n, MaxBodyBytes)
+ }
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ writeProto(t, w, exact)
+ }))
+ defer srv.Close()
+ info, err := New(srv.URL, NewHTTPClient(5*time.Second, nil)).FetchInfo(context.Background())
+ if err != nil {
+ t.Fatalf("FetchInfo at exactly the cap: %v", err)
+ }
+ if len(info.RouterAddresses) != 1 || len(info.RouterAddresses[0]) != len(exact.RouterAddresses[0]) {
+ t.Fatal("body of exactly the cap did not arrive whole")
}
})
}
+func TestFetchPolicy(t *testing.T) {
+ want := &api.PolicyConfigGetResponse{
+ Roles: []*api.PolicyRole{{Name: "developer", AllowedServices: []string{"mcp://*"}}},
+ Bindings: []*api.PolicyBinding{{Role: "developer", Members: []string{"group:eng"}}},
+ }
+ var gotAuth string
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotAuth = r.Header.Get("Authorization")
+ if r.URL.Path != "/policies" {
+ http.NotFound(w, r)
+ return
+ }
+ writeProto(t, w, want)
+ }))
+ defer srv.Close()
+
+ policy, err := New(srv.URL, NewHTTPClient(time.Second, nil)).FetchPolicy(context.Background(), []byte("biscuit"))
+ if err != nil {
+ t.Fatalf("FetchPolicy: %v", err)
+ }
+ if !proto.Equal(policy, want) {
+ t.Errorf("FetchPolicy = %v, want %v", policy, want)
+ }
+ if gotAuth != "Bearer "+base64.StdEncoding.EncodeToString([]byte("biscuit")) {
+ t.Errorf("Authorization = %q, want the biscuit as a base64 bearer token", gotAuth)
+ }
+}
+
// The control plane is the trust root, so a plaintext hop to it is refused
// unless the operator opted in; loopback is the standalone case and is fine.
// The choice is read per request, since the node learns it after its clients
diff --git a/internal/node/controlplane.go b/internal/node/controlplane.go
index 086437d4..99540802 100644
--- a/internal/node/controlplane.go
+++ b/internal/node/controlplane.go
@@ -32,7 +32,9 @@ import (
// maxControlPlaneBodyBytes caps every response body read from the control
// plane or an IdP: a misbehaving or impersonated server must not be able to
-// make the node buffer arbitrary amounts of memory.
+// make the node buffer arbitrary amounts of memory. Bodies that carry a
+// message go through cpclient.ReadBody, which turns an oversized answer into
+// an error rather than a truncated message.
const maxControlPlaneBodyBytes = cpclient.MaxBodyBytes
// controlPlaneClient speaks the pull endpoints of controlPlaneURL through the
@@ -83,41 +85,7 @@ func publicKeysOf(keys []TrustedKey) []ed25519.PublicKey {
// FetchMeshPolicy retrieves the latest mesh policy from the control plane's /policies endpoint using a biscuit token.
func FetchMeshPolicy(ctx context.Context, controlPlaneURL string, biscuitToken []byte) (*api.PolicyConfigGetResponse, error) {
- if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
- controlPlaneURL = "https://" + controlPlaneURL
- }
- controlPlaneURL = strings.TrimSuffix(controlPlaneURL, "/")
-
- urlStr := controlPlaneURL + "/policies"
- req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
- if err != nil {
- return nil, fmt.Errorf("failed to create HTTP request: %w", err)
- }
-
- req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(biscuitToken))
-
- client := controlPlaneHTTPClient(10 * time.Second)
- resp, err := client.Do(req)
- if err != nil {
- return nil, fmt.Errorf("HTTP request failed: %w", err)
- }
- defer resp.Body.Close() //nolint:errcheck
-
- body, err := io.ReadAll(io.LimitReader(resp.Body, maxControlPlaneBodyBytes))
- if err != nil {
- return nil, fmt.Errorf("failed to read response body: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))
- }
-
- var policyResp api.PolicyConfigGetResponse
- if err := proto.Unmarshal(body, &policyResp); err != nil {
- return nil, fmt.Errorf("failed to decode /policies response: %w", err)
- }
-
- return &policyResp, nil
+ return controlPlaneClient(controlPlaneURL).FetchPolicy(ctx, biscuitToken)
}
// ReportNodeCatalog self-reports this node's locally registered services to
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index b1f3efd7..c113a3bb 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -28,6 +28,7 @@ import (
"time"
"github.com/google/sam/api"
+ cpclient "github.com/google/sam/internal/controlplane/client"
"github.com/google/sam/internal/identity"
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p/core/crypto"
@@ -171,9 +172,9 @@ func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollRespons
return nil, fmt.Errorf("enrollment failed with status %s: %s", resp.Status, string(body))
}
- respData, err := io.ReadAll(io.LimitReader(resp.Body, maxControlPlaneBodyBytes))
+ respData, err := cpclient.ReadBody(resp.Body)
if err != nil {
- return nil, fmt.Errorf("failed to read response body: %v", err)
+ return nil, fmt.Errorf("enrollment response: %w", err)
}
var enrollResp api.EnrollResponse
@@ -289,9 +290,9 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
return fmt.Errorf("enrollment failed with status %s: %s", resp.Status, string(body))
}
- respData, err := io.ReadAll(io.LimitReader(resp.Body, maxControlPlaneBodyBytes))
+ respData, err := cpclient.ReadBody(resp.Body)
if err != nil {
- return fmt.Errorf("failed to read response body: %w", err)
+ return fmt.Errorf("bootstrap enrollment response: %w", err)
}
enrollResp := &api.BootstrapEnrollResponse{}
@@ -351,10 +352,10 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
continue
}
- hRespData, err := io.ReadAll(io.LimitReader(hResp.Body, maxControlPlaneBodyBytes))
+ hRespData, err := cpclient.ReadBody(hResp.Body)
_ = hResp.Body.Close()
if err != nil {
- logger.Warnf("Failed to read status response body: %v", err)
+ logger.Warnf("Enrollment status response: %v", err)
continue
}
diff --git a/internal/node/node.go b/internal/node/node.go
index 6fa60b8e..e726f118 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -37,6 +37,7 @@ import (
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/google/sam/api"
+ cpclient "github.com/google/sam/internal/controlplane/client"
"github.com/google/sam/internal/identity"
samdiscovery "github.com/google/sam/internal/node/discovery"
"github.com/google/sam/internal/ratelimit"
@@ -1178,9 +1179,9 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
}
}
- respData, err := io.ReadAll(io.LimitReader(resp.Body, maxControlPlaneBodyBytes))
+ respData, err := cpclient.ReadBody(resp.Body)
if err != nil {
- return fmt.Errorf("failed to read response: %w", err)
+ return fmt.Errorf("refresh response: %w", err)
}
var refreshResp api.TokenRefreshResponse
diff --git a/internal/router/router.go b/internal/router/router.go
index 3a441d7b..b2364023 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -444,9 +444,9 @@ func (r *Router) enroll(peerID peer.ID) error {
return fmt.Errorf("enrollment response status %s: %s", resp.Status, string(body))
}
- body, err := io.ReadAll(resp.Body)
+ body, err := cpclient.ReadBody(resp.Body)
if err != nil {
- return err
+ return fmt.Errorf("enrollment response: %w", err)
}
var enrollResp api.EnrollResponse
@@ -505,9 +505,9 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {
return fmt.Errorf("bootstrap enrollment request response status %s: %s", resp.Status, string(body))
}
- respData, err := io.ReadAll(resp.Body)
+ respData, err := cpclient.ReadBody(resp.Body)
if err != nil {
- return err
+ return fmt.Errorf("bootstrap enrollment response: %w", err)
}
enrollResp := &api.BootstrapEnrollResponse{}
@@ -566,10 +566,10 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {
continue
}
- statusBody, err := io.ReadAll(statusResp.Body)
+ statusBody, err := cpclient.ReadBody(statusResp.Body)
_ = statusResp.Body.Close()
if err != nil {
- logger.Warnf("failed to read status body: %v", err)
+ logger.Warnf("enrollment status response: %v", err)
continue
}
@@ -901,8 +901,13 @@ func (r *Router) renewLease() {
return
}
- body, _ := io.ReadAll(resp.Body)
+ body, readErr := cpclient.ReadBody(resp.Body)
_ = resp.Body.Close()
+ if readErr != nil {
+ logger.Errorf("Control plane lease renewal response: %v", readErr)
+ leaseRenewalsTotal.WithLabelValues(leaseRejected).Inc()
+ return
+ }
if resp.StatusCode == http.StatusUnauthorized && attempt == 0 {
logger.Warnf("Control plane lease renewal rejected (401 Unauthorized: %s), attempting recovery...", string(body))
@@ -1362,9 +1367,9 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {
return fmt.Errorf("refresh failed with status %s: %s", resp.Status, string(body))
}
- respData, err := io.ReadAll(resp.Body)
+ respData, err := cpclient.ReadBody(resp.Body)
if err != nil {
- return fmt.Errorf("failed to read response: %w", err)
+ return fmt.Errorf("refresh response: %w", err)
}
var refreshResp api.TokenRefreshResponse