Skip to content

Agent Mesh

controlplane: validate, store and distribute egress destinations

Antonio Ojea774b38493e72
diff --git a/internal/controlplane/client/client.go b/internal/controlplane/client/client.go
index 4aa46304..b9fc97e2 100644
--- a/internal/controlplane/client/client.go
+++ b/internal/controlplane/client/client.go
@@ -48,6 +48,11 @@ const MaxBodyBytes = 8 << 20
// truncated ban set or router list would be read as a smaller, valid one.
var ErrBodyTooLarge = errors.New("control plane answer exceeds the body cap")
+// ErrNotFound marks a 404: the control plane does not serve the endpoint, as
+// one predating it does not. Callers of an endpoint added after the first
+// release check for it, so a newer node works against an older control plane.
+var ErrNotFound = errors.New("control plane does not serve this endpoint")
+
// ReadBody reads a control plane response body of at most MaxBodyBytes and
// reports ErrBodyTooLarge for anything larger.
func ReadBody(r io.Reader) ([]byte, error) {
@@ -140,6 +145,16 @@ func (c *Client) FetchPolicy(ctx context.Context, biscuit []byte) (*api.PolicyCo
return &policy, nil
}
+// FetchEgress is GET /egress, authenticated with the caller's biscuit: the
+// egress destinations the control plane assigned to this node.
+func (c *Client) FetchEgress(ctx context.Context, biscuit []byte) (*api.EgressAssignmentsResponse, error) {
+ var egress api.EgressAssignmentsResponse
+ if err := c.get(ctx, "/egress", biscuit, &egress); err != nil {
+ return nil, err
+ }
+ return &egress, nil
+}
+
func (c *Client) get(ctx context.Context, path string, biscuit []byte, msg proto.Message) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
if err != nil {
@@ -158,6 +173,9 @@ func (c *Client) get(ctx context.Context, path string, biscuit []byte, msg proto
if err != nil {
return fmt.Errorf("%s: %w", path, err)
}
+ if resp.StatusCode == http.StatusNotFound {
+ return fmt.Errorf("%w: %s", ErrNotFound, path)
+ }
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))
}
diff --git a/internal/controlplane/egress_test.go b/internal/controlplane/egress_test.go
new file mode 100644
index 00000000..68ffadc2
--- /dev/null
+++ b/internal/controlplane/egress_test.go
@@ -0,0 +1,258 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+ "context"
+ "encoding/base64"
+ "io"
+ "net/http"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "google.golang.org/protobuf/proto"
+)
+
+// TestEgressPolicyIsDistributedToServingNodes covers the admin-to-node path
+// of an egress destination: the policy document is accepted and stored with
+// its http and egress sections, GET /policies renders the serving grants,
+// and GET /egress hands each node the destinations that select it, by role
+// or by label, and nothing else.
+func TestEgressPolicyIsDistributedToServingNodes(t *testing.T) {
+ issuer, _ := startCustomMockOIDC(t)
+ srv, store, baseURL := setupTestServer(t, issuer)
+ defer func() {
+ _ = srv.Close()
+ _ = store.Close()
+ }()
+ const adminToken = "super-secret-admin-token"
+ srv.config.AdminToken = adminToken
+ srv.config.AutoApproveEnrollment = true
+ ctx := context.Background()
+ client := &http.Client{Timeout: 5 * time.Second}
+
+ // Bootstrap tokens need the roles to exist before the policy below is
+ // posted, and labels need a role that permits them.
+ if err := store.SaveMeshPolicy(ctx, []*api.PolicyRole{
+ {Name: "pep", AllowedLabels: []string{"*"}},
+ {Name: api.RoleNode, AllowedLabels: []string{"*"}},
+ }, nil); err != nil {
+ t.Fatal(err)
+ }
+
+ policy := `{
+ "roles": [
+ {"name": "pep", "allowed_services": ["egress://api.github.com"], "allowed_targets": ["*"], "allowed_labels": ["*"]},
+ {"name": "sam:role:node", "allowed_services": ["egress://mam.internal.example.com"], "allowed_targets": ["*"], "allowed_labels": ["*"],
+ "http": [{"service": "egress://mam.internal.example.com", "methods": ["GET"], "paths": ["/v2/public/*"]}]}
+ ],
+ "bindings": [{"role": "sam:role:node", "members": ["sam:system:authenticated"]}],
+ "egress": [
+ {"name": "api.github.com", "credential": "github-eu", "served_by": ["pep"]},
+ {"name": "mam.internal.example.com", "target_url": "http://mam.internal.example.com:8080", "served_by": ["site=dc1"]}
+ ]
+ }`
+ postPolicy := func(t *testing.T, body string) (int, string) {
+ t.Helper()
+ req, _ := http.NewRequest(http.MethodPost, baseURL+"/policies", strings.NewReader(body))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Bearer "+adminToken)
+ resp, err := client.Do(req)
+ if err != nil {
+ t.Fatalf("POST /policies: %v", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ out, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, string(out)
+ }
+ if status, body := postPolicy(t, policy); status != http.StatusOK {
+ t.Fatalf("POST /policies: %d %s", status, body)
+ }
+
+ // Stored and rendered back with both sections, so the console can post
+ // what it shows.
+ roles, _, err := store.GetMeshPolicy(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var nodeRole *api.PolicyRole
+ for _, r := range roles {
+ if r.Name == api.RoleNode {
+ nodeRole = r
+ }
+ }
+ if nodeRole == nil || len(nodeRole.Http) != 1 || nodeRole.Http[0].Service != "egress://mam.internal.example.com" || !slices.Equal(nodeRole.Http[0].Methods, []string{"GET"}) {
+ t.Fatalf("http section was not stored: %v", nodeRole)
+ }
+ egress, err := store.GetEgressDestinations(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(egress) != 2 || egress[0].Name != "api.github.com" || egress[0].Credential != "github-eu" || !slices.Equal(egress[1].ServedBy, []string{"site=dc1"}) {
+ t.Fatalf("egress section was not stored: %v", egress)
+ }
+ req, _ := http.NewRequest(http.MethodGet, baseURL+"/admin/policy", nil)
+ req.Header.Set("Authorization", "Bearer "+adminToken)
+ resp, err := client.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rendered, _ := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if !strings.Contains(string(rendered), `"egress"`) || !strings.Contains(string(rendered), `"http"`) {
+ t.Fatalf("GET /admin/policy lacks the new sections: %s", rendered)
+ }
+ if status, body := postPolicy(t, string(rendered)); status != http.StatusOK {
+ t.Fatalf("re-posting the rendered policy: %d %s", status, body)
+ }
+
+ // Two nodes: one holds the pep role, one is a plain node labelled
+ // site=dc1. Each is selected by exactly one destination.
+ enroll := func(t *testing.T, role string, labels map[string]string) []byte {
+ t.Helper()
+ token := createAdminBootstrapToken(t, baseURL, adminToken, role, 1)
+ priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pID, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pubBytes, err := crypto.MarshalPublicKey(pub)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := bootstrapEnroll(t, baseURL, token, priv, pID, pubBytes, role, labels)
+ if out.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {
+ t.Fatalf("enroll %s: status %v (%s)", role, out.Status, out.ErrorMessage)
+ }
+ return out.BiscuitToken
+ }
+ pepToken := enroll(t, "pep", nil)
+ dc1Token := enroll(t, api.RoleNode, map[string]string{"site": "dc1"})
+
+ getMesh := func(t *testing.T, path string, token []byte, out proto.Message) {
+ t.Helper()
+ req, _ := http.NewRequest(http.MethodGet, baseURL+path, nil)
+ req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(token))
+ resp, err := client.Do(req)
+ if err != nil {
+ t.Fatalf("GET %s: %v", path, err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("GET %s: %s %s", path, resp.Status, body)
+ }
+ if err := proto.Unmarshal(body, out); err != nil {
+ t.Fatalf("GET %s: decode: %v", path, err)
+ }
+ }
+
+ var rules api.PolicyConfigGetResponse
+ getMesh(t, "/policies", pepToken, &rules)
+ for _, want := range []string{
+ `granted_service_exact("egress", "api.github.com") <- role("pep")`,
+ `granted_service_exact("egress", "mam.internal.example.com") <- label("site", "dc1")`,
+ `http_granted_service_exact("egress", "mam.internal.example.com") <- role("sam:role:node")`,
+ `granted_method("egress", "mam.internal.example.com", ["GET"]) <- role("sam:role:node")`,
+ } {
+ if !slices.Contains(rules.DatalogRules, want) {
+ t.Errorf("datalog_rules lack %q:\n%s", want, strings.Join(rules.DatalogRules, "\n"))
+ }
+ }
+ // The response stays within the contract an older node checks.
+ if len(rules.ProtoReflect().GetUnknown()) > 0 {
+ t.Errorf("policy response carries unknown fields")
+ }
+
+ names := func(resp *api.EgressAssignmentsResponse) []string {
+ var out []string
+ for _, d := range resp.Egress {
+ out = append(out, d.Name)
+ }
+ return out
+ }
+ var pepEgress, dc1Egress api.EgressAssignmentsResponse
+ getMesh(t, "/egress", pepToken, &pepEgress)
+ if got := names(&pepEgress); !slices.Equal(got, []string{"api.github.com"}) {
+ t.Errorf("pep node assigned %v, want [api.github.com]", got)
+ }
+ if pepEgress.Egress[0].Credential != "github-eu" {
+ t.Errorf("assignment lost its credential name: %v", pepEgress.Egress[0])
+ }
+ getMesh(t, "/egress", dc1Token, &dc1Egress)
+ if got := names(&dc1Egress); !slices.Equal(got, []string{"mam.internal.example.com"}) {
+ t.Errorf("dc1 node assigned %v, want [mam.internal.example.com]", got)
+ }
+ if dc1Egress.Egress[0].TargetUrl != "http://mam.internal.example.com:8080" {
+ t.Errorf("assignment lost its target_url: %v", dc1Egress.Egress[0])
+ }
+
+ // Without a node credential there is nothing to select on.
+ resp, err = client.Get(baseURL + "/egress")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusUnauthorized {
+ t.Errorf("anonymous GET /egress: %s, want 401", resp.Status)
+ }
+
+ // A selector that matches no enrolled node is valid in form and is
+ // reported, so a typo does not surface only as 404s at the callers.
+ roles, bindings, err := store.GetMeshPolicy(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ unserved, err := srv.unservedEgress(ctx, &api.PolicyConfig{Roles: roles, Bindings: bindings, Egress: []*api.EgressDestination{
+ {Name: "api.github.com", ServedBy: []string{"pep"}},
+ {Name: "mam.internal.example.com", ServedBy: []string{"site=dc1"}},
+ {Name: "typo.example", ServedBy: []string{"site=dc-1"}},
+ {Name: "nobody.example", ServedBy: []string{"contractor"}},
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var unservedNames []string
+ for _, d := range unserved {
+ unservedNames = append(unservedNames, d.Name)
+ }
+ if want := []string{"typo.example", "nobody.example"}; !slices.Equal(unservedNames, want) {
+ t.Errorf("unserved = %v, want %v", unservedNames, want)
+ }
+
+ // Validation names the mistake.
+ for _, tc := range []struct{ name, body, want string }{
+ {"unknown served_by", `{"roles":[{"name":"pep"}],"egress":[{"name":"x.example","served_by":["ghost"]}]}`, "neither a role"},
+ {"credential is a path", `{"roles":[{"name":"pep"}],"egress":[{"name":"x.example","credential":"/etc/x","served_by":["pep"]}]}`, "not a path"},
+ {"duplicate destination", `{"roles":[{"name":"pep"}],"egress":[{"name":"x.example","served_by":["pep"]},{"name":"x.example","served_by":["pep"]}]}`, "duplicate egress"},
+ {"http narrows a service the role lacks", `{"roles":[{"name":"pep","allowed_services":["mcp://a"],"http":[{"service":"mcp://b","methods":["GET"]}]}]}`, "does not name one of the role's allowed_services"},
+ {"http narrows nothing", `{"roles":[{"name":"pep","allowed_services":["mcp://a"],"http":[{"service":"mcp://a"}]}]}`, "narrows nothing"},
+ {"a URL where a hostname belongs", `{"roles":[{"name":"pep","allowed_services":["egress://api.github.com/v3"]}]}`, "without a scheme, a port or a path"},
+ {"uppercase egress grant", `{"roles":[{"name":"pep","allowed_services":["egress://API.github.com"]}]}`, "lowercase"},
+ } {
+ status, body := postPolicy(t, tc.body)
+ if status != http.StatusBadRequest || !strings.Contains(body, tc.want) {
+ t.Errorf("%s: got %d %q, want 400 mentioning %q", tc.name, status, body, tc.want)
+ }
+ }
+}
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 661c8921..afbf8ed2 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -235,6 +235,7 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
handle("/keys", meshSurface(s.HandleKeys))
handle("/routers/lease", s.HandleRouterLease)
handle("/policies", meshSurface(s.HandlePolicies))
+ handle("/egress", meshSurface(s.HandleEgress))
handle("/enroll", meshSurface(noStore(s.HandleEnroll)))
handle("/enroll/status", meshSurface(noStore(s.HandleEnrollStatus)))
handle("/refresh", meshSurface(noStore(s.HandleRefresh)))
@@ -1190,11 +1191,20 @@ func (s *Server) HandlePolicies(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
+ egress, err := s.store.GetEgressDestinations(r.Context())
+ if err != nil && err != storage.ErrNotFound {
+ logger.Errorf("Failed to load egress destinations: %v", err)
+ http.Error(w, "Internal server error", http.StatusInternalServerError)
+ return
+ }
policyRules, warnings := api.BuildPolicyRules(roles, bindings)
for _, warning := range warnings {
logger.Warnf("mesh policy: %s", warning)
}
+ // The nodes that serve a destination are granted it, so the serving
+ // node authorizes local requests with its own credential.
+ policyRules = append(policyRules, api.BuildEgressServingRules(egress)...)
respData, _ := proto.Marshal(&api.PolicyConfigGetResponse{DatalogRules: api.PolicyRuleTexts(policyRules)})
w.Header().Set("Content-Type", "application/x-protobuf")
w.WriteHeader(http.StatusOK)
@@ -1233,11 +1243,12 @@ func (s *Server) HandlePolicies(w http.ResponseWriter, r *http.Request) {
return
}
- if err := s.store.SaveMeshPolicy(r.Context(), req.Roles, req.Bindings); err != nil {
+ if err := s.store.SavePolicyDocument(r.Context(), req.Roles, req.Bindings, req.Egress); err != nil {
logger.Errorf("Failed to save policy: %v", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
+ s.warnUnservedEgress(r.Context(), req)
if err := s.getMeshAdapter().PublishEvent(r.Context(), api.MeshEvent_POLICY_UPDATE, "", nil); err != nil {
logger.Warnf("Failed to publish POLICY_UPDATE event to mesh: %v", err)
@@ -1258,29 +1269,155 @@ func (s *Server) HandlePolicies(w http.ResponseWriter, r *http.Request) {
}
}
+// warnUnservedEgress logs each destination of a just-posted policy that no
+// enrolled, admitted node is selected to serve. A selector with a typo is
+// valid in form and would otherwise fail silently, as a 404 to every caller.
+// A warning and not an error: the node it selects may enroll later.
+func (s *Server) warnUnservedEgress(ctx context.Context, policy *api.PolicyConfig) {
+ unserved, err := s.unservedEgress(ctx, policy)
+ if err != nil {
+ logger.Warnf("mesh policy: cannot check egress selectors against enrolled nodes: %v", err)
+ return
+ }
+ for _, d := range unserved {
+ logger.Warnf("mesh policy: egress destination %s is served by no enrolled node (served_by %v); callers get 404 until one matches", d.GetName(), d.GetServedBy())
+ }
+}
+
+// unservedEgress returns the destinations of policy whose served_by selects
+// no enrolled, admitted node, with roles resolved as a refresh would.
+func (s *Server) unservedEgress(ctx context.Context, policy *api.PolicyConfig) ([]*api.EgressDestination, error) {
+ if len(policy.GetEgress()) == 0 {
+ return nil, nil
+ }
+ nodes, err := s.store.ListNodes(ctx)
+ if err != nil {
+ return nil, err
+ }
+ now := time.Now()
+ var unserved []*api.EgressDestination
+ for _, d := range policy.GetEgress() {
+ served := false
+ for i := range nodes {
+ node := &nodes[i]
+ if node.CheckAdmission(now) != nil {
+ continue
+ }
+ roles, err := nodeRoles(node, policy.GetBindings())
+ if err != nil {
+ continue
+ }
+ if api.EgressServedBy(d, roles, node.Labels) {
+ served = true
+ break
+ }
+ }
+ if !served {
+ unserved = append(unserved, d)
+ }
+ }
+ return unserved, nil
+}
+
// isAdmittedNodeRequest reports whether the bearer credential is a biscuit of
// an enrolled, admitted node. It never falls through to OIDC: running ID token
// verification on a biscuit logs a failure and would auto-register whoever's
// ID token lands here.
func (s *Server) isAdmittedNodeRequest(r *http.Request) bool {
+ return s.admittedNode(r) != nil
+}
+
+// admittedNode returns the enrolled, admitted node whose biscuit the request
+// bears, or nil.
+func (s *Server) admittedNode(r *http.Request) *storage.EnrolledNode {
authHeader := r.Header.Get("Authorization")
if !strings.HasPrefix(authHeader, "Bearer ") {
- return false
+ return nil
}
biscuitBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Bearer "))
if err != nil {
- return false
+ return nil
}
trustedKeys, err := s.store.GetAllValidPublicKeys(r.Context())
if err != nil {
- return false
+ return nil
}
peerID, err := identity.VerifyAndExtractPeerID(trustedKeys, biscuitBytes, s.config.BiscuitTimeout)
if err != nil {
- return false
+ return nil
}
nodeRecord, err := s.store.GetNode(r.Context(), peerID.String())
- return err == nil && nodeRecord != nil && nodeRecord.CheckAdmission(time.Now()) == nil
+ if err != nil || nodeRecord == nil || nodeRecord.CheckAdmission(time.Now()) != nil {
+ return nil
+ }
+ return nodeRecord
+}
+
+// HandleEgress HTTP GET `/egress`: the egress destinations the requesting
+// node serves, selected by its roles and labels (see EgressDestination).
+// Mesh protocol, biscuit-authenticated, binary protobuf. A separate endpoint
+// from /policies so a node predating it keeps syncing rules unchanged.
+func (s *Server) HandleEgress(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ nodeRecord := s.admittedNode(r)
+ if nodeRecord == nil {
+ http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)
+ return
+ }
+ _, bindings, err := s.store.GetMeshPolicy(r.Context())
+ if err != nil && err != storage.ErrNotFound {
+ logger.Errorf("Failed to load policy: %v", err)
+ http.Error(w, "Internal server error", http.StatusInternalServerError)
+ return
+ }
+ egress, err := s.store.GetEgressDestinations(r.Context())
+ if err != nil && err != storage.ErrNotFound {
+ logger.Errorf("Failed to load egress destinations: %v", err)
+ http.Error(w, "Internal server error", http.StatusInternalServerError)
+ return
+ }
+ roles, err := nodeRoles(nodeRecord, bindings)
+ if err != nil {
+ logger.Errorf("Failed to resolve roles for node %s: %v", nodeRecord.PeerID, err)
+ http.Error(w, "Internal server error", http.StatusInternalServerError)
+ return
+ }
+ resp := &api.EgressAssignmentsResponse{}
+ for _, d := range egress {
+ if api.EgressServedBy(d, roles, nodeRecord.Labels) {
+ resp.Egress = append(resp.Egress, d)
+ }
+ }
+ respData, err := proto.Marshal(resp)
+ if err != nil {
+ http.Error(w, "Failed to serialize response", http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(respData)
+}
+
+// nodeRoles is the role set a refresh would mint for the node: the enrolled
+// role plus the custom roles its identity resolves to, from the bindings.
+func nodeRoles(nodeRecord *storage.EnrolledNode, bindings []*api.PolicyBinding) ([]string, error) {
+ roles := []string{nodeRecord.Role}
+ if nodeRecord.EnrollmentType != "OIDC" {
+ return roles, nil
+ }
+ var claims jwt.MapClaims
+ if err := json.Unmarshal([]byte(nodeRecord.ClaimsJSON), &claims); err != nil {
+ return nil, err
+ }
+ for _, r := range resolveRoles(nodeRecord.PeerID, claims, bindings) {
+ if !strings.HasPrefix(r, "sam:role:") && r != nodeRecord.Role {
+ roles = append(roles, r)
+ }
+ }
+ return roles, nil
}
// HandleAdminPolicy HTTP GET `/admin/policy`: the mesh policy as the operator
@@ -1299,7 +1436,13 @@ func (s *Server) HandleAdminPolicy(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
- writeProtoJSON(w, &api.PolicyConfig{Roles: roles, Bindings: bindings})
+ egress, err := s.store.GetEgressDestinations(r.Context())
+ if err != nil && err != storage.ErrNotFound {
+ logger.Errorf("Failed to load egress destinations: %v", err)
+ http.Error(w, "Internal server error", http.StatusInternalServerError)
+ return
+ }
+ writeProtoJSON(w, &api.PolicyConfig{Roles: roles, Bindings: bindings, Egress: egress})
}
// writeProtoJSON answers an operator-plane request with protojson of msg,
@@ -2545,7 +2688,11 @@ func (s *Server) HandleUserStatus(w http.ResponseWriter, r *http.Request) {
if err != nil && err != storage.ErrNotFound {
logger.Errorf("Failed to list policy: %v", err)
}
- if rendered, err := marshalPolicyJSON(roles, bindings); err == nil {
+ egress, err := s.store.GetEgressDestinations(ctx)
+ if err != nil && err != storage.ErrNotFound {
+ logger.Errorf("Failed to list egress destinations: %v", err)
+ }
+ if rendered, err := marshalPolicyJSON(roles, bindings, egress); err == nil {
resp["policy_json"] = rendered
} else {
logger.Errorf("Failed to render policy: %v", err)
@@ -2906,8 +3053,8 @@ func toStringSlice(val any) []string {
// field names. Generated marshalling is the point: a hand-maintained mirror of
// PolicyRole silently drops any field it forgets, which is how custom_datalog
// went missing from the console for so long.
-func marshalPolicyJSON(roles []*api.PolicyRole, bindings []*api.PolicyBinding) (string, error) {
- resp := &api.PolicyConfig{Roles: roles, Bindings: bindings}
+func marshalPolicyJSON(roles []*api.PolicyRole, bindings []*api.PolicyBinding, egress []*api.EgressDestination) (string, error) {
+ resp := &api.PolicyConfig{Roles: roles, Bindings: bindings, Egress: egress}
marshaler := protojson.MarshalOptions{UseProtoNames: true, Multiline: true, Indent: " "}
out, err := marshaler.Marshal(resp)
if err != nil {
@@ -2949,6 +3096,9 @@ func validatePolicyConfig(req *api.PolicyConfig) error {
if err := api.ValidateServiceFormat(svc); err != nil {
return fmt.Errorf("invalid allowed_service %q in role %s: %w", svc, r.Name, err)
}
+ if err := api.ValidateEgressServicePattern(svc); err != nil {
+ return fmt.Errorf("in role %s: %w", r.Name, err)
+ }
}
for _, target := range r.AllowedTargets {
if err := api.ValidateTargetFormat(target); err != nil {
@@ -2965,6 +3115,11 @@ func validatePolicyConfig(req *api.PolicyConfig) error {
return fmt.Errorf("in role %s: %w", r.Name, err)
}
}
+ for _, g := range r.Http {
+ if err := api.ValidateHTTPGrant(g, r.AllowedServices); err != nil {
+ return fmt.Errorf("in role %s: %w", r.Name, err)
+ }
+ }
for _, dl := range r.CustomDatalog {
trimmed := strings.TrimRight(strings.TrimSpace(dl), ";")
if trimmed == "" {
@@ -2985,6 +3140,9 @@ func validatePolicyConfig(req *api.PolicyConfig) error {
factBudget += len(api.BuildTargetDatalogFacts(r.AllowedTargets))
factBudget += len(api.BuildAgentDatalogFacts(r.AllowedAgents))
factBudget += len(r.CustomDatalog)
+ for _, g := range r.Http {
+ factBudget += len(api.BuildHTTPGrantFacts(g))
+ }
}
if factBudget > maxIdentityFactBudget {
@@ -3023,5 +3181,19 @@ func validatePolicyConfig(req *api.PolicyConfig) error {
}
}
}
+
+ egressNames := make(map[string]bool, len(req.Egress))
+ for _, d := range req.Egress {
+ if d == nil {
+ continue
+ }
+ if err := api.ValidateEgressDestination(d, roleNames); err != nil {
+ return err
+ }
+ if egressNames[d.Name] {
+ return fmt.Errorf("duplicate egress destination: %s", d.Name)
+ }
+ egressNames[d.Name] = true
+ }
return nil
}
diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go
index 959ea277..250987e8 100644
--- a/internal/controlplane/server_test.go
+++ b/internal/controlplane/server_test.go
@@ -582,7 +582,7 @@ func TestMarshalPolicyJSONRoundTrip(t *testing.T) {
{Role: "ops", Members: []string{"user:root"}},
}
- rendered, err := marshalPolicyJSON(roles, bindings)
+ rendered, err := marshalPolicyJSON(roles, bindings, nil)
if err != nil {
t.Fatalf("rendering the policy: %v", err)
}
@@ -675,7 +675,7 @@ func TestPoliciesAcceptConsoleJSON(t *testing.T) {
}
// What /status hands the console must be postable back unchanged.
- rendered, err := marshalPolicyJSON(roles, bindings)
+ rendered, err := marshalPolicyJSON(roles, bindings, nil)
if err != nil {
t.Fatalf("rendering the stored policy: %v", err)
}
diff --git a/internal/controlplane/ui.go b/internal/controlplane/ui.go
index 064f5a04..27e152bb 100644
--- a/internal/controlplane/ui.go
+++ b/internal/controlplane/ui.go
@@ -72,9 +72,13 @@ func (s *Server) HandleAdminStatus(w http.ResponseWriter, r *http.Request) {
if err != nil {
logger.Errorf("Failed to list policy: %v", err)
}
+ egress, err := s.store.GetEgressDestinations(r.Context())
+ if err != nil {
+ logger.Errorf("Failed to list egress destinations: %v", err)
+ }
var policyJSON string
- if rendered, err := marshalPolicyJSON(roles, bindings); err == nil {
+ if rendered, err := marshalPolicyJSON(roles, bindings, egress); err == nil {
policyJSON = rendered
} else {
logger.Errorf("Failed to render policy: %v", err)
diff --git a/internal/storage/mesh_policy_test.go b/internal/storage/mesh_policy_test.go
index 23808961..6cc95c65 100644
--- a/internal/storage/mesh_policy_test.go
+++ b/internal/storage/mesh_policy_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"github.com/google/sam/api"
+ "google.golang.org/protobuf/proto"
)
// TestMeshPolicyRoundTripsEveryRoleField stores a role with every repeated
@@ -46,6 +47,7 @@ func TestMeshPolicyRoundTripsEveryRoleField(t *testing.T) {
CustomDatalog: []string{`region("emea")`},
AllowedAgents: []string{"*.prod.acme.example"},
AllowedLabels: []string{"region=*"},
+ Http: []*api.HTTPGrant{{Service: "mcp://tool", Methods: []string{"GET"}, Paths: []string{"/v1/*"}}},
}
if err := store.SaveMeshPolicy(ctx, []*api.PolicyRole{want}, nil); err != nil {
@@ -72,8 +74,63 @@ func TestMeshPolicyRoundTripsEveryRoleField(t *testing.T) {
}
w := wantVal.Field(i).Interface()
g := gotVal.Field(i).Interface()
+ if field.Type.Elem().Implements(reflect.TypeOf((*proto.Message)(nil)).Elem()) {
+ // Message slices carry internal state DeepEqual would compare.
+ wm, gm := reflect.ValueOf(w), reflect.ValueOf(g)
+ if wm.Len() != gm.Len() {
+ t.Errorf("field %s did not round trip: saved %d entries, loaded %d", field.Name, wm.Len(), gm.Len())
+ continue
+ }
+ for j := 0; j < wm.Len(); j++ {
+ if !proto.Equal(wm.Index(j).Interface().(proto.Message), gm.Index(j).Interface().(proto.Message)) {
+ t.Errorf("field %s[%d] did not round trip: saved %v, loaded %v", field.Name, j, wm.Index(j), gm.Index(j))
+ }
+ }
+ continue
+ }
if !reflect.DeepEqual(w, g) {
t.Errorf("field %s did not round trip: saved %v, loaded %v", field.Name, w, g)
}
}
}
+
+// TestSavePolicyDocumentIsAtomic: a document is applied whole or not at all.
+// The control plane validates before saving, so the failing row here is one
+// validation would have refused; what matters is that the roles written in
+// the same call are rolled back with it.
+func TestSavePolicyDocumentIsAtomic(t *testing.T) {
+ store, err := NewSQLStore("sqlite", filepath.Join(t.TempDir(), "policy.db"))
+ if err != nil {
+ t.Fatalf("NewSQLStore: %v", err)
+ }
+ defer func() { _ = store.Close() }()
+ ctx := context.Background()
+
+ first := []*api.PolicyRole{{Name: "first"}}
+ egress := []*api.EgressDestination{{Name: "api.github.com", Credential: "gh", ServedBy: []string{"first"}}}
+ if err := store.SavePolicyDocument(ctx, first, nil, egress); err != nil {
+ t.Fatalf("SavePolicyDocument: %v", err)
+ }
+ got, err := store.GetEgressDestinations(ctx)
+ if err != nil || len(got) != 1 || !proto.Equal(got[0], egress[0]) {
+ t.Fatalf("GetEgressDestinations = %v, %v", got, err)
+ }
+
+ // Two rows with the same name violate the primary key after the roles
+ // were already replaced inside the transaction.
+ bad := []*api.EgressDestination{{Name: "dup.example", ServedBy: []string{"second"}}, {Name: "dup.example", ServedBy: []string{"second"}}}
+ if err := store.SavePolicyDocument(ctx, []*api.PolicyRole{{Name: "second"}}, nil, bad); err == nil {
+ t.Fatal("SavePolicyDocument accepted a duplicate destination")
+ }
+ roles, _, err := store.GetMeshPolicy(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(roles) != 1 || roles[0].Name != "first" {
+ t.Errorf("roles after a failed document = %v, want the previous document's", roles)
+ }
+ got, err = store.GetEgressDestinations(ctx)
+ if err != nil || len(got) != 1 || got[0].Name != "api.github.com" {
+ t.Errorf("egress after a failed document = %v, %v, want the previous document's", got, err)
+ }
+}
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index 6c3f01fa..836f0efd 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -27,6 +27,7 @@ import (
"github.com/google/sam/api"
log "github.com/ipfs/go-log/v2"
+ "google.golang.org/protobuf/encoding/protojson"
// Register PG and SQLite drivers
_ "github.com/jackc/pgx/v5/stdlib"
@@ -451,6 +452,28 @@ var migrations = []migration{
`ALTER TABLE users ADD COLUMN issuer TEXT DEFAULT '' NOT NULL`,
},
},
+ {
+ // Egress destinations are part of the policy document (PolicyConfig.egress)
+ // but are not role-scoped, so they get their own table. served_by is a
+ // JSON array of role names and key=value labels.
+ version: 12,
+ postgres: []string{
+ `CREATE TABLE IF NOT EXISTS egress_destinations (
+ name VARCHAR(253) PRIMARY KEY,
+ target_url TEXT NOT NULL,
+ credential VARCHAR(64) NOT NULL,
+ served_by TEXT NOT NULL
+ )`,
+ },
+ sqlite: []string{
+ `CREATE TABLE IF NOT EXISTS egress_destinations (
+ name TEXT PRIMARY KEY,
+ target_url TEXT NOT NULL,
+ credential TEXT NOT NULL,
+ served_by TEXT NOT NULL
+ )`,
+ },
+ },
}
func (s *SQLStore) initSchema() error {
@@ -992,12 +1015,36 @@ func (s *SQLStore) GetActiveRouters(ctx context.Context) ([]RouterLease, error)
// SaveMeshPolicy replaces the entire mesh policy with the provided roles and bindings.
func (s *SQLStore) SaveMeshPolicy(ctx context.Context, roles []*api.PolicyRole, bindings []*api.PolicyBinding) error {
+ return s.inTx(ctx, func(tx *sql.Tx) error {
+ return s.saveMeshPolicyTx(ctx, tx, roles, bindings)
+ })
+}
+
+// SavePolicyDocument replaces roles, bindings and egress destinations in one
+// transaction, so a POST /policies is applied whole or not at all.
+func (s *SQLStore) SavePolicyDocument(ctx context.Context, roles []*api.PolicyRole, bindings []*api.PolicyBinding, egress []*api.EgressDestination) error {
+ return s.inTx(ctx, func(tx *sql.Tx) error {
+ if err := s.saveMeshPolicyTx(ctx, tx, roles, bindings); err != nil {
+ return err
+ }
+ return s.saveEgressDestinationsTx(ctx, tx, egress)
+ })
+}
+
+// inTx runs fn in a transaction and commits when it returns nil.
+func (s *SQLStore) inTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
+ if err := fn(tx); err != nil {
+ return err
+ }
+ return tx.Commit()
+}
+func (s *SQLStore) saveMeshPolicyTx(ctx context.Context, tx *sql.Tx, roles []*api.PolicyRole, bindings []*api.PolicyBinding) error {
// For simplicity in replacing policy, we clear all and insert new.
if _, err := tx.ExecContext(ctx, "DELETE FROM role_permissions"); err != nil {
return err
@@ -1041,6 +1088,20 @@ func (s *SQLStore) SaveMeshPolicy(ctx context.Context, roles []*api.PolicyRole,
return err
}
}
+ // One row per narrowed entry, as protojson: the shape is the proto's
+ // and a new HTTPGrant field needs no schema change here.
+ for _, g := range r.Http {
+ if g == nil {
+ continue
+ }
+ encoded, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(g)
+ if err != nil {
+ return err
+ }
+ if _, err := tx.ExecContext(ctx, s.rebind("INSERT INTO role_permissions (role_name, resource_type, resource_value) VALUES (?, 'http', ?)"), r.Name, string(encoded)); err != nil {
+ return err
+ }
+ }
}
for _, b := range bindings {
@@ -1054,7 +1115,7 @@ func (s *SQLStore) SaveMeshPolicy(ctx context.Context, roles []*api.PolicyRole,
}
}
- return tx.Commit()
+ return nil
}
// GetMeshPolicy retrieves the entire mesh policy as structured data.
@@ -1104,6 +1165,12 @@ func (s *SQLStore) GetMeshPolicy(ctx context.Context) ([]*api.PolicyRole, []*api
r.AllowedAgents = append(r.AllowedAgents, resValue)
case "label":
r.AllowedLabels = append(r.AllowedLabels, resValue)
+ case "http":
+ g := &api.HTTPGrant{}
+ if err := protojson.Unmarshal([]byte(resValue), g); err != nil {
+ return nil, nil, fmt.Errorf("role %s: stored http grant does not parse: %w", roleName, err)
+ }
+ r.Http = append(r.Http, g)
}
}
}
@@ -1142,6 +1209,57 @@ func (s *SQLStore) GetMeshPolicy(ctx context.Context) ([]*api.PolicyRole, []*api
return roles, bindings, nil
}
+// SaveEgressDestinations replaces the egress section of the mesh policy.
+func (s *SQLStore) SaveEgressDestinations(ctx context.Context, egress []*api.EgressDestination) error {
+ return s.inTx(ctx, func(tx *sql.Tx) error {
+ return s.saveEgressDestinationsTx(ctx, tx, egress)
+ })
+}
+
+func (s *SQLStore) saveEgressDestinationsTx(ctx context.Context, tx *sql.Tx, egress []*api.EgressDestination) error {
+ if _, err := tx.ExecContext(ctx, "DELETE FROM egress_destinations"); err != nil {
+ return err
+ }
+ for _, d := range egress {
+ if d == nil {
+ continue
+ }
+ servedBy, err := json.Marshal(d.GetServedBy())
+ if err != nil {
+ return err
+ }
+ if _, err := tx.ExecContext(ctx, s.rebind("INSERT INTO egress_destinations (name, target_url, credential, served_by) VALUES (?, ?, ?, ?)"),
+ d.GetName(), d.GetTargetUrl(), d.GetCredential(), string(servedBy)); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// GetEgressDestinations loads the egress section of the mesh policy, in name
+// order so the rendered document and rules are stable.
+func (s *SQLStore) GetEgressDestinations(ctx context.Context) ([]*api.EgressDestination, error) {
+ rows, err := s.db.QueryContext(ctx, s.rebind("SELECT name, target_url, credential, served_by FROM egress_destinations ORDER BY name"))
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = rows.Close() }()
+
+ var egress []*api.EgressDestination
+ for rows.Next() {
+ var name, targetURL, credential, servedByJSON string
+ if err := rows.Scan(&name, &targetURL, &credential, &servedByJSON); err != nil {
+ return nil, err
+ }
+ var servedBy []string
+ if err := json.Unmarshal([]byte(servedByJSON), &servedBy); err != nil {
+ return nil, fmt.Errorf("egress %s: stored served_by does not parse: %w", name, err)
+ }
+ egress = append(egress, &api.EgressDestination{Name: name, TargetUrl: targetURL, Credential: credential, ServedBy: servedBy})
+ }
+ return egress, rows.Err()
+}
+
// SaveBootstrapToken persists a new bootstrap token.
func (s *SQLStore) SaveBootstrapToken(ctx context.Context, token *BootstrapToken) error {
var query string
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 48103a99..c3151b82 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -244,6 +244,17 @@ type Store interface {
// GetMeshPolicy loads the mesh configurations.
GetMeshPolicy(ctx context.Context) ([]*api.PolicyRole, []*api.PolicyBinding, error)
+ // SaveEgressDestinations replaces the egress section of the mesh policy
+ // (PolicyConfig.egress).
+ SaveEgressDestinations(ctx context.Context, egress []*api.EgressDestination) error
+
+ // GetEgressDestinations loads the egress section of the mesh policy.
+ GetEgressDestinations(ctx context.Context) ([]*api.EgressDestination, error)
+
+ // SavePolicyDocument replaces roles, bindings and egress destinations in
+ // one transaction: a policy post is applied whole or not at all.
+ SavePolicyDocument(ctx context.Context, roles []*api.PolicyRole, bindings []*api.PolicyBinding, egress []*api.EgressDestination) error
+
// SaveBootstrapToken persists a new bootstrap token.
SaveBootstrapToken(ctx context.Context, token *BootstrapToken) error