Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
tests: the egress destination CUJ with real binaries
Antonio Ojea8f292beb4fea
diff --git a/tests/integration/egress_test.go b/tests/integration/egress_test.gonew file mode 100644index 00000000..b3c7c0f3--- /dev/null+++ b/tests/integration/egress_test.go@@ -0,0 +1,328 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package integration_test++import (+ "context"+ "encoding/base64"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/google/sam/api"+)++// TestEgressDestinationCUJ is the egress story end to end, with real+// binaries: an admin posts one policy document naming destinations, the+// credentials to use and the roles or labels that serve them; a node+// selected by role and by label starts serving them with no configuration+// of its own, and refuses the one whose credential the platform did not+// deliver; an application on that node's host reaches a destination through+// the local API, naming the agent it acts for; another mesh member reaches+// it through the PEP node, narrowed to the methods and paths its role+// allows; each destination sees the node's credential for it and nothing of+// the callers; and a policy update withdraws a destination without a+// restart.+func TestEgressDestinationCUJ(t *testing.T) {+ nodeBin := buildBinary(t, "./cmd/sam-node")+ tmpDir := t.TempDir()+ oidcURL, mintToken := startCustomMockOIDC(t)++ // Two destinations, standing in for api.github.com and an internal API.+ recordingServer := func(seen *[]*http.Request, mu *sync.Mutex) *httptest.Server {+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ mu.Lock()+ *seen = append(*seen, r.Clone(context.Background()))+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ }))+ }+ var mu sync.Mutex+ var seen, seenInternal []*http.Request+ upstream := recordingServer(&seen, &mu)+ defer upstream.Close()+ internal := recordingServer(&seenInternal, &mu)+ defer internal.Close()++ // One document: who serves what with which credential, and who may call+ // it how. api.github.com is served by role, the internal API by label; the+ // third destination names a credential nobody delivered. The contractor+ // may only GET under /repos/acme/.+ policyFile := filepath.Join(tmpDir, "policies.yaml")+ policyYAML := fmt.Sprintf(`roles:+ - name: pep+ allowed_targets: ["*"]+ allowed_labels: ["site=eu"]+ allowed_agents: ["*.acme.example"]+ - name: contractor+ allowed_services: ["egress://api.github.com"]+ allowed_targets: ["*"]+ http:+ - service: "egress://api.github.com"+ methods: ["GET"]+ paths: ["/repos/acme/*"]+bindings:+ - role: pep+ members: ["user:pep-user"]+ - role: contractor+ members: ["user:contractor-user"]+ - role: sam:role:node+ members: ["user:pep-user", "user:contractor-user"]+egress:+ - name: api.github.com+ target_url: %q+ credential: github-eu+ served_by: ["pep"]+ - name: mam.internal.example.com+ target_url: %q+ credential: mam-basic+ served_by: ["site=eu"]+ - name: broken.example+ target_url: %q+ credential: never-delivered+ served_by: ["pep"]+`, upstream.URL, internal.URL, internal.URL)+ if err := os.WriteFile(policyFile, []byte(policyYAML), 0o644); err != nil {+ t.Fatal(err)+ }+ cpPort, cleanupCP := startControlPlaneAndRouter(t, tmpDir, oidcURL, mintToken, policyFile)+ defer cleanupCP()+ controlPlane := fmt.Sprintf("http://127.0.0.1:%d", cpPort)++ // The platform delivered two credentials to the PEP host as files, in+ // the two forms the node accepts; the third destination's is missing.+ secrets := filepath.Join(tmpDir, "secrets")+ if err := os.MkdirAll(secrets, 0o700); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(secrets, "github-eu"), []byte("ghp_pep_secret\n"), 0o600); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(secrets, "mam-basic"), []byte("svc:s3cret"), 0o600); err != nil {+ t.Fatal(err)+ }++ // The PEP declares the label the second destination selects on. Nothing+ // about egress appears here.+ pepConfig := filepath.Join(tmpDir, "pep.yaml")+ if err := os.WriteFile(pepConfig, []byte("version: \"v1alpha1\"\nlabels:\n site: eu\n"), 0o644); err != nil {+ t.Fatal(err)+ }++ pepToken := "test-token"+ pep := launchNode(t, nodeBin, os.Environ(), filepath.Join(tmpDir, "pep"), "run",+ "--control-plane", controlPlane,+ "--data-dir", filepath.Join(tmpDir, "pep"),+ "--api-token-path", tokenPath(t, pepToken),+ "--jwt", mintToken(map[string]interface{}{"sub": "pep-user", "roles": []string{api.RoleNode}}),+ "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",+ "--listen", "/ip4/127.0.0.1/tcp/0",+ "--allow-loopback",+ "--discovery-interval", "100ms",+ "--control-plane-sync-interval", "1s",+ "--config", pepConfig,+ "--secrets-dir", secrets,+ )+ pepAPI := pep.waitForAPI(t)++ callerToken := "test-token"+ caller := launchNode(t, nodeBin, os.Environ(), filepath.Join(tmpDir, "caller"), "run",+ "--control-plane", controlPlane,+ "--data-dir", filepath.Join(tmpDir, "caller"),+ "--api-token-path", tokenPath(t, callerToken),+ "--jwt", mintToken(map[string]interface{}{"sub": "contractor-user", "roles": []string{api.RoleNode}}),+ "--listen", "/ip4/127.0.0.1/udp/0/quic-v1",+ "--listen", "/ip4/127.0.0.1/tcp/0",+ "--allow-loopback",+ "--discovery-interval", "100ms",+ )+ callerAPI := caller.waitForAPI(t)+ connectPeer(t, callerAPI, pep.p2pAddr)+ waitForDHTPeers(t, pepAPI)+ pepPeer := extractPeerID(pep.p2pAddr)++ client := &http.Client{Timeout: 10 * time.Second}+ do := func(t *testing.T, method, rawURL, token string, headers map[string]string) int {+ t.Helper()+ req, err := http.NewRequest(method, rawURL, nil)+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set(api.HeaderSamAuthentication, "Bearer "+token)+ for k, v := range headers {+ req.Header.Set(k, v)+ }+ resp, err := client.Do(req)+ if err != nil {+ t.Fatalf("%s %s: %v", method, rawURL, err)+ }+ _ = resp.Body.Close()+ return resp.StatusCode+ }++ // The PEP picked its assignments up from the control plane and announced+ // them: the caller can discover egress://api.github.com like any service.+ waitForDiscoverableService(t, callerAPI, callerToken, "egress", "api.github.com")++ t.Run("an application on the PEP host, through the local API", func(t *testing.T) {+ // The app is configured with the node's API as its base URL and the+ // API token as its bearer, the way it would hold a provider key.+ req, _ := http.NewRequest(http.MethodGet, "http://"+pepAPI+"/egress/api.github.com/repos/acme/dubbing/pulls?state=open", nil)+ req.Header.Set("Authorization", "Bearer "+pepToken)+ resp, err := client.Do(req)+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusNoContent {+ t.Fatalf("local GET: %d, want 204", resp.StatusCode)+ }+ // The serving role is not narrowed, so any method passes here.+ if got := do(t, http.MethodPost, "http://"+pepAPI+"/egress/api.github.com/repos/acme/x", pepToken, nil); got != http.StatusNoContent {+ t.Errorf("local POST on an un-narrowed grant: %d, want 204", got)+ }+ if got := do(t, http.MethodGet, "http://"+pepAPI+"/egress/other.example/x", pepToken, nil); got != http.StatusNotFound {+ t.Errorf("a destination not assigned: %d, want 404", got)+ }+ // Selected by the node's attested label, with a Basic credential.+ if got := do(t, http.MethodGet, "http://"+pepAPI+"/egress/mam.internal.example.com/v2/clips/99", pepToken, nil); got != http.StatusNoContent {+ t.Errorf("a destination served by label: %d, want 204", got)+ }+ // Assigned, but its credential was never delivered: refused at+ // registration, so it is not served, and the others still are.+ if got := do(t, http.MethodGet, "http://"+pepAPI+"/egress/broken.example/x", pepToken, nil); got != http.StatusNotFound {+ t.Errorf("a destination with a missing credential: %d, want 404", got)+ }+ // The agent the client names is checked against the node's grant.+ agent := func(id string) map[string]string { return map[string]string{api.HeaderSamAgent: id} }+ if got := do(t, http.MethodGet, "http://"+pepAPI+"/egress/api.github.com/repos/acme/x", pepToken, agent("reviewer-7.acme.example")); got != http.StatusNoContent {+ t.Errorf("an agent inside the granted namespace: %d, want 204", got)+ }+ if got := do(t, http.MethodGet, "http://"+pepAPI+"/egress/api.github.com/repos/acme/x", pepToken, agent("intruder.evil-acme.example")); got != http.StatusForbidden {+ t.Errorf("an agent outside it: %d, want 403", got)+ }+ })++ t.Run("a mesh member, through the PEP node", func(t *testing.T) {+ base := fmt.Sprintf("http://%s/sam/%s/egress/api.github.com", callerAPI, pepPeer)+ for _, tc := range []struct {+ name string+ method string+ path string+ want int+ }{+ {"GET under the granted prefix", http.MethodGet, "/repos/acme/dubbing/pulls?state=open", http.StatusNoContent},+ {"POST is outside the narrowed grant", http.MethodPost, "/repos/acme/dubbing/pulls", http.StatusForbidden},+ {"a path outside the narrowed grant", http.MethodGet, "/user", http.StatusForbidden},+ } {+ t.Run(tc.name, func(t *testing.T) {+ if got := do(t, tc.method, base+tc.path, callerToken, nil); got != tc.want {+ t.Errorf("%s %s: %d, want %d", tc.method, tc.path, got, tc.want)+ }+ })+ }+ })++ // What the destinations saw: the node's credential for each, the requested+ // path, and no trace of the callers.+ mu.Lock()+ if len(seen) == 0 || len(seenInternal) == 0 {+ mu.Unlock()+ t.Fatalf("the destinations saw %d and %d requests", len(seen), len(seenInternal))+ }+ for _, r := range seen {+ if r.Header.Get("Authorization") != "Bearer ghp_pep_secret" {+ t.Errorf("api.github.com saw Authorization %q, want the PEP's bearer credential", r.Header.Get("Authorization"))+ }+ if r.URL.Path == "/user" || strings.Contains(r.URL.Path, "..") {+ t.Errorf("a refused request reached the destination: %s", r.URL.Path)+ }+ }+ for _, r := range seenInternal {+ if r.Header.Get("Authorization") != "Basic "+base64.StdEncoding.EncodeToString([]byte("svc:s3cret")) {+ t.Errorf("internal API saw Authorization %q, want the PEP's Basic credential", r.Header.Get("Authorization"))+ }+ if r.URL.Path != "/v2/clips/99" {+ t.Errorf("internal API saw path %q", r.URL.Path)+ }+ }+ for _, r := range append(append([]*http.Request{}, seen...), seenInternal...) {+ for name := range r.Header {+ if strings.HasPrefix(name, "X-Sam-") || strings.HasPrefix(name, "X-Forwarded-") || name == api.HeaderPeerID {+ t.Errorf("a destination saw %s", name)+ }+ }+ }+ if seen[0].URL.RequestURI() != "/repos/acme/dubbing/pulls?state=open" {+ t.Errorf("first request URI = %q", seen[0].URL.RequestURI())+ }+ mu.Unlock()++ t.Run("a policy update withdraws a destination without a restart", func(t *testing.T) {+ // The admin reassigns api.github.com to nodes that do not exist. The+ // control plane publishes the update; the PEP re-syncs and withdraws.+ current, err := os.ReadFile(policyFile)+ if err != nil {+ t.Fatal(err)+ }+ updated := strings.Replace(string(current), `served_by: ["pep"]`, `served_by: ["site=nowhere"]`, 1)+ if updated == string(current) {+ t.Fatal("policy file did not contain the assignment to change")+ }+ if err := os.WriteFile(policyFile, []byte(updated), 0o644); err != nil {+ t.Fatal(err)+ }+ injectPolicyYAML(t, cpPort, "test-admin-token", policyFile)++ deadline := time.Now().Add(8 * time.Second)+ for {+ got := do(t, http.MethodGet, "http://"+pepAPI+"/egress/api.github.com/repos/acme/x", pepToken, nil)+ if got == http.StatusNotFound {+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("api.github.com still served after the update: %d", got)+ }+ time.Sleep(200 * time.Millisecond)+ }+ // The destination selected by label is untouched.+ if got := do(t, http.MethodGet, "http://"+pepAPI+"/egress/mam.internal.example.com/v2/clips/1", pepToken, nil); got != http.StatusNoContent {+ t.Errorf("the other destination after the update: %d, want 204", got)+ }+ })++ // The node's log tells the operator what it serves, what it refused and+ // why, and what it withdrew.+ log := pep.log()+ for _, want := range []string{+ "Serving egress://api.github.com",+ "Serving egress://mam.internal.example.com",+ `credential "never-delivered"`,+ "Withdrawn egress://api.github.com",+ } {+ if !strings.Contains(log, want) {+ t.Errorf("PEP log lacks %q", want)+ }+ }+ if strings.Contains(log, "Serving egress://broken.example") {+ t.Error("PEP served a destination whose credential was never delivered")+ }+}