Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
test(node): add Envoy ext_authz and ext_proc integration and E2E datapath CUJs
Antonio Ojea379f8c2a6e76
diff --git a/internal/node/egress_inspect_test.go b/internal/node/egress_inspect_test.goindex 69ecdec6..221ff4e7 100644--- a/internal/node/egress_inspect_test.go+++ b/internal/node/egress_inspect_test.go@@ -512,7 +512,11 @@ func TestGatewayExtProcServer(t *testing.T) {setHdrs := resp1Body.GetRequestBody().GetResponse().GetHeaderMutation().GetSetHeaders()foundTaskID := falsefor _, h := range setHdrs {- if strings.EqualFold(h.GetHeader().GetKey(), "X-Sam-Task-Id") && h.GetHeader().GetValue() == "task-extproc-gateway" {+ val := h.GetHeader().GetValue()+ if val == "" {+ val = string(h.GetHeader().GetRawValue())+ }+ if strings.EqualFold(h.GetHeader().GetKey(), "X-Sam-Task-Id") && val == "task-extproc-gateway" {foundTaskID = true}}@@ -585,7 +589,11 @@ func TestGatewayExtProcServer(t *testing.T) {}foundAuth := falsefor _, h := range resp3Hdr.GetRequestHeaders().GetResponse().GetHeaderMutation().GetSetHeaders() {- if strings.EqualFold(h.GetHeader().GetKey(), "Authorization") && h.GetHeader().GetValue() == "Bearer gateway-injected-github-token" {+ val := h.GetHeader().GetValue()+ if val == "" {+ val = string(h.GetHeader().GetRawValue())+ }+ if strings.EqualFold(h.GetHeader().GetKey(), "Authorization") && val == "Bearer gateway-injected-github-token" {foundAuth = true}}diff --git a/internal/node/ext_authz.go b/internal/node/ext_authz.goindex 7f1670c1..e51f084f 100644--- a/internal/node/ext_authz.go+++ b/internal/node/ext_authz.go@@ -37,10 +37,11 @@ const ()type extAuthzCheckInput struct {- Method string- Path string- Host string- Headers map[string]string+ Method string+ Path string+ Host string+ Headers map[string]string+ AllowMCPStreamInit bool}type extAuthzCheckResult struct {@@ -74,10 +75,11 @@ func handleExtAuthzHTTP(node *SamNode, w http.ResponseWriter, r *http.Request) {}res := evaluateExtAuthz(r.Context(), node, extAuthzCheckInput{- Method: method,- Path: checkPath,- Host: r.Host,- Headers: headers,+ Method: method,+ Path: checkPath,+ Host: r.Host,+ Headers: headers,+ AllowMCPStreamInit: headers[strings.ToLower(HeaderSamMCPTool)] == "",})if !res.Allowed {http.Error(w, res.Message, res.HTTPStatus)@@ -108,6 +110,7 @@ func handleExtAuthzGRPC(node *SamNode, w http.ResponseWriter, r *http.Request) {writeGRPCError(w, 3, fmt.Sprintf("invalid CheckRequest: %v", err))return}+ in.AllowMCPStreamInit = in.Headers[strings.ToLower(HeaderSamMCPTool)] == ""res := evaluateExtAuthz(r.Context(), node, in)respPayload := marshalEnvoyCheckResponse(res)@@ -146,7 +149,7 @@ func evaluateExtAuthz(ctx context.Context, node *SamNode, in extAuthzCheckInput)}}- target, reqPath := resolveExtAuthzTarget(in)+ target, reqPath := resolveExtAuthzTarget(node, in)if target != "" {var callerPeer peer.IDvar isLocal bool@@ -183,12 +186,13 @@ func evaluateExtAuthz(ctx context.Context, node *SamNode, in extAuthzCheckInput)reqPath = "/"}reqCtx := RequestContext{- PeerID: callerPeer,- Protocol: "ext_authz",- Target: target,- MCPTool: in.Headers[strings.ToLower(HeaderSamMCPTool)],- HTTP: &HTTPRequestFacts{Method: method, Path: reqPath},- Local: isLocal,+ PeerID: callerPeer,+ Protocol: "ext_authz",+ Target: target,+ MCPTool: in.Headers[strings.ToLower(HeaderSamMCPTool)],+ AllowMCPStreamInit: in.AllowMCPStreamInit,+ HTTP: &HTTPRequestFacts{Method: method, Path: reqPath},+ Local: isLocal,}if after, ok := strings.CutPrefix(target, api.EgressServicePrefix); ok {reqCtx.Egress = &EgressFacts{Host: after, Port: 443}@@ -295,7 +299,7 @@ func isExtAuthzServiceScheme(scheme string) bool {}}-func resolveExtAuthzTarget(in extAuthzCheckInput) (target, reqPath string) {+func resolveExtAuthzTarget(node *SamNode, in extAuthzCheckInput) (target, reqPath string) {path := in.Pathif idx := strings.IndexByte(path, '?'); idx >= 0 {path = path[:idx]@@ -328,6 +332,17 @@ func resolveExtAuthzTarget(in extAuthzCheckInput) (target, reqPath string) {return route.serviceType + "://" + route.serviceName, up}}+ if node != nil && node.services != nil && strings.TrimSpace(in.Host) != "" {+ normHost := api.NormalizeMeshHost(in.Host)+ if normHost != "" {+ if _, ok := node.services.GetTyped(api.ServiceType_SERVICE_TYPE_EGRESS, normHost); ok {+ if path == "" {+ path = "/"+ }+ return api.EgressServicePrefix + normHost, path+ }+ }+ }return "", path}@@ -566,6 +581,9 @@ func marshalEnvoyCheckResponse(res extAuthzCheckResult) []byte {var hvoBytes []bytehvoBytes = protowire.AppendTag(hvoBytes, 1, protowire.BytesType)hvoBytes = protowire.AppendBytes(hvoBytes, hvBytes)+ // Field 3: HeaderAppendAction append_action = OVERWRITE_IF_EXISTS_OR_ADD (2)+ hvoBytes = protowire.AppendTag(hvoBytes, 3, protowire.VarintType)+ hvoBytes = protowire.AppendVarint(hvoBytes, 2)okBytes = protowire.AppendTag(okBytes, 2, protowire.BytesType)okBytes = protowire.AppendBytes(okBytes, hvoBytes)diff --git a/internal/node/extproc_grpc.go b/internal/node/extproc_grpc.goindex f0cdcf71..06e3d0bf 100644--- a/internal/node/extproc_grpc.go+++ b/internal/node/extproc_grpc.go@@ -15,6 +15,7 @@package nodeimport (+ "bytes""context""crypto/tls""crypto/x509"@@ -38,7 +39,6 @@ import (extprocv3 "github.com/google/sam/third_party/envoy/envoy/service/ext_proc/v3"typev3 "github.com/google/sam/third_party/envoy/envoy/type/v3""google.golang.org/protobuf/proto"- "google.golang.org/protobuf/types/known/wrapperspb")// ExtProcMethodPath is the gRPC HTTP/2 path for Envoy ExternalProcessor.Process.@@ -344,18 +344,21 @@ func handleGatewayExtProc(node *SamNode, w http.ResponseWriter, r *http.Request)},}} else {- resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, false)+ resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, false, false)}case *extprocv3.ProcessingRequest_RequestBody:+ var allowStreamInit boolif len(phase.RequestBody.GetBody()) > 0 && capturedHeaders != nil {- if tool := extractJSONRPCMCPTool(phase.RequestBody.GetBody()); tool != "" {+ tool, allowInit := inspectJSONRPCMCPBody(phase.RequestBody.GetBody())+ if tool != "" {capturedHeaders[strings.ToLower(HeaderSamMCPTool)] = tool}+ allowStreamInit = allowInit}if pendingCheck {pendingCheck = false- resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, true)+ resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, true, allowStreamInit)} else {resp = &extprocv3.ProcessingResponse{Response: &extprocv3.ProcessingResponse_RequestBody{@@ -420,12 +423,13 @@ func handleGatewayExtProc(node *SamNode, w http.ResponseWriter, r *http.Request)w.Header().Set("Grpc-Message", "")}-func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method, path, host string, headers map[string]string, isBodyPhase bool) *extprocv3.ProcessingResponse {+func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method, path, host string, headers map[string]string, isBodyPhase, allowMCPStreamInit bool) *extprocv3.ProcessingResponse {res := evaluateExtAuthz(ctx, node, extAuthzCheckInput{- Method: method,- Path: path,- Host: host,- Headers: headers,+ Method: method,+ Path: path,+ Host: host,+ Headers: headers,+ AllowMCPStreamInit: allowMCPStreamInit,})if !res.Allowed {status := typev3.StatusCode_Forbidden@@ -447,17 +451,23 @@ func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method,for k, v := range res.ResponseHeaders {setHeaders = append(setHeaders, &corev3.HeaderValueOption{Header: &corev3.HeaderValue{- Key: k,- Value: v,+ Key: k,+ RawValue: []byte(v),},- Append: wrapperspb.Bool(false),AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD,})}+ var removeHeaders []string+ if _, hasUpstreamAuth := res.ResponseHeaders["Authorization"]; !hasUpstreamAuth {+ if headers["authorization"] != "" {+ removeHeaders = append(removeHeaders, "authorization")+ }+ }common := &extprocv3.CommonResponse{Status: extprocv3.CommonResponse_CONTINUE,HeaderMutation: &extprocv3.HeaderMutation{- SetHeaders: setHeaders,+ SetHeaders: setHeaders,+ RemoveHeaders: removeHeaders,},}if isBodyPhase {@@ -474,7 +484,7 @@ func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method,}}-func extractJSONRPCMCPTool(body []byte) string {+func inspectJSONRPCMCPBody(body []byte) (mcpTool string, allowStreamInit bool) {var rpc struct {Method string `json:"method"`Params struct {@@ -482,14 +492,37 @@ func extractJSONRPCMCPTool(body []byte) string {} `json:"params"`}if err := json.Unmarshal(body, &rpc); err != nil {- return ""+ return "", false}- if rpc.Method == "tools/call" && strings.TrimSpace(rpc.Params.Name) != "" {+ switch rpc.Method {+ case "initialize", "ping", "tools/list":+ return "", true+ case "tools/call":rawTool := strings.TrimSpace(rpc.Params.Name)+ if rawTool == "" {+ return "", false+ }if _, stripped, err := api.SplitToolName(rawTool); err == nil {- return stripped+ return stripped, false}- return rawTool+ return rawTool, false+ default:+ return "", false+ }+}++func inspectMCPHTTPRequestBody(r *http.Request) (string, bool, error) {+ if r == nil || r.Body == nil || !strings.EqualFold(r.Method, http.MethodPost) {+ return "", false, nil+ }+ bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, maxGRPCFrameBytes+1))+ if err != nil {+ return "", false, err+ }+ if int64(len(bodyBytes)) > maxGRPCFrameBytes {+ return "", false, fmt.Errorf("MCP request body exceeds maximum inspection size (%d bytes)", maxGRPCFrameBytes)}- return ""+ r.Body = io.NopCloser(bytes.NewReader(bodyBytes))+ tool, allowInit := inspectJSONRPCMCPBody(bodyBytes)+ return tool, allowInit, nil}diff --git a/internal/node/middleware.go b/internal/node/middleware.goindex 1bb91d26..7a4a7524 100644--- a/internal/node/middleware.go+++ b/internal/node/middleware.go@@ -58,6 +58,10 @@ type RequestContext struct {// MCPTool is set when evaluating a specific MCP tools/call invocation.MCPTool string+ // AllowMCPStreamInit is set when an MCP request is a session handshake or+ // discovery method (initialize, ping, tools/list).+ AllowMCPStreamInit bool+// TaskRules holds the verified TaskAuthorizationRule chain extracted from// the caller's Biscuit during stream authentication so downstream stream// handlers (such as MCP pass-through) can enforce tool-level constraints.@@ -378,7 +382,7 @@ func (n *SamNode) authorizeWithRules(rawToken []byte, req RequestContext, pubKeyServiceType: opType,ServiceName: opName,MCPTool: req.MCPTool,- AllowMCPStreamInit: req.HTTP == nil && req.MCPTool == "" && req.Protocol == string(api.MCPProtocolID),+ AllowMCPStreamInit: req.AllowMCPStreamInit || (req.HTTP == nil && req.MCPTool == "" && req.Protocol == string(api.MCPProtocolID)),}if req.HTTP != nil {taskReq.HasHTTP = truediff --git a/internal/node/node.go b/internal/node/node.goindex d87e11cd..72e69c46 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -2286,6 +2286,15 @@ func (n *SamNode) StartIngressServer(ctx context.Context) error {reqCtx.Egress = &EgressFacts{Host: serviceName, Port: reqPort}}}+ if serviceType == api.ServiceType_SERVICE_TYPE_MCP {+ mcpTool, allowInit, err := inspectMCPHTTPRequestBody(r)+ if err != nil {+ refuse(w, http.StatusRequestEntityTooLarge, err.Error(), proxyStatusDenied)+ return+ }+ reqCtx.MCPTool = mcpTool+ reqCtx.AllowMCPStreamInit = allowInit+ }// Verify authorizationif err := n.VerifyBiscuitToken(biscuitBytes, reqCtx); err != nil {diff --git a/internal/node/sts.go b/internal/node/sts.goindex 6873b44f..b0418c19 100644--- a/internal/node/sts.go+++ b/internal/node/sts.go@@ -350,14 +350,10 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallbareturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {logger.Debugf("[SidecarAuth] Incoming request: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr)- // Extract optional X-Sam-Authentication or Authorization bearer value.+ // Extract optional X-Sam-Authentication, X-Sam-Biscuit (injected by an+ // upstream Envoy ext_authz / ext_proc filter), or Authorization bearer value.headerName := api.HeaderSamAuthenticationauthHeader := r.Header.Get(headerName)- if authHeader == "" && allowAuthorizationFallback {- headerName = "Authorization"- authHeader = r.Header.Get(headerName)- }-var bearer stringvar hasBearer boolif authHeader != "" {@@ -366,13 +362,28 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallbabearer = strings.TrimSpace(parts[1])hasBearer = true}+ } else if b64 := strings.TrimSpace(r.Header.Get(api.HeaderSamBiscuit)); b64 != "" {+ headerName = api.HeaderSamBiscuit+ authHeader = b64+ bearer = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(b64, "Bearer "), "bearer "))+ hasBearer = bearer != ""+ } else if allowAuthorizationFallback {+ headerName = "Authorization"+ authHeader = r.Header.Get(headerName)+ if authHeader != "" {+ parts := strings.SplitN(authHeader, " ", 2)+ if len(parts) == 2 && strings.EqualFold(parts[0], "bearer") {+ bearer = strings.TrimSpace(parts[1])+ hasBearer = true+ }+ }}// Local Unix socket or mTLS (token == ""): transport already authenticates// the local process, but if the caller also supplied a Task Biscuit or JWT,// bind it to the request context (and fail closed if that Biscuit/JWT is invalid/revoked).if fromLocalSocket(r) || token == "" {- if hasBearer && (token == "" || !constantTimeEqual(bearer, token)) {+ if hasBearer && (token == "" || headerName == api.HeaderSamBiscuit || !constantTimeEqual(bearer, token)) {biscuitBytes, isMeshCred, err := node.resolveCallerCredential(r.Context(), bearer)if isMeshCred {if err != nil {@@ -381,12 +392,13 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallba}r = r.WithContext(WithCallerBiscuit(r.Context(), biscuitBytes))r.Header.Del(headerName)- } else if headerName == api.HeaderSamAuthentication {+ } else if headerName == api.HeaderSamAuthentication || headerName == api.HeaderSamBiscuit {http.Error(w, "Forbidden", http.StatusForbidden)return}}r.Header.Del(api.HeaderSamAuthentication)+ r.Header.Del(api.HeaderSamBiscuit)stripSidecarTokenFromAuthorization(r, token)next.ServeHTTP(w, r)return@@ -406,7 +418,7 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallbareturn}- if constantTimeEqual(bearer, token) {+ if headerName != api.HeaderSamBiscuit && constantTimeEqual(bearer, token) {r.Header.Del(headerName)stripSidecarTokenFromAuthorization(r, token)next.ServeHTTP(w, r)diff --git a/internal/node/sts_test.go b/internal/node/sts_test.goindex c88896ab..4312d564 100644--- a/internal/node/sts_test.go+++ b/internal/node/sts_test.go@@ -22,6 +22,7 @@ import ("encoding/base64""encoding/hex""encoding/json"+ "io""net/http""net/http/httptest""net/url"@@ -525,3 +526,25 @@ func buildEnvoyCheckRequestPayload(method, path string, headers map[string]strincheckReq = protowire.AppendBytes(checkReq, attrBytes)return checkReq}++func TestInspectMCPHTTPRequestBody(t *testing.T) {+ body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mcp://github/get_pr"}}`+ req := httptest.NewRequest(http.MethodPost, "/mcp/github", strings.NewReader(body))+ tool, allowInit, err := inspectMCPHTTPRequestBody(req)+ if err != nil {+ t.Fatalf("inspectMCPHTTPRequestBody: %v", err)+ }+ if tool != "get_pr" || allowInit {+ t.Fatalf("got tool=%q allowInit=%v, want tool=\"get_pr\" allowInit=false", tool, allowInit)+ }+ restored, _ := io.ReadAll(req.Body)+ if string(restored) != body {+ t.Fatalf("body was not restored: got %q, want %q", string(restored), body)+ }++ initReq := httptest.NewRequest(http.MethodPost, "/mcp/github", strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"initialize"}`))+ tool, allowInit, err = inspectMCPHTTPRequestBody(initReq)+ if err != nil || tool != "" || !allowInit {+ t.Fatalf("initialize: got tool=%q allowInit=%v err=%v, want allowInit=true", tool, allowInit, err)+ }+}diff --git a/tests/e2e/fixtures/fake_openai.py b/tests/e2e/fixtures/fake_openai.pyindex dd3ffa54..ab48c57c 100644--- a/tests/e2e/fixtures/fake_openai.py+++ b/tests/e2e/fixtures/fake_openai.py@@ -2,7 +2,7 @@# A minimal OpenAI-compatible backend for e2e tests: one model, one canned# completion. Standard library only, so it runs wherever python3 does.import json-from http.server import BaseHTTPRequestHandler, HTTPServer+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServerMODEL = "e2e-model"@@ -36,6 +36,51 @@ class Handler(BaseHTTPRequestHandler):"usage": {"prompt_tokens": 1, "completion_tokens": 4, "total_tokens": 5},})return+ if self.path == "/mcp" or self.path.startswith("/mcp/"):+ method = req.get("method", "")+ req_id = req.get("id", 1)+ if method == "initialize":+ self._json(200, {+ "jsonrpc": "2.0",+ "id": req_id,+ "result": {+ "protocolVersion": "2024-11-05",+ "capabilities": {"tools": {}},+ "serverInfo": {"name": "github", "version": "1.0"},+ },+ })+ return+ if method == "notifications/initialized":+ self.send_response(202)+ self.send_header("Content-Length", "0")+ self.end_headers()+ return+ if method == "tools/list":+ self._json(200, {+ "jsonrpc": "2.0",+ "id": req_id,+ "result": {+ "tools": [+ {"name": "get_pr", "description": "Get pull request"},+ {"name": "delete_repo", "description": "Delete repository"},+ ]+ },+ })+ return+ if method == "tools/call":+ peer_id = self.headers.get("X-Peer-Id", "")+ tool_name = (req.get("params") or {}).get("name", "")+ self._json(200, {+ "jsonrpc": "2.0",+ "id": req_id,+ "result": {+ "content": [{+ "type": "text",+ "text": f"tool={tool_name} peer={peer_id}",+ }]+ },+ })+ returnself._json(404, {"error": {"message": "unknown model", "code": "model_not_found"}})def log_message(self, *args):@@ -43,6 +88,7 @@ class Handler(BaseHTTPRequestHandler):if __name__ == "__main__":- server = HTTPServer(("127.0.0.1", 0), Handler)+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)print(f"listening on port {server.server_address[1]}", flush=True)server.serve_forever()+diff --git a/tests/e2e/standalone.bats b/tests/e2e/standalone.batsindex eca74cfd..f25bbcc3 100644--- a/tests/e2e/standalone.bats+++ b/tests/e2e/standalone.bats@@ -22,6 +22,9 @@ setup() {}teardown() {+ if [[ -n "${ENVOY_CONTAINER:-}" ]]; then+ docker rm -f "$ENVOY_CONTAINER" >/dev/null 2>&1 || true+ fifor pid in "${BACKEND_PID:-}" "${NODE_A_PID:-}" "${NODE_B_PID:-}" "${SAM_ONE_PID:-}"; do[[ -n "$pid" ]] && kill "$pid" 2>/dev/null || truedone@@ -92,13 +95,13 @@ start_node() {--data-dir "$TEST_TMPDIR/node-$name" \--bind-addr= \--allow-loopback \- --listen "/ip4/127.0.0.1/tcp/0" "$@" > "$TEST_TMPDIR/node-$name.log" 2>&1 &+ --listen "/ip4/127.0.0.1/tcp/0" "$@" > "$TEST_TMPDIR/node-$name.log" 2>&1 3>&- &eval "NODE_${name^^}_PID=$!"}@test "sam-one boots, nodes join over the single port, and the dataplane carries a service call" {"$SAM_ONE_BINARY" --bind-address 127.0.0.1 --port 0 \- --data-dir "$SAM_ONE_DATA" > "$TEST_TMPDIR/sam-one.log" 2>&1 &+ --data-dir "$SAM_ONE_DATA" > "$TEST_TMPDIR/sam-one.log" 2>&1 3>&- &SAM_ONE_PID=$!# The random port is published in the banner, like the generated tokens.@@ -118,13 +121,14 @@ start_node() {JOIN_TOKEN="$(cat "$SAM_ONE_DATA/join-token")"[[ "$JOIN_TOKEN" == sam_tok_* ]]- # An OpenAI-compatible backend on node A's host, declared as an inference- # service in node A's configuration, as in docs/getting-started/your-own-mesh.- # Services only exist by declaration at startup, there is no runtime- # registration endpoint. The same server is also declared as `mcp`: the- # type is a contract the node verifies, so that one must never be advertised.+ # An OpenAI-compatible and MCP backend on node A's host, declared in node A's+ # configuration, as in docs/getting-started/your-own-mesh. Services only exist+ # by declaration at startup, there is no runtime registration endpoint. The+ # root URL (/) is also declared as `mcp` (not-an-mcp-server): the type is a+ # contract the node verifies, so that one must never be advertised, while+ # /mcp speaks MCP JSON-RPC and is advertised as mcp://github.python3 -u "$BATS_TEST_DIRNAME/fixtures/fake_openai.py" \- > "$TEST_TMPDIR/backend.log" 2>&1 &+ > "$TEST_TMPDIR/backend.log" 2>&1 3>&- &BACKEND_PID=$!wait_for_log "$TEST_TMPDIR/backend.log" "listening on port"backend_port="$(grep -oE 'port [0-9]+' "$TEST_TMPDIR/backend.log" | grep -oE '[0-9]+')"@@ -136,6 +140,10 @@ services:name: laptop-llmdescription: "e2e inference backend"target_url: "http://127.0.0.1:${backend_port}"+ - type: mcp+ name: github+ description: "e2e mcp backend"+ target_url: "http://127.0.0.1:${backend_port}/mcp"- type: mcpname: not-an-mcp-serverdescription: "a plain HTTP server mislabelled as mcp"@@ -143,11 +151,19 @@ services:EOFstart_node a --config "$TEST_TMPDIR/node-a-services.yaml"- start_node b+ # Start node B with a local TCP sidecar listener (127.0.0.1:0) in addition to+ # its Unix socket so a real Envoy proxy can dial node B over h2c and forward+ # authorized traffic through node B into the mesh.+ start_node b --bind-addr "127.0.0.1:0"wait_for_node await_for_node b+ wait_for_log "$TEST_TMPDIR/node-b.log" "Starting MCP server on TCP address"+ node_b_tcp_port="$(grep -oE 'Starting MCP server on TCP address 127\.0\.0\.1:[0-9]+' "$TEST_TMPDIR/node-b.log" | grep -oE '[0-9]+$')"+ [[ -n "$node_b_tcp_port" ]]+peer_a="$(node_peer_id a)"- [[ -n "$peer_a" ]]+ peer_b="$(node_peer_id b)"+ [[ -n "$peer_a" && -n "$peer_b" ]]sock_a="$TEST_TMPDIR/node-a/sam.sock"sock_b="$TEST_TMPDIR/node-b/sam.sock"@@ -201,6 +217,161 @@ EOF-d '{"model":"e2e-model","messages":[{"role":"user","content":"should fail"}]}')"[[ "$http_code" -ne 200 ]]+ # Real Envoy proxy CUJ: when Docker is available, place Envoy in front of node B+ # with both ext_authz and ext_proc configured over gRPC h2c, and drive real+ # inference (/v1/chat/completions) and MCP (/sam/<peer_a>/mcp/github) calls+ # through Envoy -> Node B -> sam-one Router -> Node A -> Backend.+ if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then+ envoy_port="$(python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"++ cat > "$TEST_TMPDIR/envoy.yaml" <<EOF+static_resources:+ listeners:+ - name: listener_0+ address:+ socket_address:+ address: 127.0.0.1+ port_value: ${envoy_port}+ filter_chains:+ - filters:+ - name: envoy.filters.network.http_connection_manager+ typed_config:+ "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager+ stat_prefix: ingress_http+ route_config:+ name: local_route+ virtual_hosts:+ - name: local_service+ domains: ["*"]+ routes:+ - match:+ prefix: "/"+ route:+ cluster: sam_node_b_sidecar+ http_filters:+ - name: envoy.filters.http.ext_authz+ typed_config:+ "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz+ transport_api_version: V3+ grpc_service:+ envoy_grpc:+ cluster_name: sam_node_b_h2c+ timeout: 5s+ - name: envoy.filters.http.ext_proc+ typed_config:+ "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor+ grpc_service:+ envoy_grpc:+ cluster_name: sam_node_b_h2c+ timeout: 5s+ allow_mode_override: true+ message_timeout: 5s+ processing_mode:+ request_header_mode: SEND+ response_header_mode: SKIP+ request_body_mode: NONE+ response_body_mode: NONE+ - name: envoy.filters.http.router+ typed_config:+ "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router+ clusters:+ - name: sam_node_b_h2c+ type: STATIC+ connect_timeout: 2s+ typed_extension_protocol_options:+ envoy.extensions.upstreams.http.v3.HttpProtocolOptions:+ "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions+ explicit_http_config:+ http2_protocol_options: {}+ load_assignment:+ cluster_name: sam_node_b_h2c+ endpoints:+ - lb_endpoints:+ - endpoint:+ address:+ socket_address:+ address: 127.0.0.1+ port_value: ${node_b_tcp_port}+ - name: sam_node_b_sidecar+ type: STATIC+ connect_timeout: 2s+ load_assignment:+ cluster_name: sam_node_b_sidecar+ endpoints:+ - lb_endpoints:+ - endpoint:+ address:+ socket_address:+ address: 127.0.0.1+ port_value: ${node_b_tcp_port}+EOF++ chmod 0644 "$TEST_TMPDIR/envoy.yaml"+ ENVOY_CONTAINER="sam-e2e-envoy-$$"+ docker run -d --name "$ENVOY_CONTAINER" \+ --network host \+ -v "$TEST_TMPDIR/envoy.yaml:/etc/envoy/envoy.yaml:ro" \+ envoyproxy/envoy:v1.32-latest \+ -c /etc/envoy/envoy.yaml --log-level warning >/dev/null 3>&-++ envoy_ready=0+ for _ in $(seq 1 50); do+ code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${envoy_port}/sam/${peer_a}/mcp/github" -X POST || true)"+ if [[ "$code" == "401" ]]; then+ envoy_ready=1+ break+ fi+ sleep 0.2+ done+ if [[ "$envoy_ready" -ne 1 ]]; then+ docker logs "$ENVOY_CONTAINER" >&2 || true+ cat "$TEST_TMPDIR/node-b.log" >&2 || true+ return 1+ fi++ # 1. OpenAI inference completion through Envoy -> Node B -> Router -> Node A -> fake_openai.py+ run curl -sS -i "http://127.0.0.1:${envoy_port}/v1/chat/completions" \+ -H "Authorization: Bearer $task_biscuit" \+ -H "X-Sam-Target-Service: inference://laptop-llm" \+ -H 'Content-Type: application/json' \+ -d '{"model":"e2e-model","messages":[{"role":"user","content":"hi through envoy"}]}'+ if [[ "$status" -ne 0 || "$output" != *"hello from the mesh"* ]]; then+ echo "Envoy /v1/chat/completions failed ($status): $output" >&2+ docker logs "$ENVOY_CONTAINER" >&2 || true+ tail -n 50 "$TEST_TMPDIR/node-b.log" >&2 || true+ return 1+ fi++ # 2. Mint a Task Biscuit narrowed to mcp://github and tool:get_pr.+ mcp_task_biscuit="$(curl -sf --unix-socket "$sock_b" "http://localhost/oauth/token" \+ -d 'grant_type=urn:ietf:params:oauth:grant-type:token-exchange' \+ -d 'resource=mcp://github' \+ -d 'scope=tool:get_pr' |+ python3 -c 'import json, sys; print(json.load(sys.stdin)["access_token"])')"+ [[ -n "$mcp_task_biscuit" ]]++ # 2a. Allowed MCP tools/call ("get_pr") traverses Envoy (ext_authz + ext_proc body buffering) ->+ # Node B (/sam/<peer_a>/mcp/github) -> Router -> Node A -> MCP backend!+ run curl -sS -i "http://127.0.0.1:${envoy_port}/sam/${peer_a}/mcp/github" \+ -H "Authorization: Bearer $mcp_task_biscuit" \+ -H "Content-Type: application/json" \+ -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"get_pr"}}'+ if [[ "$status" -ne 0 || "$output" != *"tool=get_pr peer=${peer_b}"* ]]; then+ echo "Envoy MCP tools/call failed ($status): $output" >&2+ docker logs "$ENVOY_CONTAINER" >&2 || true+ tail -n 50 "$TEST_TMPDIR/node-b.log" >&2 || true+ tail -n 50 "$TEST_TMPDIR/node-a.log" >&2 || true+ return 1+ fi++ # 2b. Disallowed MCP tools/call ("delete_repo") is blocked at Envoy by ext_proc body inspection with 403 Forbidden.+ deny_code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${envoy_port}/sam/${peer_a}/mcp/github" \+ -H "Authorization: Bearer $mcp_task_biscuit" \+ -H "Content-Type: application/json" \+ -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"delete_repo"}}')"+ [[ "$deny_code" -eq 403 ]]+ fi+# Revoking the task Biscuit via RFC 7009 POST /oauth/revoke immediately blocks# reuse of that task Biscuit without revoking node B's standing identity.run curl -sf --unix-socket "$sock_b" "http://localhost/oauth/revoke" \@@ -214,6 +385,15 @@ EOF-d '{"model":"e2e-model","messages":[{"role":"user","content":"revoked task"}]}')"[[ "$revoked_code" -eq 403 ]]+ if [[ -n "${envoy_port:-}" ]]; then+ envoy_revoked_code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${envoy_port}/v1/chat/completions" \+ -H "Authorization: Bearer $task_biscuit" \+ -H "X-Sam-Target-Service: inference://laptop-llm" \+ -H 'Content-Type: application/json' \+ -d '{"model":"e2e-model","messages":[{"role":"user","content":"revoked task via envoy"}]}')"+ [[ "$envoy_revoked_code" -eq 403 ]]+ fi+# The plain HTTP server never shows up as an MCP provider: node A had# registered every configured service before its API came up, so its# absence here is a verdict, not a race.@@ -235,3 +415,4 @@ EOFjoin_row="$(echo "$output" | grep "sam-one join token")"[[ "$join_row" == *" 2/"* ]]}+diff --git a/tests/integration/sts_cuj_test.go b/tests/integration/sts_cuj_test.goindex 8a7d153d..b8b9611b 100644--- a/tests/integration/sts_cuj_test.go+++ b/tests/integration/sts_cuj_test.go@@ -17,42 +17,175 @@ package integration_testimport ("crypto/sha256""encoding/base64"+ "encoding/binary""encoding/json""fmt""io"+ "net""net/http""net/http/httptest""net/url""os""path/filepath""strings"+ "sync/atomic""testing""time""github.com/google/sam/api"+ corev3 "github.com/google/sam/third_party/envoy/envoy/config/core/v3"+ extprocv3http "github.com/google/sam/third_party/envoy/envoy/extensions/filters/http/ext_proc/v3"+ extprocv3 "github.com/google/sam/third_party/envoy/envoy/service/ext_proc/v3"+ typev3 "github.com/google/sam/third_party/envoy/envoy/type/v3"+ "google.golang.org/protobuf/proto")+func writeExtProcFrame(w io.Writer, msg proto.Message) error {+ payload, err := proto.Marshal(msg)+ if err != nil {+ return err+ }+ var hdr [5]byte+ hdr[0] = 0+ binary.BigEndian.PutUint32(hdr[1:5], uint32(len(payload)))+ if _, err := w.Write(hdr[:]); err != nil {+ return err+ }+ _, err = w.Write(payload)+ return err+}++func readExtProcFrame(r io.Reader, msg proto.Message) error {+ var hdr [5]byte+ if _, err := io.ReadFull(r, hdr[:]); err != nil {+ return err+ }+ length := binary.BigEndian.Uint32(hdr[1:5])+ payload := make([]byte, length)+ if _, err := io.ReadFull(r, payload); err != nil {+ return err+ }+ return proto.Unmarshal(payload, msg)+}++func startH2CExtProcCallout(t *testing.T, handler http.Handler) string {+ t.Helper()+ ln, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen tcp: %v", err)+ }+ var protocols http.Protocols+ protocols.SetHTTP1(true)+ protocols.SetUnencryptedHTTP2(true)+ srv := &http.Server{+ Handler: handler,+ Protocols: &protocols,+ }+ go func() { _ = srv.Serve(ln) }()+ t.Cleanup(func() {+ _ = srv.Close()+ _ = ln.Close()+ })+ return ln.Addr().String()+}+// TestSTSTaskScopedSecurityCUJ validates the Phase 3 Security Token Service (STS),-// Task-Scoped Biscuit Attenuation, OAuth 2.1 / PKCE, Envoy ext_authz, RFC 7009-// Revocation, and Outbound Border JWT Minting journeys end-to-end across real-// sam-control-plane, sam-router, and sam-node processes.+// Task-Scoped Biscuit Attenuation, OAuth 2.1 / PKCE, Envoy ext_authz and ext_proc,+// RFC 7009 Revocation, and Outbound Border JWT Minting journeys end-to-end across+// real sam-control-plane, sam-router, and sam-node processes.func TestSTSTaskScopedSecurityCUJ(t *testing.T) {nodeBin := buildBinary(t, "./cmd/sam-node")tmpDir := t.TempDir()oidcURL, mintToken := startCustomMockOIDC(t)+ var upstreamAuthHeader atomic.Value+ var upstreamInspectedHeader atomic.Valueupstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- w.WriteHeader(http.StatusNoContent)+ upstreamAuthHeader.Store(r.Header.Get("Authorization"))+ upstreamInspectedHeader.Store(r.Header.Get("X-Ext-Proc-Inspected"))+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ _, _ = fmt.Fprintf(w, `{"path":%q,"auth":%q,"roles":%q}`, r.URL.Path, r.Header.Get("Authorization"), r.Header.Get(api.HeaderSamRoles))}))defer upstream.Close()+ extProcCalloutAddr := startH2CExtProcCallout(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/grpc+proto")+ w.Header().Set("Trailer", "Grpc-Status, Grpc-Message")+ w.WriteHeader(http.StatusOK)+ rc := http.NewResponseController(w)+ _ = rc.Flush()+ for {+ var req extprocv3.ProcessingRequest+ if err := readExtProcFrame(r.Body, &req); err != nil {+ break+ }+ var resp *extprocv3.ProcessingResponse+ if rh := req.GetRequestHeaders(); rh != nil {+ var path string+ for _, hv := range rh.GetHeaders().GetHeaders() {+ if hv.GetKey() == ":path" {+ path = hv.GetValue()+ if path == "" {+ path = string(hv.GetRawValue())+ }+ }+ }+ if strings.HasSuffix(path, "/blocked-by-dlp") {+ resp = &extprocv3.ProcessingResponse{+ Response: &extprocv3.ProcessingResponse_ImmediateResponse{+ ImmediateResponse: &extprocv3.ImmediateResponse{+ Status: &typev3.HttpStatus{Code: typev3.StatusCode_Forbidden},+ Body: []byte("blocked by ext_proc callout"),+ Details: "dlp_violation",+ },+ },+ }+ } else {+ resp = &extprocv3.ProcessingResponse{+ Response: &extprocv3.ProcessingResponse_RequestHeaders{+ RequestHeaders: &extprocv3.HeadersResponse{+ Response: &extprocv3.CommonResponse{+ Status: extprocv3.CommonResponse_CONTINUE,+ HeaderMutation: &extprocv3.HeaderMutation{+ SetHeaders: []*corev3.HeaderValueOption{+ {Header: &corev3.HeaderValue{Key: "X-Ext-Proc-Inspected", Value: "true"}},+ },+ },+ },+ },+ },+ }+ }+ } else if req.GetResponseHeaders() != nil {+ resp = &extprocv3.ProcessingResponse{+ Response: &extprocv3.ProcessingResponse_ResponseHeaders{+ ResponseHeaders: &extprocv3.HeadersResponse{+ Response: &extprocv3.CommonResponse{+ Status: extprocv3.CommonResponse_CONTINUE,+ },+ },+ },+ }+ }+ if resp != nil {+ _ = writeExtProcFrame(w, resp)+ _ = rc.Flush()+ if resp.GetImmediateResponse() != nil {+ break+ }+ }+ }+ w.Header().Set("Grpc-Status", "0")+ }))+policyFile := filepath.Join(tmpDir, "policies.yaml")policyYAML := fmt.Sprintf(`roles:- name: pep- allowed_services: ["egress://api.github.com"]+ allowed_services: ["egress://api.github.com", "mcp://github"]allowed_targets: ["*"]- name: developer- allowed_services: ["egress://api.github.com"]+ allowed_services: ["egress://api.github.com", "mcp://github"]allowed_targets: ["*"]http:- service: "egress://api.github.com"@@ -79,7 +212,12 @@ egress:target_url: %qcredential: github-tokenserved_by: ["pep"]-`, upstream.URL)+ inspection:+ inspectors:+ - ext_proc:+ target: %q+ allow_mode_override: true+`, upstream.URL, extProcCalloutAddr)if err := os.WriteFile(policyFile, []byte(policyYAML), 0o644); err != nil {t.Fatal(err)}@@ -96,10 +234,41 @@ egress:t.Fatal(err)}+ var mcpBackendCalls atomic.Int32+ mcpBackend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var rpc struct {+ ID any `json:"id"`+ Method string `json:"method"`+ }+ body, _ := io.ReadAll(r.Body)+ _ = json.Unmarshal(body, &rpc)+ w.Header().Set("Content-Type", "application/json")+ switch rpc.Method {+ case "initialize":+ _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"github","version":"1.0"}}}`))+ case "notifications/initialized":+ w.WriteHeader(http.StatusAccepted)+ case "tools/list":+ _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"tools":[{"name":"get_pr","description":"Get PR"},{"name":"delete_repo","description":"Delete repo"}]}}`))+ default:+ mcpBackendCalls.Add(1)+ _, _ = fmt.Fprintf(w, `{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"PR #42 from %s peer=%s biscuit=%s"}]}}`,+ r.Header.Get(api.HeaderSamPrincipal),+ r.Header.Get(api.HeaderPeerID),+ r.Header.Get(api.HeaderSamBiscuit))+ }+ }))+ defer mcpBackend.Close()++ pepDir := filepath.Join(tmpDir, "pep")+ if err := os.MkdirAll(pepDir, 0o700); err != nil {+ t.Fatal(err)+ }+ pepCfg := writeNodeConfig(t, pepDir, nil, svcDecl{Type: "mcp", Name: "github", TargetURL: mcpBackend.URL})pepToken := "test-token"- pep := launchNode(t, nodeBin, os.Environ(), filepath.Join(tmpDir, "pep"), "run",+ pep := launchNode(t, nodeBin, os.Environ(), pepDir, "run","--control-plane", controlPlane,- "--data-dir", filepath.Join(tmpDir, "pep"),+ "--data-dir", pepDir,"--api-token-path", tokenPath(t, pepToken),"--jwt", mintToken(map[string]any{"sub": "pep-user", "roles": []string{api.RoleNode}}),"--listen", "/ip4/127.0.0.1/udp/0/quic-v1",@@ -108,6 +277,7 @@ egress:"--discovery-interval", "100ms","--control-plane-sync-interval", "1s","--secrets-dir", secrets,+ "--config", pepCfg,)pepAPI := pep.waitForAPI(t)@@ -127,6 +297,7 @@ egress:connectPeer(t, callerAPI, pep.p2pAddr)waitForDHTPeers(t, pepAPI)pepPeer := extractPeerID(pep.p2pAddr)+ callerPeer := extractPeerID(caller.p2pAddr)waitForDiscoverableService(t, callerAPI, callerToken, "egress", "api.github.com")client := &http.Client{@@ -206,38 +377,93 @@ egress:}})- t.Run("Inbound external JWT exchange and role isolation via ext_authz", func(t *testing.T) {+ t.Run("Inbound external JWT exchange and role isolation via ext_authz datapath", func(t *testing.T) {contractorJWT := mintToken(map[string]any{"sub": "contractor-user", "email": "[email protected]"})- // contractor is allowed GET /repos/acme/public/*- reqAllow, _ := http.NewRequest(http.MethodGet, "http://"+callerAPI+"/ext_authz/egress/api.github.com/repos/acme/public/readme", nil)+ // Gateway proxy in front of caller node's mesh dataplane that consults+ // caller node's /ext_authz before forwarding through /sam/<pepPeer>/egress/api.github.com.+ extAuthzGateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ checkURL := "http://" + callerAPI + "/ext_authz/egress/api.github.com" + r.URL.Path+ checkReq, err := http.NewRequestWithContext(r.Context(), r.Method, checkURL, nil)+ if err != nil {+ http.Error(w, err.Error(), http.StatusInternalServerError)+ return+ }+ checkReq.Header = r.Header.Clone()+ checkResp, err := client.Do(checkReq)+ if err != nil {+ http.Error(w, err.Error(), http.StatusBadGateway)+ return+ }+ _ = checkResp.Body.Close()+ if checkResp.StatusCode != http.StatusOK {+ w.WriteHeader(checkResp.StatusCode)+ return+ }++ // Forward the authorized request into caller node's mesh dataplane using the+ // X-Sam-Biscuit injected by ext_authz.+ meshURL := fmt.Sprintf("http://%s/sam/%s/egress/api.github.com%s", callerAPI, pepPeer, r.URL.Path)+ fwdReq, err := http.NewRequestWithContext(r.Context(), r.Method, meshURL, r.Body)+ if err != nil {+ http.Error(w, err.Error(), http.StatusInternalServerError)+ return+ }+ for _, h := range []string{api.HeaderSamBiscuit, api.HeaderSamPrincipal, api.HeaderSamRoles, api.HeaderSamTask} {+ if v := checkResp.Header.Get(h); v != "" {+ fwdReq.Header.Set(h, v)+ }+ }+ fwdResp, err := client.Do(fwdReq)+ if err != nil {+ http.Error(w, err.Error(), http.StatusBadGateway)+ return+ }+ defer func() { _ = fwdResp.Body.Close() }()+ for k, vals := range fwdResp.Header {+ for _, v := range vals {+ w.Header().Add(k, v)+ }+ }+ w.Header().Set(api.HeaderSamRoles, checkResp.Header.Get(api.HeaderSamRoles))+ w.WriteHeader(fwdResp.StatusCode)+ _, _ = io.Copy(w, fwdResp.Body)+ }))+ defer extAuthzGateway.Close()++ // 1. contractor is allowed GET /repos/acme/public/readme -> traverses ext_authz -> caller node -> libp2p -> pep node -> ext_proc callout -> upstream!+ reqAllow, _ := http.NewRequest(http.MethodGet, extAuthzGateway.URL+"/repos/acme/public/readme", nil)reqAllow.Header.Set("Authorization", "Bearer "+contractorJWT)respAllow, err := client.Do(reqAllow)if err != nil {- t.Fatalf("ext_authz allow request: %v", err)+ t.Fatalf("ext_authz gateway allow request: %v", err)}+ allowBody, _ := io.ReadAll(respAllow.Body)_ = respAllow.Body.Close()if respAllow.StatusCode != http.StatusOK {- t.Fatalf("ext_authz allow status = %d, want 200", respAllow.StatusCode)+ t.Fatalf("ext_authz datapath allow status = %d, want 200: %s", respAllow.StatusCode, string(allowBody))}if got := respAllow.Header.Get(api.HeaderSamRoles); got != "contractor" {t.Fatalf("ext_authz X-Sam-Roles = %q, want \"contractor\" (no node role leakage)", got)}+ if !strings.Contains(string(allowBody), `"path":"/repos/acme/public/readme"`) || !strings.Contains(string(allowBody), `"auth":"Bearer ghp_upstream_secret"`) {+ t.Fatalf("unexpected upstream datapath payload: %s", string(allowBody))+ }- // contractor is denied GET /repos/acme/private/secret (even though caller node has developer role!)- reqDeny, _ := http.NewRequest(http.MethodGet, "http://"+callerAPI+"/ext_authz/egress/api.github.com/repos/acme/private/secret", nil)+ // 2. contractor is denied GET /repos/acme/private/secret (even though caller node has developer role!)+ reqDeny, _ := http.NewRequest(http.MethodGet, extAuthzGateway.URL+"/repos/acme/private/secret", nil)reqDeny.Header.Set("Authorization", "Bearer "+contractorJWT)respDeny, err := client.Do(reqDeny)if err != nil {- t.Fatalf("ext_authz deny request: %v", err)+ t.Fatalf("ext_authz gateway deny request: %v", err)}_ = respDeny.Body.Close()if respDeny.StatusCode != http.StatusForbidden {- t.Fatalf("ext_authz deny status = %d, want 403", respDeny.StatusCode)+ t.Fatalf("ext_authz datapath deny status = %d, want 403", respDeny.StatusCode)}})- t.Run("Task biscuit attenuation, mesh enforcement, outbound STS JWT minting, and task revocation", func(t *testing.T) {+ t.Run("Task biscuit attenuation, mesh enforcement, egress ext_proc callout, outbound STS JWT minting, and task revocation", func(t *testing.T) {// 1. Exchange caller node's standing Biscuit into a narrowed Task Biscuit// only allowing GET /repos/acme/public/* on egress://api.github.com.form := url.Values{}@@ -264,29 +490,43 @@ egress:}baseMeshURL := fmt.Sprintf("http://%s/sam/%s/egress/api.github.com", callerAPI, pepPeer)- doWithCred := func(method, path, cred string) int {+ doWithCred := func(method, path, cred string) (int, string) {req, _ := http.NewRequest(method, baseMeshURL+path, nil)req.Header.Set(api.HeaderSamAuthentication, "Bearer "+cred)resp, err := client.Do(req)if err != nil {t.Fatalf("%s %s: %v", method, path, err)}+ body, _ := io.ReadAll(resp.Body)_ = resp.Body.Close()- return resp.StatusCode+ return resp.StatusCode, string(body)}// 2. Allowed by both developer role and tar_block: GET /repos/acme/public/readme- if got := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusNoContent {- t.Fatalf("GET /repos/acme/public/readme with taskBiscuit: %d, want 204", got)+ // Also verifies the egress ext_proc callout mutated X-Ext-Proc-Inspected=true+ // and sam-node injected Authorization: Bearer ghp_upstream_secret.+ if got, body := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusOK || !strings.Contains(body, `"auth":"Bearer ghp_upstream_secret"`) {+ t.Fatalf("GET /repos/acme/public/readme with taskBiscuit: status=%d body=%s", got, body)+ }+ if got, _ := upstreamAuthHeader.Load().(string); got != "Bearer ghp_upstream_secret" {+ t.Fatalf("upstream Authorization = %q, want Bearer ghp_upstream_secret", got)+ }+ if got, _ := upstreamInspectedHeader.Load().(string); got != "true" {+ t.Fatalf("upstream X-Ext-Proc-Inspected = %q, want true", got)+ }++ // 2b. Blocked by egress ext_proc callout ImmediateResponse (path allowed by RBAC + TAR, blocked by DLP callout):+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/public/blocked-by-dlp", taskBiscuit); got != http.StatusForbidden {+ t.Fatalf("GET /repos/acme/public/blocked-by-dlp with taskBiscuit: %d, want 403 (blocked by ext_proc callout)", got)}// 3. Denied by tar_block method constraint (developer role allows POST, but tar_block only allows GET):- if got := doWithCred(http.MethodPost, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {+ if got, _ := doWithCred(http.MethodPost, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {t.Fatalf("POST /repos/acme/public/readme with taskBiscuit: %d, want 403", got)}// 4. Denied by tar_block path constraint (developer role allows /repos/acme/*, but tar_block only allows /repos/acme/public/*):- if got := doWithCred(http.MethodGet, "/repos/acme/private/secret", taskBiscuit); got != http.StatusForbidden {+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/private/secret", taskBiscuit); got != http.StatusForbidden {t.Fatalf("GET /repos/acme/private/secret with taskBiscuit: %d, want 403", got)}@@ -328,11 +568,256 @@ egress:}// Revoked taskBiscuit is now rejected immediately, while caller's parent node identity still works!- if got := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {t.Fatalf("revoked taskBiscuit status = %d, want 403", got)}- if got := doWithCred(http.MethodGet, "/repos/acme/private/secret", callerToken); got != http.StatusNoContent {- t.Fatalf("parent node identity status = %d, want 204", got)+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/private/secret", callerToken); got != http.StatusOK {+ t.Fatalf("parent node identity status = %d, want 200", got)+ }+ })++ t.Run("Gateway ext_proc over h2c datapath on live sam-node (egress credential injection and MCP tools/call across mesh)", func(t *testing.T) {+ var h2cProtocols http.Protocols+ h2cProtocols.SetUnencryptedHTTP2(true)+ h2cClient := &http.Client{+ Timeout: 10 * time.Second,+ Transport: &http.Transport{+ ForceAttemptHTTP2: true,+ Protocols: &h2cProtocols,+ },+ }+ extProcURL := "http://" + pepAPI + "/envoy.service.ext_proc.v3.ExternalProcessor/Process"+ mcpBackendCalls.Store(0)++ // Envoy-compatible Gateway proxy that runs the ExternalProcessor/Process h2c stream+ // against pepAPI (including ModeOverride BUFFERED body inspection and HeaderMutation)+ // before forwarding to the target backend or mesh dataplane.+ gatewayProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var bodyBytes []byte+ if r.Body != nil {+ bodyBytes, _ = io.ReadAll(r.Body)+ _ = r.Body.Close()+ }++ pr, pw := io.Pipe()+ procReq, _ := http.NewRequestWithContext(r.Context(), http.MethodPost, extProcURL, pr)+ procReq.Header.Set("Content-Type", "application/grpc+proto")+ procReq.Header.Set("TE", "trailers")++ respCh := make(chan *http.Response, 1)+ errCh := make(chan error, 1)+ go func() {+ resp, err := h2cClient.Do(procReq)+ if err != nil {+ errCh <- err+ return+ }+ respCh <- resp+ }()++ hdrs := []*corev3.HeaderValue{+ {Key: ":method", Value: r.Method},+ {Key: ":authority", Value: r.Host},+ {Key: ":path", Value: r.URL.RequestURI()},+ }+ for k, vals := range r.Header {+ hdrs = append(hdrs, &corev3.HeaderValue{Key: strings.ToLower(k), Value: strings.Join(vals, ", ")})+ }+ _ = writeExtProcFrame(pw, &extprocv3.ProcessingRequest{+ Request: &extprocv3.ProcessingRequest_RequestHeaders{+ RequestHeaders: &extprocv3.HttpHeaders{+ EndOfStream: len(bodyBytes) == 0,+ Headers: &corev3.HeaderMap{Headers: hdrs},+ },+ },+ })+ if len(bodyBytes) == 0 {+ _ = pw.Close()+ }++ var procHTTPResp *http.Response+ select {+ case procHTTPResp = <-respCh:+ case err := <-errCh:+ http.Error(w, err.Error(), http.StatusBadGateway)+ return+ }+ defer func() { _ = procHTTPResp.Body.Close() }()++ var procResp extprocv3.ProcessingResponse+ if err := readExtProcFrame(procHTTPResp.Body, &procResp); err != nil {+ http.Error(w, err.Error(), http.StatusBadGateway)+ return+ }+ if imm := procResp.GetImmediateResponse(); imm != nil {+ w.WriteHeader(int(imm.GetStatus().GetCode()))+ _, _ = w.Write(imm.GetBody())+ return+ }++ applyMuts := func(mut *extprocv3.HeaderMutation) {+ if mut == nil {+ return+ }+ for _, rem := range mut.GetRemoveHeaders() {+ r.Header.Del(rem)+ }+ for _, sh := range mut.GetSetHeaders() {+ k := sh.GetHeader().GetKey()+ v := sh.GetHeader().GetValue()+ if v == "" {+ v = string(sh.GetHeader().GetRawValue())+ }+ r.Header.Set(k, v)+ }+ }+ applyMuts(procResp.GetRequestHeaders().GetResponse().GetHeaderMutation())++ if len(bodyBytes) > 0 && procResp.GetModeOverride().GetRequestBodyMode() == extprocv3http.ProcessingMode_BUFFERED {+ _ = writeExtProcFrame(pw, &extprocv3.ProcessingRequest{+ Request: &extprocv3.ProcessingRequest_RequestBody{+ RequestBody: &extprocv3.HttpBody{+ Body: bodyBytes,+ EndOfStream: true,+ },+ },+ })+ _ = pw.Close()++ var bodyProcResp extprocv3.ProcessingResponse+ if err := readExtProcFrame(procHTTPResp.Body, &bodyProcResp); err != nil {+ http.Error(w, err.Error(), http.StatusBadGateway)+ return+ }+ if imm := bodyProcResp.GetImmediateResponse(); imm != nil {+ w.WriteHeader(int(imm.GetStatus().GetCode()))+ _, _ = w.Write(imm.GetBody())+ return+ }+ applyMuts(bodyProcResp.GetRequestBody().GetResponse().GetHeaderMutation())+ } else {+ _ = pw.Close()+ }++ // Route /sam/... requests through caller node's real libp2p mesh proxy,+ // and direct egress Host-routed requests to upstream.+ targetBase := upstream.URL+ if strings.HasPrefix(r.URL.Path, "/sam/") {+ targetBase = "http://" + callerAPI+ }+ fwdReq, _ := http.NewRequestWithContext(r.Context(), r.Method, targetBase+r.URL.RequestURI(), strings.NewReader(string(bodyBytes)))+ fwdReq.Header = r.Header.Clone()+ fwdResp, err := client.Do(fwdReq)+ if err != nil {+ http.Error(w, err.Error(), http.StatusBadGateway)+ return+ }+ defer func() { _ = fwdResp.Body.Close() }()+ w.WriteHeader(fwdResp.StatusCode)+ _, _ = io.Copy(w, fwdResp.Body)+ }))+ defer gatewayProxy.Close()++ // 1. Egress credential brokering datapath over Gateway ext_proc:+ // Contractor sends contractorJWT with Host: api.github.com -> gateway ext_proc exchanges JWT,+ // strips contractorJWT, injects Authorization: Bearer ghp_upstream_secret & X-Sam-Roles: contractor,+ // and forwards to upstream.+ contractorJWT := mintToken(map[string]any{"sub": "contractor-user", "email": "[email protected]"})+ egressReq, _ := http.NewRequest(http.MethodGet, gatewayProxy.URL+"/repos/acme/public/readme", nil)+ egressReq.Host = "api.github.com"+ egressReq.Header.Set("Authorization", "Bearer "+contractorJWT)+ egressResp, err := client.Do(egressReq)+ if err != nil {+ t.Fatalf("gateway ext_proc egress request: %v", err)+ }+ egressBody, _ := io.ReadAll(egressResp.Body)+ _ = egressResp.Body.Close()+ if egressResp.StatusCode != http.StatusOK {+ t.Fatalf("gateway ext_proc egress status = %d, want 200: %s", egressResp.StatusCode, string(egressBody))+ }+ if !strings.Contains(string(egressBody), `"auth":"Bearer ghp_upstream_secret"`) || !strings.Contains(string(egressBody), `"roles":"contractor"`) {+ t.Fatalf("expected upstream to receive injected secret and contractor role, got %s", string(egressBody))+ }++ // 2. MCP tools/call datapath over Gateway ext_proc AND across the libp2p mesh:+ // Mint a Task Biscuit on caller node scoped to tool:get_pr on mcp://github.+ mcpForm := url.Values{}+ mcpForm.Set("grant_type", api.GrantTypeTokenExchange)+ mcpForm.Add("resource", "mcp://github")+ mcpForm.Set("scope", "tool:get_pr")+ reqMCPTok, _ := http.NewRequest(http.MethodPost, "http://"+callerAPI+"/oauth/token", strings.NewReader(mcpForm.Encode()))+ reqMCPTok.Header.Set("Content-Type", "application/x-www-form-urlencoded")+ reqMCPTok.Header.Set("Authorization", "Bearer "+callerToken)+ respMCPTok, err := client.Do(reqMCPTok)+ if err != nil {+ t.Fatalf("POST /oauth/token (mcp task biscuit): %v", err)+ }+ var mcpTokData map[string]any+ _ = json.NewDecoder(respMCPTok.Body).Decode(&mcpTokData)+ _ = respMCPTok.Body.Close()+ mcpTaskBiscuit, _ := mcpTokData["access_token"].(string)+ if mcpTaskBiscuit == "" {+ t.Fatalf("expected mcpTaskBiscuit, got %v", mcpTokData)+ }++ mcpMeshPath := fmt.Sprintf("/sam/%s/mcp/github", pepPeer)++ // 2a. Allowed MCP tool "get_pr" traverses:+ // gatewayProxy -> ext_proc (RequestHeaders -> ModeOverride BUFFERED -> RequestBody) ->+ // caller node (/sam/<pepPeer>/mcp/github) -> libp2p (/libp2p-http) -> pep node -> mcpBackend!+ mcpAllowReq, _ := http.NewRequest(http.MethodPost, gatewayProxy.URL+mcpMeshPath,+ strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"get_pr"}}`))+ mcpAllowReq.Header.Set("Authorization", "Bearer "+mcpTaskBiscuit)+ mcpAllowReq.Header.Set("Content-Type", "application/json")+ mcpAllowResp, err := client.Do(mcpAllowReq)+ if err != nil {+ t.Fatalf("gateway ext_proc MCP allow: %v", err)+ }+ mcpAllowBody, _ := io.ReadAll(mcpAllowResp.Body)+ _ = mcpAllowResp.Body.Close()+ wantText := fmt.Sprintf("PR #42 from dev-user peer=%s biscuit=", callerPeer)+ if mcpAllowResp.StatusCode != http.StatusOK || !strings.Contains(string(mcpAllowBody), wantText) {+ t.Fatalf("expected 200 OK with %q, got %d: %s", wantText, mcpAllowResp.StatusCode, string(mcpAllowBody))+ }+ if mcpBackendCalls.Load() != 1 {+ t.Fatalf("expected 1 mcpBackend call, got %d", mcpBackendCalls.Load())+ }++ // 2b. Unauthorized MCP tool "delete_repo" is blocked at the gateway by ext_proc ImmediateResponse(403)+ // before ever entering the mesh or reaching mcpBackend.+ mcpDenyReq, _ := http.NewRequest(http.MethodPost, gatewayProxy.URL+mcpMeshPath,+ strings.NewReader(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"delete_repo"}}`))+ mcpDenyReq.Header.Set("Authorization", "Bearer "+mcpTaskBiscuit)+ mcpDenyReq.Header.Set("Content-Type", "application/json")+ mcpDenyResp, err := client.Do(mcpDenyReq)+ if err != nil {+ t.Fatalf("gateway ext_proc MCP deny: %v", err)+ }+ _ = mcpDenyResp.Body.Close()+ if mcpDenyResp.StatusCode != http.StatusForbidden {+ t.Fatalf("expected 403 Forbidden for unauthorized MCP tool delete_repo at gateway, got %d", mcpDenyResp.StatusCode)+ }+ if mcpBackendCalls.Load() != 1 {+ t.Fatalf("expected mcpBackendCalls to remain 1 after gateway blocked tool call, got %d", mcpBackendCalls.Load())+ }++ // 2c. Even if a caller bypasses the gateway proxy and sends "delete_repo" directly across+ // the libp2p mesh to pep node, pep node's /libp2p-http ingress inspects the JSON-RPC body+ // and enforces the Task Biscuit's tool:get_pr constraint with 403 Forbidden!+ directMeshDenyReq, _ := http.NewRequest(http.MethodPost, "http://"+callerAPI+mcpMeshPath,+ strings.NewReader(`{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"delete_repo"}}`))+ directMeshDenyReq.Header.Set(api.HeaderSamAuthentication, "Bearer "+mcpTaskBiscuit)+ directMeshDenyReq.Header.Set("Content-Type", "application/json")+ directMeshDenyResp, err := client.Do(directMeshDenyReq)+ if err != nil {+ t.Fatalf("direct mesh MCP deny: %v", err)+ }+ _ = directMeshDenyResp.Body.Close()+ if directMeshDenyResp.StatusCode != http.StatusForbidden {+ t.Fatalf("expected 403 Forbidden from provider node on direct mesh call for unauthorized tool, got %d", directMeshDenyResp.StatusCode)+ }+ if mcpBackendCalls.Load() != 1 {+ t.Fatalf("expected mcpBackendCalls to remain 1 after provider node blocked tool call, got %d", mcpBackendCalls.Load())}})}