Skip to content

Agent Mesh

test(node): add Envoy ext_authz and ext_proc integration and E2E datapath CUJs

Antonio Ojea379f8c2a6e76
diff --git a/internal/node/egress_inspect_test.go b/internal/node/egress_inspect_test.go
index 69ecdec6..221ff4e7 100644
--- a/internal/node/egress_inspect_test.go
+++ b/internal/node/egress_inspect_test.go
@@ -512,7 +512,11 @@ func TestGatewayExtProcServer(t *testing.T) {
setHdrs := resp1Body.GetRequestBody().GetResponse().GetHeaderMutation().GetSetHeaders()
foundTaskID := false
for _, h := range setHdrs {
- if strings.EqualFold(h.GetHeader().GetKey(), "X-Sam-Task-Id") && h.GetHeader().GetValue() == "task-extproc-gateway" {
+ val := h.GetHeader().GetValue()
+ if val == "" {
+ val = string(h.GetHeader().GetRawValue())
+ }
+ if strings.EqualFold(h.GetHeader().GetKey(), "X-Sam-Task-Id") && val == "task-extproc-gateway" {
foundTaskID = true
}
}
@@ -585,7 +589,11 @@ func TestGatewayExtProcServer(t *testing.T) {
}
foundAuth := false
for _, h := range resp3Hdr.GetRequestHeaders().GetResponse().GetHeaderMutation().GetSetHeaders() {
- if strings.EqualFold(h.GetHeader().GetKey(), "Authorization") && h.GetHeader().GetValue() == "Bearer gateway-injected-github-token" {
+ val := h.GetHeader().GetValue()
+ if val == "" {
+ val = string(h.GetHeader().GetRawValue())
+ }
+ if strings.EqualFold(h.GetHeader().GetKey(), "Authorization") && val == "Bearer gateway-injected-github-token" {
foundAuth = true
}
}
diff --git a/internal/node/ext_authz.go b/internal/node/ext_authz.go
index 7f1670c1..e51f084f 100644
--- a/internal/node/ext_authz.go
+++ b/internal/node/ext_authz.go
@@ -37,10 +37,11 @@ const (
)
type extAuthzCheckInput struct {
- Method string
- Path string
- Host string
- Headers map[string]string
+ Method string
+ Path string
+ Host string
+ Headers map[string]string
+ AllowMCPStreamInit bool
}
type extAuthzCheckResult struct {
@@ -74,10 +75,11 @@ func handleExtAuthzHTTP(node *SamNode, w http.ResponseWriter, r *http.Request) {
}
res := evaluateExtAuthz(r.Context(), node, extAuthzCheckInput{
- Method: method,
- Path: checkPath,
- Host: r.Host,
- Headers: headers,
+ Method: method,
+ Path: checkPath,
+ Host: r.Host,
+ Headers: headers,
+ AllowMCPStreamInit: headers[strings.ToLower(HeaderSamMCPTool)] == "",
})
if !res.Allowed {
http.Error(w, res.Message, res.HTTPStatus)
@@ -108,6 +110,7 @@ func handleExtAuthzGRPC(node *SamNode, w http.ResponseWriter, r *http.Request) {
writeGRPCError(w, 3, fmt.Sprintf("invalid CheckRequest: %v", err))
return
}
+ in.AllowMCPStreamInit = in.Headers[strings.ToLower(HeaderSamMCPTool)] == ""
res := evaluateExtAuthz(r.Context(), node, in)
respPayload := marshalEnvoyCheckResponse(res)
@@ -146,7 +149,7 @@ func evaluateExtAuthz(ctx context.Context, node *SamNode, in extAuthzCheckInput)
}
}
- target, reqPath := resolveExtAuthzTarget(in)
+ target, reqPath := resolveExtAuthzTarget(node, in)
if target != "" {
var callerPeer peer.ID
var isLocal bool
@@ -183,12 +186,13 @@ func evaluateExtAuthz(ctx context.Context, node *SamNode, in extAuthzCheckInput)
reqPath = "/"
}
reqCtx := RequestContext{
- PeerID: callerPeer,
- Protocol: "ext_authz",
- Target: target,
- MCPTool: in.Headers[strings.ToLower(HeaderSamMCPTool)],
- HTTP: &HTTPRequestFacts{Method: method, Path: reqPath},
- Local: isLocal,
+ PeerID: callerPeer,
+ Protocol: "ext_authz",
+ Target: target,
+ MCPTool: in.Headers[strings.ToLower(HeaderSamMCPTool)],
+ AllowMCPStreamInit: in.AllowMCPStreamInit,
+ HTTP: &HTTPRequestFacts{Method: method, Path: reqPath},
+ Local: isLocal,
}
if after, ok := strings.CutPrefix(target, api.EgressServicePrefix); ok {
reqCtx.Egress = &EgressFacts{Host: after, Port: 443}
@@ -295,7 +299,7 @@ func isExtAuthzServiceScheme(scheme string) bool {
}
}
-func resolveExtAuthzTarget(in extAuthzCheckInput) (target, reqPath string) {
+func resolveExtAuthzTarget(node *SamNode, in extAuthzCheckInput) (target, reqPath string) {
path := in.Path
if idx := strings.IndexByte(path, '?'); idx >= 0 {
path = path[:idx]
@@ -328,6 +332,17 @@ func resolveExtAuthzTarget(in extAuthzCheckInput) (target, reqPath string) {
return route.serviceType + "://" + route.serviceName, up
}
}
+ if node != nil && node.services != nil && strings.TrimSpace(in.Host) != "" {
+ normHost := api.NormalizeMeshHost(in.Host)
+ if normHost != "" {
+ if _, ok := node.services.GetTyped(api.ServiceType_SERVICE_TYPE_EGRESS, normHost); ok {
+ if path == "" {
+ path = "/"
+ }
+ return api.EgressServicePrefix + normHost, path
+ }
+ }
+ }
return "", path
}
@@ -566,6 +581,9 @@ func marshalEnvoyCheckResponse(res extAuthzCheckResult) []byte {
var hvoBytes []byte
hvoBytes = protowire.AppendTag(hvoBytes, 1, protowire.BytesType)
hvoBytes = protowire.AppendBytes(hvoBytes, hvBytes)
+ // Field 3: HeaderAppendAction append_action = OVERWRITE_IF_EXISTS_OR_ADD (2)
+ hvoBytes = protowire.AppendTag(hvoBytes, 3, protowire.VarintType)
+ hvoBytes = protowire.AppendVarint(hvoBytes, 2)
okBytes = protowire.AppendTag(okBytes, 2, protowire.BytesType)
okBytes = protowire.AppendBytes(okBytes, hvoBytes)
diff --git a/internal/node/extproc_grpc.go b/internal/node/extproc_grpc.go
index f0cdcf71..06e3d0bf 100644
--- a/internal/node/extproc_grpc.go
+++ b/internal/node/extproc_grpc.go
@@ -15,6 +15,7 @@
package node
import (
+ "bytes"
"context"
"crypto/tls"
"crypto/x509"
@@ -38,7 +39,6 @@ import (
extprocv3 "github.com/google/sam/third_party/envoy/envoy/service/ext_proc/v3"
typev3 "github.com/google/sam/third_party/envoy/envoy/type/v3"
"google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/types/known/wrapperspb"
)
// ExtProcMethodPath is the gRPC HTTP/2 path for Envoy ExternalProcessor.Process.
@@ -344,18 +344,21 @@ func handleGatewayExtProc(node *SamNode, w http.ResponseWriter, r *http.Request)
},
}
} else {
- resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, false)
+ resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, false, false)
}
case *extprocv3.ProcessingRequest_RequestBody:
+ var allowStreamInit bool
if len(phase.RequestBody.GetBody()) > 0 && capturedHeaders != nil {
- if tool := extractJSONRPCMCPTool(phase.RequestBody.GetBody()); tool != "" {
+ tool, allowInit := inspectJSONRPCMCPBody(phase.RequestBody.GetBody())
+ if tool != "" {
capturedHeaders[strings.ToLower(HeaderSamMCPTool)] = tool
}
+ allowStreamInit = allowInit
}
if pendingCheck {
pendingCheck = false
- resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, true)
+ resp = evaluateGatewayExtProcDecision(r.Context(), node, capturedMethod, capturedPath, capturedHost, capturedHeaders, true, allowStreamInit)
} else {
resp = &extprocv3.ProcessingResponse{
Response: &extprocv3.ProcessingResponse_RequestBody{
@@ -420,12 +423,13 @@ func handleGatewayExtProc(node *SamNode, w http.ResponseWriter, r *http.Request)
w.Header().Set("Grpc-Message", "")
}
-func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method, path, host string, headers map[string]string, isBodyPhase bool) *extprocv3.ProcessingResponse {
+func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method, path, host string, headers map[string]string, isBodyPhase, allowMCPStreamInit bool) *extprocv3.ProcessingResponse {
res := evaluateExtAuthz(ctx, node, extAuthzCheckInput{
- Method: method,
- Path: path,
- Host: host,
- Headers: headers,
+ Method: method,
+ Path: path,
+ Host: host,
+ Headers: headers,
+ AllowMCPStreamInit: allowMCPStreamInit,
})
if !res.Allowed {
status := typev3.StatusCode_Forbidden
@@ -447,17 +451,23 @@ func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method,
for k, v := range res.ResponseHeaders {
setHeaders = append(setHeaders, &corev3.HeaderValueOption{
Header: &corev3.HeaderValue{
- Key: k,
- Value: v,
+ Key: k,
+ RawValue: []byte(v),
},
- Append: wrapperspb.Bool(false),
AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD,
})
}
+ var removeHeaders []string
+ if _, hasUpstreamAuth := res.ResponseHeaders["Authorization"]; !hasUpstreamAuth {
+ if headers["authorization"] != "" {
+ removeHeaders = append(removeHeaders, "authorization")
+ }
+ }
common := &extprocv3.CommonResponse{
Status: extprocv3.CommonResponse_CONTINUE,
HeaderMutation: &extprocv3.HeaderMutation{
- SetHeaders: setHeaders,
+ SetHeaders: setHeaders,
+ RemoveHeaders: removeHeaders,
},
}
if isBodyPhase {
@@ -474,7 +484,7 @@ func evaluateGatewayExtProcDecision(ctx context.Context, node *SamNode, method,
}
}
-func extractJSONRPCMCPTool(body []byte) string {
+func inspectJSONRPCMCPBody(body []byte) (mcpTool string, allowStreamInit bool) {
var rpc struct {
Method string `json:"method"`
Params struct {
@@ -482,14 +492,37 @@ func extractJSONRPCMCPTool(body []byte) string {
} `json:"params"`
}
if err := json.Unmarshal(body, &rpc); err != nil {
- return ""
+ return "", false
}
- if rpc.Method == "tools/call" && strings.TrimSpace(rpc.Params.Name) != "" {
+ switch rpc.Method {
+ case "initialize", "ping", "tools/list":
+ return "", true
+ case "tools/call":
rawTool := strings.TrimSpace(rpc.Params.Name)
+ if rawTool == "" {
+ return "", false
+ }
if _, stripped, err := api.SplitToolName(rawTool); err == nil {
- return stripped
+ return stripped, false
}
- return rawTool
+ return rawTool, false
+ default:
+ return "", false
+ }
+}
+
+func inspectMCPHTTPRequestBody(r *http.Request) (string, bool, error) {
+ if r == nil || r.Body == nil || !strings.EqualFold(r.Method, http.MethodPost) {
+ return "", false, nil
+ }
+ bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, maxGRPCFrameBytes+1))
+ if err != nil {
+ return "", false, err
+ }
+ if int64(len(bodyBytes)) > maxGRPCFrameBytes {
+ return "", false, fmt.Errorf("MCP request body exceeds maximum inspection size (%d bytes)", maxGRPCFrameBytes)
}
- return ""
+ r.Body = io.NopCloser(bytes.NewReader(bodyBytes))
+ tool, allowInit := inspectJSONRPCMCPBody(bodyBytes)
+ return tool, allowInit, nil
}
diff --git a/internal/node/middleware.go b/internal/node/middleware.go
index 1bb91d26..7a4a7524 100644
--- a/internal/node/middleware.go
+++ b/internal/node/middleware.go
@@ -58,6 +58,10 @@ type RequestContext struct {
// MCPTool is set when evaluating a specific MCP tools/call invocation.
MCPTool string
+ // AllowMCPStreamInit is set when an MCP request is a session handshake or
+ // discovery method (initialize, ping, tools/list).
+ AllowMCPStreamInit bool
+
// TaskRules holds the verified TaskAuthorizationRule chain extracted from
// the caller's Biscuit during stream authentication so downstream stream
// handlers (such as MCP pass-through) can enforce tool-level constraints.
@@ -378,7 +382,7 @@ func (n *SamNode) authorizeWithRules(rawToken []byte, req RequestContext, pubKey
ServiceType: opType,
ServiceName: opName,
MCPTool: req.MCPTool,
- AllowMCPStreamInit: req.HTTP == nil && req.MCPTool == "" && req.Protocol == string(api.MCPProtocolID),
+ AllowMCPStreamInit: req.AllowMCPStreamInit || (req.HTTP == nil && req.MCPTool == "" && req.Protocol == string(api.MCPProtocolID)),
}
if req.HTTP != nil {
taskReq.HasHTTP = true
diff --git a/internal/node/node.go b/internal/node/node.go
index d87e11cd..72e69c46 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -2286,6 +2286,15 @@ func (n *SamNode) StartIngressServer(ctx context.Context) error {
reqCtx.Egress = &EgressFacts{Host: serviceName, Port: reqPort}
}
}
+ if serviceType == api.ServiceType_SERVICE_TYPE_MCP {
+ mcpTool, allowInit, err := inspectMCPHTTPRequestBody(r)
+ if err != nil {
+ refuse(w, http.StatusRequestEntityTooLarge, err.Error(), proxyStatusDenied)
+ return
+ }
+ reqCtx.MCPTool = mcpTool
+ reqCtx.AllowMCPStreamInit = allowInit
+ }
// Verify authorization
if err := n.VerifyBiscuitToken(biscuitBytes, reqCtx); err != nil {
diff --git a/internal/node/sts.go b/internal/node/sts.go
index 6873b44f..b0418c19 100644
--- a/internal/node/sts.go
+++ b/internal/node/sts.go
@@ -350,14 +350,10 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallba
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
logger.Debugf("[SidecarAuth] Incoming request: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr)
- // Extract optional X-Sam-Authentication or Authorization bearer value.
+ // Extract optional X-Sam-Authentication, X-Sam-Biscuit (injected by an
+ // upstream Envoy ext_authz / ext_proc filter), or Authorization bearer value.
headerName := api.HeaderSamAuthentication
authHeader := r.Header.Get(headerName)
- if authHeader == "" && allowAuthorizationFallback {
- headerName = "Authorization"
- authHeader = r.Header.Get(headerName)
- }
-
var bearer string
var hasBearer bool
if authHeader != "" {
@@ -366,13 +362,28 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallba
bearer = strings.TrimSpace(parts[1])
hasBearer = true
}
+ } else if b64 := strings.TrimSpace(r.Header.Get(api.HeaderSamBiscuit)); b64 != "" {
+ headerName = api.HeaderSamBiscuit
+ authHeader = b64
+ bearer = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(b64, "Bearer "), "bearer "))
+ hasBearer = bearer != ""
+ } else if allowAuthorizationFallback {
+ headerName = "Authorization"
+ authHeader = r.Header.Get(headerName)
+ if authHeader != "" {
+ parts := strings.SplitN(authHeader, " ", 2)
+ if len(parts) == 2 && strings.EqualFold(parts[0], "bearer") {
+ bearer = strings.TrimSpace(parts[1])
+ hasBearer = true
+ }
+ }
}
// Local Unix socket or mTLS (token == ""): transport already authenticates
// the local process, but if the caller also supplied a Task Biscuit or JWT,
// bind it to the request context (and fail closed if that Biscuit/JWT is invalid/revoked).
if fromLocalSocket(r) || token == "" {
- if hasBearer && (token == "" || !constantTimeEqual(bearer, token)) {
+ if hasBearer && (token == "" || headerName == api.HeaderSamBiscuit || !constantTimeEqual(bearer, token)) {
biscuitBytes, isMeshCred, err := node.resolveCallerCredential(r.Context(), bearer)
if isMeshCred {
if err != nil {
@@ -381,12 +392,13 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallba
}
r = r.WithContext(WithCallerBiscuit(r.Context(), biscuitBytes))
r.Header.Del(headerName)
- } else if headerName == api.HeaderSamAuthentication {
+ } else if headerName == api.HeaderSamAuthentication || headerName == api.HeaderSamBiscuit {
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
}
r.Header.Del(api.HeaderSamAuthentication)
+ r.Header.Del(api.HeaderSamBiscuit)
stripSidecarTokenFromAuthorization(r, token)
next.ServeHTTP(w, r)
return
@@ -406,7 +418,7 @@ func withCallerOrTokenAuth(node *SamNode, token string, allowAuthorizationFallba
return
}
- if constantTimeEqual(bearer, token) {
+ if headerName != api.HeaderSamBiscuit && constantTimeEqual(bearer, token) {
r.Header.Del(headerName)
stripSidecarTokenFromAuthorization(r, token)
next.ServeHTTP(w, r)
diff --git a/internal/node/sts_test.go b/internal/node/sts_test.go
index c88896ab..4312d564 100644
--- a/internal/node/sts_test.go
+++ b/internal/node/sts_test.go
@@ -22,6 +22,7 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
+ "io"
"net/http"
"net/http/httptest"
"net/url"
@@ -525,3 +526,25 @@ func buildEnvoyCheckRequestPayload(method, path string, headers map[string]strin
checkReq = protowire.AppendBytes(checkReq, attrBytes)
return checkReq
}
+
+func TestInspectMCPHTTPRequestBody(t *testing.T) {
+ body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mcp://github/get_pr"}}`
+ req := httptest.NewRequest(http.MethodPost, "/mcp/github", strings.NewReader(body))
+ tool, allowInit, err := inspectMCPHTTPRequestBody(req)
+ if err != nil {
+ t.Fatalf("inspectMCPHTTPRequestBody: %v", err)
+ }
+ if tool != "get_pr" || allowInit {
+ t.Fatalf("got tool=%q allowInit=%v, want tool=\"get_pr\" allowInit=false", tool, allowInit)
+ }
+ restored, _ := io.ReadAll(req.Body)
+ if string(restored) != body {
+ t.Fatalf("body was not restored: got %q, want %q", string(restored), body)
+ }
+
+ initReq := httptest.NewRequest(http.MethodPost, "/mcp/github", strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"initialize"}`))
+ tool, allowInit, err = inspectMCPHTTPRequestBody(initReq)
+ if err != nil || tool != "" || !allowInit {
+ t.Fatalf("initialize: got tool=%q allowInit=%v err=%v, want allowInit=true", tool, allowInit, err)
+ }
+}
diff --git a/tests/e2e/fixtures/fake_openai.py b/tests/e2e/fixtures/fake_openai.py
index dd3ffa54..ab48c57c 100644
--- a/tests/e2e/fixtures/fake_openai.py
+++ b/tests/e2e/fixtures/fake_openai.py
@@ -2,7 +2,7 @@
# A minimal OpenAI-compatible backend for e2e tests: one model, one canned
# completion. Standard library only, so it runs wherever python3 does.
import json
-from http.server import BaseHTTPRequestHandler, HTTPServer
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
MODEL = "e2e-model"
@@ -36,6 +36,51 @@ class Handler(BaseHTTPRequestHandler):
"usage": {"prompt_tokens": 1, "completion_tokens": 4, "total_tokens": 5},
})
return
+ if self.path == "/mcp" or self.path.startswith("/mcp/"):
+ method = req.get("method", "")
+ req_id = req.get("id", 1)
+ if method == "initialize":
+ self._json(200, {
+ "jsonrpc": "2.0",
+ "id": req_id,
+ "result": {
+ "protocolVersion": "2024-11-05",
+ "capabilities": {"tools": {}},
+ "serverInfo": {"name": "github", "version": "1.0"},
+ },
+ })
+ return
+ if method == "notifications/initialized":
+ self.send_response(202)
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ return
+ if method == "tools/list":
+ self._json(200, {
+ "jsonrpc": "2.0",
+ "id": req_id,
+ "result": {
+ "tools": [
+ {"name": "get_pr", "description": "Get pull request"},
+ {"name": "delete_repo", "description": "Delete repository"},
+ ]
+ },
+ })
+ return
+ if method == "tools/call":
+ peer_id = self.headers.get("X-Peer-Id", "")
+ tool_name = (req.get("params") or {}).get("name", "")
+ self._json(200, {
+ "jsonrpc": "2.0",
+ "id": req_id,
+ "result": {
+ "content": [{
+ "type": "text",
+ "text": f"tool={tool_name} peer={peer_id}",
+ }]
+ },
+ })
+ return
self._json(404, {"error": {"message": "unknown model", "code": "model_not_found"}})
def log_message(self, *args):
@@ -43,6 +88,7 @@ class Handler(BaseHTTPRequestHandler):
if __name__ == "__main__":
- server = HTTPServer(("127.0.0.1", 0), Handler)
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
print(f"listening on port {server.server_address[1]}", flush=True)
server.serve_forever()
+
diff --git a/tests/e2e/standalone.bats b/tests/e2e/standalone.bats
index eca74cfd..f25bbcc3 100644
--- a/tests/e2e/standalone.bats
+++ b/tests/e2e/standalone.bats
@@ -22,6 +22,9 @@ setup() {
}
teardown() {
+ if [[ -n "${ENVOY_CONTAINER:-}" ]]; then
+ docker rm -f "$ENVOY_CONTAINER" >/dev/null 2>&1 || true
+ fi
for pid in "${BACKEND_PID:-}" "${NODE_A_PID:-}" "${NODE_B_PID:-}" "${SAM_ONE_PID:-}"; do
[[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true
done
@@ -92,13 +95,13 @@ start_node() {
--data-dir "$TEST_TMPDIR/node-$name" \
--bind-addr= \
--allow-loopback \
- --listen "/ip4/127.0.0.1/tcp/0" "$@" > "$TEST_TMPDIR/node-$name.log" 2>&1 &
+ --listen "/ip4/127.0.0.1/tcp/0" "$@" > "$TEST_TMPDIR/node-$name.log" 2>&1 3>&- &
eval "NODE_${name^^}_PID=$!"
}
@test "sam-one boots, nodes join over the single port, and the dataplane carries a service call" {
"$SAM_ONE_BINARY" --bind-address 127.0.0.1 --port 0 \
- --data-dir "$SAM_ONE_DATA" > "$TEST_TMPDIR/sam-one.log" 2>&1 &
+ --data-dir "$SAM_ONE_DATA" > "$TEST_TMPDIR/sam-one.log" 2>&1 3>&- &
SAM_ONE_PID=$!
# The random port is published in the banner, like the generated tokens.
@@ -118,13 +121,14 @@ start_node() {
JOIN_TOKEN="$(cat "$SAM_ONE_DATA/join-token")"
[[ "$JOIN_TOKEN" == sam_tok_* ]]
- # An OpenAI-compatible backend on node A's host, declared as an inference
- # service in node A's configuration, as in docs/getting-started/your-own-mesh.
- # Services only exist by declaration at startup, there is no runtime
- # registration endpoint. The same server is also declared as `mcp`: the
- # type is a contract the node verifies, so that one must never be advertised.
+ # An OpenAI-compatible and MCP backend on node A's host, declared in node A's
+ # configuration, as in docs/getting-started/your-own-mesh. Services only exist
+ # by declaration at startup, there is no runtime registration endpoint. The
+ # root URL (/) is also declared as `mcp` (not-an-mcp-server): the type is a
+ # contract the node verifies, so that one must never be advertised, while
+ # /mcp speaks MCP JSON-RPC and is advertised as mcp://github.
python3 -u "$BATS_TEST_DIRNAME/fixtures/fake_openai.py" \
- > "$TEST_TMPDIR/backend.log" 2>&1 &
+ > "$TEST_TMPDIR/backend.log" 2>&1 3>&- &
BACKEND_PID=$!
wait_for_log "$TEST_TMPDIR/backend.log" "listening on port"
backend_port="$(grep -oE 'port [0-9]+' "$TEST_TMPDIR/backend.log" | grep -oE '[0-9]+')"
@@ -136,6 +140,10 @@ services:
name: laptop-llm
description: "e2e inference backend"
target_url: "http://127.0.0.1:${backend_port}"
+ - type: mcp
+ name: github
+ description: "e2e mcp backend"
+ target_url: "http://127.0.0.1:${backend_port}/mcp"
- type: mcp
name: not-an-mcp-server
description: "a plain HTTP server mislabelled as mcp"
@@ -143,11 +151,19 @@ services:
EOF
start_node a --config "$TEST_TMPDIR/node-a-services.yaml"
- start_node b
+ # Start node B with a local TCP sidecar listener (127.0.0.1:0) in addition to
+ # its Unix socket so a real Envoy proxy can dial node B over h2c and forward
+ # authorized traffic through node B into the mesh.
+ start_node b --bind-addr "127.0.0.1:0"
wait_for_node a
wait_for_node b
+ wait_for_log "$TEST_TMPDIR/node-b.log" "Starting MCP server on TCP address"
+ node_b_tcp_port="$(grep -oE 'Starting MCP server on TCP address 127\.0\.0\.1:[0-9]+' "$TEST_TMPDIR/node-b.log" | grep -oE '[0-9]+$')"
+ [[ -n "$node_b_tcp_port" ]]
+
peer_a="$(node_peer_id a)"
- [[ -n "$peer_a" ]]
+ peer_b="$(node_peer_id b)"
+ [[ -n "$peer_a" && -n "$peer_b" ]]
sock_a="$TEST_TMPDIR/node-a/sam.sock"
sock_b="$TEST_TMPDIR/node-b/sam.sock"
@@ -201,6 +217,161 @@ EOF
-d '{"model":"e2e-model","messages":[{"role":"user","content":"should fail"}]}')"
[[ "$http_code" -ne 200 ]]
+ # Real Envoy proxy CUJ: when Docker is available, place Envoy in front of node B
+ # with both ext_authz and ext_proc configured over gRPC h2c, and drive real
+ # inference (/v1/chat/completions) and MCP (/sam/<peer_a>/mcp/github) calls
+ # through Envoy -> Node B -> sam-one Router -> Node A -> Backend.
+ if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+ envoy_port="$(python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
+
+ cat > "$TEST_TMPDIR/envoy.yaml" <<EOF
+static_resources:
+ listeners:
+ - name: listener_0
+ address:
+ socket_address:
+ address: 127.0.0.1
+ port_value: ${envoy_port}
+ filter_chains:
+ - filters:
+ - name: envoy.filters.network.http_connection_manager
+ typed_config:
+ "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+ stat_prefix: ingress_http
+ route_config:
+ name: local_route
+ virtual_hosts:
+ - name: local_service
+ domains: ["*"]
+ routes:
+ - match:
+ prefix: "/"
+ route:
+ cluster: sam_node_b_sidecar
+ http_filters:
+ - name: envoy.filters.http.ext_authz
+ typed_config:
+ "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz
+ transport_api_version: V3
+ grpc_service:
+ envoy_grpc:
+ cluster_name: sam_node_b_h2c
+ timeout: 5s
+ - name: envoy.filters.http.ext_proc
+ typed_config:
+ "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor
+ grpc_service:
+ envoy_grpc:
+ cluster_name: sam_node_b_h2c
+ timeout: 5s
+ allow_mode_override: true
+ message_timeout: 5s
+ processing_mode:
+ request_header_mode: SEND
+ response_header_mode: SKIP
+ request_body_mode: NONE
+ response_body_mode: NONE
+ - name: envoy.filters.http.router
+ typed_config:
+ "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+ clusters:
+ - name: sam_node_b_h2c
+ type: STATIC
+ connect_timeout: 2s
+ typed_extension_protocol_options:
+ envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+ "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+ explicit_http_config:
+ http2_protocol_options: {}
+ load_assignment:
+ cluster_name: sam_node_b_h2c
+ endpoints:
+ - lb_endpoints:
+ - endpoint:
+ address:
+ socket_address:
+ address: 127.0.0.1
+ port_value: ${node_b_tcp_port}
+ - name: sam_node_b_sidecar
+ type: STATIC
+ connect_timeout: 2s
+ load_assignment:
+ cluster_name: sam_node_b_sidecar
+ endpoints:
+ - lb_endpoints:
+ - endpoint:
+ address:
+ socket_address:
+ address: 127.0.0.1
+ port_value: ${node_b_tcp_port}
+EOF
+
+ chmod 0644 "$TEST_TMPDIR/envoy.yaml"
+ ENVOY_CONTAINER="sam-e2e-envoy-$$"
+ docker run -d --name "$ENVOY_CONTAINER" \
+ --network host \
+ -v "$TEST_TMPDIR/envoy.yaml:/etc/envoy/envoy.yaml:ro" \
+ envoyproxy/envoy:v1.32-latest \
+ -c /etc/envoy/envoy.yaml --log-level warning >/dev/null 3>&-
+
+ envoy_ready=0
+ for _ in $(seq 1 50); do
+ code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${envoy_port}/sam/${peer_a}/mcp/github" -X POST || true)"
+ if [[ "$code" == "401" ]]; then
+ envoy_ready=1
+ break
+ fi
+ sleep 0.2
+ done
+ if [[ "$envoy_ready" -ne 1 ]]; then
+ docker logs "$ENVOY_CONTAINER" >&2 || true
+ cat "$TEST_TMPDIR/node-b.log" >&2 || true
+ return 1
+ fi
+
+ # 1. OpenAI inference completion through Envoy -> Node B -> Router -> Node A -> fake_openai.py
+ run curl -sS -i "http://127.0.0.1:${envoy_port}/v1/chat/completions" \
+ -H "Authorization: Bearer $task_biscuit" \
+ -H "X-Sam-Target-Service: inference://laptop-llm" \
+ -H 'Content-Type: application/json' \
+ -d '{"model":"e2e-model","messages":[{"role":"user","content":"hi through envoy"}]}'
+ if [[ "$status" -ne 0 || "$output" != *"hello from the mesh"* ]]; then
+ echo "Envoy /v1/chat/completions failed ($status): $output" >&2
+ docker logs "$ENVOY_CONTAINER" >&2 || true
+ tail -n 50 "$TEST_TMPDIR/node-b.log" >&2 || true
+ return 1
+ fi
+
+ # 2. Mint a Task Biscuit narrowed to mcp://github and tool:get_pr.
+ mcp_task_biscuit="$(curl -sf --unix-socket "$sock_b" "http://localhost/oauth/token" \
+ -d 'grant_type=urn:ietf:params:oauth:grant-type:token-exchange' \
+ -d 'resource=mcp://github' \
+ -d 'scope=tool:get_pr' |
+ python3 -c 'import json, sys; print(json.load(sys.stdin)["access_token"])')"
+ [[ -n "$mcp_task_biscuit" ]]
+
+ # 2a. Allowed MCP tools/call ("get_pr") traverses Envoy (ext_authz + ext_proc body buffering) ->
+ # Node B (/sam/<peer_a>/mcp/github) -> Router -> Node A -> MCP backend!
+ run curl -sS -i "http://127.0.0.1:${envoy_port}/sam/${peer_a}/mcp/github" \
+ -H "Authorization: Bearer $mcp_task_biscuit" \
+ -H "Content-Type: application/json" \
+ -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"get_pr"}}'
+ if [[ "$status" -ne 0 || "$output" != *"tool=get_pr peer=${peer_b}"* ]]; then
+ echo "Envoy MCP tools/call failed ($status): $output" >&2
+ docker logs "$ENVOY_CONTAINER" >&2 || true
+ tail -n 50 "$TEST_TMPDIR/node-b.log" >&2 || true
+ tail -n 50 "$TEST_TMPDIR/node-a.log" >&2 || true
+ return 1
+ fi
+
+ # 2b. Disallowed MCP tools/call ("delete_repo") is blocked at Envoy by ext_proc body inspection with 403 Forbidden.
+ deny_code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${envoy_port}/sam/${peer_a}/mcp/github" \
+ -H "Authorization: Bearer $mcp_task_biscuit" \
+ -H "Content-Type: application/json" \
+ -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"delete_repo"}}')"
+ [[ "$deny_code" -eq 403 ]]
+ fi
+
# Revoking the task Biscuit via RFC 7009 POST /oauth/revoke immediately blocks
# reuse of that task Biscuit without revoking node B's standing identity.
run curl -sf --unix-socket "$sock_b" "http://localhost/oauth/revoke" \
@@ -214,6 +385,15 @@ EOF
-d '{"model":"e2e-model","messages":[{"role":"user","content":"revoked task"}]}')"
[[ "$revoked_code" -eq 403 ]]
+ if [[ -n "${envoy_port:-}" ]]; then
+ envoy_revoked_code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${envoy_port}/v1/chat/completions" \
+ -H "Authorization: Bearer $task_biscuit" \
+ -H "X-Sam-Target-Service: inference://laptop-llm" \
+ -H 'Content-Type: application/json' \
+ -d '{"model":"e2e-model","messages":[{"role":"user","content":"revoked task via envoy"}]}')"
+ [[ "$envoy_revoked_code" -eq 403 ]]
+ fi
+
# The plain HTTP server never shows up as an MCP provider: node A had
# registered every configured service before its API came up, so its
# absence here is a verdict, not a race.
@@ -235,3 +415,4 @@ EOF
join_row="$(echo "$output" | grep "sam-one join token")"
[[ "$join_row" == *" 2/"* ]]
}
+
diff --git a/tests/integration/sts_cuj_test.go b/tests/integration/sts_cuj_test.go
index 8a7d153d..b8b9611b 100644
--- a/tests/integration/sts_cuj_test.go
+++ b/tests/integration/sts_cuj_test.go
@@ -17,42 +17,175 @@ package integration_test
import (
"crypto/sha256"
"encoding/base64"
+ "encoding/binary"
"encoding/json"
"fmt"
"io"
+ "net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
+ "sync/atomic"
"testing"
"time"
"github.com/google/sam/api"
+ corev3 "github.com/google/sam/third_party/envoy/envoy/config/core/v3"
+ extprocv3http "github.com/google/sam/third_party/envoy/envoy/extensions/filters/http/ext_proc/v3"
+ extprocv3 "github.com/google/sam/third_party/envoy/envoy/service/ext_proc/v3"
+ typev3 "github.com/google/sam/third_party/envoy/envoy/type/v3"
+ "google.golang.org/protobuf/proto"
)
+func writeExtProcFrame(w io.Writer, msg proto.Message) error {
+ payload, err := proto.Marshal(msg)
+ if err != nil {
+ return err
+ }
+ var hdr [5]byte
+ hdr[0] = 0
+ binary.BigEndian.PutUint32(hdr[1:5], uint32(len(payload)))
+ if _, err := w.Write(hdr[:]); err != nil {
+ return err
+ }
+ _, err = w.Write(payload)
+ return err
+}
+
+func readExtProcFrame(r io.Reader, msg proto.Message) error {
+ var hdr [5]byte
+ if _, err := io.ReadFull(r, hdr[:]); err != nil {
+ return err
+ }
+ length := binary.BigEndian.Uint32(hdr[1:5])
+ payload := make([]byte, length)
+ if _, err := io.ReadFull(r, payload); err != nil {
+ return err
+ }
+ return proto.Unmarshal(payload, msg)
+}
+
+func startH2CExtProcCallout(t *testing.T, handler http.Handler) string {
+ t.Helper()
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen tcp: %v", err)
+ }
+ var protocols http.Protocols
+ protocols.SetHTTP1(true)
+ protocols.SetUnencryptedHTTP2(true)
+ srv := &http.Server{
+ Handler: handler,
+ Protocols: &protocols,
+ }
+ go func() { _ = srv.Serve(ln) }()
+ t.Cleanup(func() {
+ _ = srv.Close()
+ _ = ln.Close()
+ })
+ return ln.Addr().String()
+}
+
// TestSTSTaskScopedSecurityCUJ validates the Phase 3 Security Token Service (STS),
-// Task-Scoped Biscuit Attenuation, OAuth 2.1 / PKCE, Envoy ext_authz, RFC 7009
-// Revocation, and Outbound Border JWT Minting journeys end-to-end across real
-// sam-control-plane, sam-router, and sam-node processes.
+// Task-Scoped Biscuit Attenuation, OAuth 2.1 / PKCE, Envoy ext_authz and ext_proc,
+// RFC 7009 Revocation, and Outbound Border JWT Minting journeys end-to-end across
+// real sam-control-plane, sam-router, and sam-node processes.
func TestSTSTaskScopedSecurityCUJ(t *testing.T) {
nodeBin := buildBinary(t, "./cmd/sam-node")
tmpDir := t.TempDir()
oidcURL, mintToken := startCustomMockOIDC(t)
+ var upstreamAuthHeader atomic.Value
+ var upstreamInspectedHeader atomic.Value
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusNoContent)
+ upstreamAuthHeader.Store(r.Header.Get("Authorization"))
+ upstreamInspectedHeader.Store(r.Header.Get("X-Ext-Proc-Inspected"))
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ _, _ = fmt.Fprintf(w, `{"path":%q,"auth":%q,"roles":%q}`, r.URL.Path, r.Header.Get("Authorization"), r.Header.Get(api.HeaderSamRoles))
}))
defer upstream.Close()
+ extProcCalloutAddr := startH2CExtProcCallout(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/grpc+proto")
+ w.Header().Set("Trailer", "Grpc-Status, Grpc-Message")
+ w.WriteHeader(http.StatusOK)
+ rc := http.NewResponseController(w)
+ _ = rc.Flush()
+ for {
+ var req extprocv3.ProcessingRequest
+ if err := readExtProcFrame(r.Body, &req); err != nil {
+ break
+ }
+ var resp *extprocv3.ProcessingResponse
+ if rh := req.GetRequestHeaders(); rh != nil {
+ var path string
+ for _, hv := range rh.GetHeaders().GetHeaders() {
+ if hv.GetKey() == ":path" {
+ path = hv.GetValue()
+ if path == "" {
+ path = string(hv.GetRawValue())
+ }
+ }
+ }
+ if strings.HasSuffix(path, "/blocked-by-dlp") {
+ resp = &extprocv3.ProcessingResponse{
+ Response: &extprocv3.ProcessingResponse_ImmediateResponse{
+ ImmediateResponse: &extprocv3.ImmediateResponse{
+ Status: &typev3.HttpStatus{Code: typev3.StatusCode_Forbidden},
+ Body: []byte("blocked by ext_proc callout"),
+ Details: "dlp_violation",
+ },
+ },
+ }
+ } else {
+ resp = &extprocv3.ProcessingResponse{
+ Response: &extprocv3.ProcessingResponse_RequestHeaders{
+ RequestHeaders: &extprocv3.HeadersResponse{
+ Response: &extprocv3.CommonResponse{
+ Status: extprocv3.CommonResponse_CONTINUE,
+ HeaderMutation: &extprocv3.HeaderMutation{
+ SetHeaders: []*corev3.HeaderValueOption{
+ {Header: &corev3.HeaderValue{Key: "X-Ext-Proc-Inspected", Value: "true"}},
+ },
+ },
+ },
+ },
+ },
+ }
+ }
+ } else if req.GetResponseHeaders() != nil {
+ resp = &extprocv3.ProcessingResponse{
+ Response: &extprocv3.ProcessingResponse_ResponseHeaders{
+ ResponseHeaders: &extprocv3.HeadersResponse{
+ Response: &extprocv3.CommonResponse{
+ Status: extprocv3.CommonResponse_CONTINUE,
+ },
+ },
+ },
+ }
+ }
+ if resp != nil {
+ _ = writeExtProcFrame(w, resp)
+ _ = rc.Flush()
+ if resp.GetImmediateResponse() != nil {
+ break
+ }
+ }
+ }
+ w.Header().Set("Grpc-Status", "0")
+ }))
+
policyFile := filepath.Join(tmpDir, "policies.yaml")
policyYAML := fmt.Sprintf(`roles:
- name: pep
- allowed_services: ["egress://api.github.com"]
+ allowed_services: ["egress://api.github.com", "mcp://github"]
allowed_targets: ["*"]
- name: developer
- allowed_services: ["egress://api.github.com"]
+ allowed_services: ["egress://api.github.com", "mcp://github"]
allowed_targets: ["*"]
http:
- service: "egress://api.github.com"
@@ -79,7 +212,12 @@ egress:
target_url: %q
credential: github-token
served_by: ["pep"]
-`, upstream.URL)
+ inspection:
+ inspectors:
+ - ext_proc:
+ target: %q
+ allow_mode_override: true
+`, upstream.URL, extProcCalloutAddr)
if err := os.WriteFile(policyFile, []byte(policyYAML), 0o644); err != nil {
t.Fatal(err)
}
@@ -96,10 +234,41 @@ egress:
t.Fatal(err)
}
+ var mcpBackendCalls atomic.Int32
+ mcpBackend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var rpc struct {
+ ID any `json:"id"`
+ Method string `json:"method"`
+ }
+ body, _ := io.ReadAll(r.Body)
+ _ = json.Unmarshal(body, &rpc)
+ w.Header().Set("Content-Type", "application/json")
+ switch rpc.Method {
+ case "initialize":
+ _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2024-11-05","capabilities":{"tools":{}},"serverInfo":{"name":"github","version":"1.0"}}}`))
+ case "notifications/initialized":
+ w.WriteHeader(http.StatusAccepted)
+ case "tools/list":
+ _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"tools":[{"name":"get_pr","description":"Get PR"},{"name":"delete_repo","description":"Delete repo"}]}}`))
+ default:
+ mcpBackendCalls.Add(1)
+ _, _ = fmt.Fprintf(w, `{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"PR #42 from %s peer=%s biscuit=%s"}]}}`,
+ r.Header.Get(api.HeaderSamPrincipal),
+ r.Header.Get(api.HeaderPeerID),
+ r.Header.Get(api.HeaderSamBiscuit))
+ }
+ }))
+ defer mcpBackend.Close()
+
+ pepDir := filepath.Join(tmpDir, "pep")
+ if err := os.MkdirAll(pepDir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ pepCfg := writeNodeConfig(t, pepDir, nil, svcDecl{Type: "mcp", Name: "github", TargetURL: mcpBackend.URL})
pepToken := "test-token"
- pep := launchNode(t, nodeBin, os.Environ(), filepath.Join(tmpDir, "pep"), "run",
+ pep := launchNode(t, nodeBin, os.Environ(), pepDir, "run",
"--control-plane", controlPlane,
- "--data-dir", filepath.Join(tmpDir, "pep"),
+ "--data-dir", pepDir,
"--api-token-path", tokenPath(t, pepToken),
"--jwt", mintToken(map[string]any{"sub": "pep-user", "roles": []string{api.RoleNode}}),
"--listen", "/ip4/127.0.0.1/udp/0/quic-v1",
@@ -108,6 +277,7 @@ egress:
"--discovery-interval", "100ms",
"--control-plane-sync-interval", "1s",
"--secrets-dir", secrets,
+ "--config", pepCfg,
)
pepAPI := pep.waitForAPI(t)
@@ -127,6 +297,7 @@ egress:
connectPeer(t, callerAPI, pep.p2pAddr)
waitForDHTPeers(t, pepAPI)
pepPeer := extractPeerID(pep.p2pAddr)
+ callerPeer := extractPeerID(caller.p2pAddr)
waitForDiscoverableService(t, callerAPI, callerToken, "egress", "api.github.com")
client := &http.Client{
@@ -206,38 +377,93 @@ egress:
}
})
- t.Run("Inbound external JWT exchange and role isolation via ext_authz", func(t *testing.T) {
+ t.Run("Inbound external JWT exchange and role isolation via ext_authz datapath", func(t *testing.T) {
contractorJWT := mintToken(map[string]any{"sub": "contractor-user", "email": "[email protected]"})
- // contractor is allowed GET /repos/acme/public/*
- reqAllow, _ := http.NewRequest(http.MethodGet, "http://"+callerAPI+"/ext_authz/egress/api.github.com/repos/acme/public/readme", nil)
+ // Gateway proxy in front of caller node's mesh dataplane that consults
+ // caller node's /ext_authz before forwarding through /sam/<pepPeer>/egress/api.github.com.
+ extAuthzGateway := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ checkURL := "http://" + callerAPI + "/ext_authz/egress/api.github.com" + r.URL.Path
+ checkReq, err := http.NewRequestWithContext(r.Context(), r.Method, checkURL, nil)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ checkReq.Header = r.Header.Clone()
+ checkResp, err := client.Do(checkReq)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ _ = checkResp.Body.Close()
+ if checkResp.StatusCode != http.StatusOK {
+ w.WriteHeader(checkResp.StatusCode)
+ return
+ }
+
+ // Forward the authorized request into caller node's mesh dataplane using the
+ // X-Sam-Biscuit injected by ext_authz.
+ meshURL := fmt.Sprintf("http://%s/sam/%s/egress/api.github.com%s", callerAPI, pepPeer, r.URL.Path)
+ fwdReq, err := http.NewRequestWithContext(r.Context(), r.Method, meshURL, r.Body)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ for _, h := range []string{api.HeaderSamBiscuit, api.HeaderSamPrincipal, api.HeaderSamRoles, api.HeaderSamTask} {
+ if v := checkResp.Header.Get(h); v != "" {
+ fwdReq.Header.Set(h, v)
+ }
+ }
+ fwdResp, err := client.Do(fwdReq)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ defer func() { _ = fwdResp.Body.Close() }()
+ for k, vals := range fwdResp.Header {
+ for _, v := range vals {
+ w.Header().Add(k, v)
+ }
+ }
+ w.Header().Set(api.HeaderSamRoles, checkResp.Header.Get(api.HeaderSamRoles))
+ w.WriteHeader(fwdResp.StatusCode)
+ _, _ = io.Copy(w, fwdResp.Body)
+ }))
+ defer extAuthzGateway.Close()
+
+ // 1. contractor is allowed GET /repos/acme/public/readme -> traverses ext_authz -> caller node -> libp2p -> pep node -> ext_proc callout -> upstream!
+ reqAllow, _ := http.NewRequest(http.MethodGet, extAuthzGateway.URL+"/repos/acme/public/readme", nil)
reqAllow.Header.Set("Authorization", "Bearer "+contractorJWT)
respAllow, err := client.Do(reqAllow)
if err != nil {
- t.Fatalf("ext_authz allow request: %v", err)
+ t.Fatalf("ext_authz gateway allow request: %v", err)
}
+ allowBody, _ := io.ReadAll(respAllow.Body)
_ = respAllow.Body.Close()
if respAllow.StatusCode != http.StatusOK {
- t.Fatalf("ext_authz allow status = %d, want 200", respAllow.StatusCode)
+ t.Fatalf("ext_authz datapath allow status = %d, want 200: %s", respAllow.StatusCode, string(allowBody))
}
if got := respAllow.Header.Get(api.HeaderSamRoles); got != "contractor" {
t.Fatalf("ext_authz X-Sam-Roles = %q, want \"contractor\" (no node role leakage)", got)
}
+ if !strings.Contains(string(allowBody), `"path":"/repos/acme/public/readme"`) || !strings.Contains(string(allowBody), `"auth":"Bearer ghp_upstream_secret"`) {
+ t.Fatalf("unexpected upstream datapath payload: %s", string(allowBody))
+ }
- // contractor is denied GET /repos/acme/private/secret (even though caller node has developer role!)
- reqDeny, _ := http.NewRequest(http.MethodGet, "http://"+callerAPI+"/ext_authz/egress/api.github.com/repos/acme/private/secret", nil)
+ // 2. contractor is denied GET /repos/acme/private/secret (even though caller node has developer role!)
+ reqDeny, _ := http.NewRequest(http.MethodGet, extAuthzGateway.URL+"/repos/acme/private/secret", nil)
reqDeny.Header.Set("Authorization", "Bearer "+contractorJWT)
respDeny, err := client.Do(reqDeny)
if err != nil {
- t.Fatalf("ext_authz deny request: %v", err)
+ t.Fatalf("ext_authz gateway deny request: %v", err)
}
_ = respDeny.Body.Close()
if respDeny.StatusCode != http.StatusForbidden {
- t.Fatalf("ext_authz deny status = %d, want 403", respDeny.StatusCode)
+ t.Fatalf("ext_authz datapath deny status = %d, want 403", respDeny.StatusCode)
}
})
- t.Run("Task biscuit attenuation, mesh enforcement, outbound STS JWT minting, and task revocation", func(t *testing.T) {
+ t.Run("Task biscuit attenuation, mesh enforcement, egress ext_proc callout, outbound STS JWT minting, and task revocation", func(t *testing.T) {
// 1. Exchange caller node's standing Biscuit into a narrowed Task Biscuit
// only allowing GET /repos/acme/public/* on egress://api.github.com.
form := url.Values{}
@@ -264,29 +490,43 @@ egress:
}
baseMeshURL := fmt.Sprintf("http://%s/sam/%s/egress/api.github.com", callerAPI, pepPeer)
- doWithCred := func(method, path, cred string) int {
+ doWithCred := func(method, path, cred string) (int, string) {
req, _ := http.NewRequest(method, baseMeshURL+path, nil)
req.Header.Set(api.HeaderSamAuthentication, "Bearer "+cred)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("%s %s: %v", method, path, err)
}
+ body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
- return resp.StatusCode
+ return resp.StatusCode, string(body)
}
// 2. Allowed by both developer role and tar_block: GET /repos/acme/public/readme
- if got := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusNoContent {
- t.Fatalf("GET /repos/acme/public/readme with taskBiscuit: %d, want 204", got)
+ // Also verifies the egress ext_proc callout mutated X-Ext-Proc-Inspected=true
+ // and sam-node injected Authorization: Bearer ghp_upstream_secret.
+ if got, body := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusOK || !strings.Contains(body, `"auth":"Bearer ghp_upstream_secret"`) {
+ t.Fatalf("GET /repos/acme/public/readme with taskBiscuit: status=%d body=%s", got, body)
+ }
+ if got, _ := upstreamAuthHeader.Load().(string); got != "Bearer ghp_upstream_secret" {
+ t.Fatalf("upstream Authorization = %q, want Bearer ghp_upstream_secret", got)
+ }
+ if got, _ := upstreamInspectedHeader.Load().(string); got != "true" {
+ t.Fatalf("upstream X-Ext-Proc-Inspected = %q, want true", got)
+ }
+
+ // 2b. Blocked by egress ext_proc callout ImmediateResponse (path allowed by RBAC + TAR, blocked by DLP callout):
+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/public/blocked-by-dlp", taskBiscuit); got != http.StatusForbidden {
+ t.Fatalf("GET /repos/acme/public/blocked-by-dlp with taskBiscuit: %d, want 403 (blocked by ext_proc callout)", got)
}
// 3. Denied by tar_block method constraint (developer role allows POST, but tar_block only allows GET):
- if got := doWithCred(http.MethodPost, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {
+ if got, _ := doWithCred(http.MethodPost, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {
t.Fatalf("POST /repos/acme/public/readme with taskBiscuit: %d, want 403", got)
}
// 4. Denied by tar_block path constraint (developer role allows /repos/acme/*, but tar_block only allows /repos/acme/public/*):
- if got := doWithCred(http.MethodGet, "/repos/acme/private/secret", taskBiscuit); got != http.StatusForbidden {
+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/private/secret", taskBiscuit); got != http.StatusForbidden {
t.Fatalf("GET /repos/acme/private/secret with taskBiscuit: %d, want 403", got)
}
@@ -328,11 +568,256 @@ egress:
}
// Revoked taskBiscuit is now rejected immediately, while caller's parent node identity still works!
- if got := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {
+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/public/readme", taskBiscuit); got != http.StatusForbidden {
t.Fatalf("revoked taskBiscuit status = %d, want 403", got)
}
- if got := doWithCred(http.MethodGet, "/repos/acme/private/secret", callerToken); got != http.StatusNoContent {
- t.Fatalf("parent node identity status = %d, want 204", got)
+ if got, _ := doWithCred(http.MethodGet, "/repos/acme/private/secret", callerToken); got != http.StatusOK {
+ t.Fatalf("parent node identity status = %d, want 200", got)
+ }
+ })
+
+ t.Run("Gateway ext_proc over h2c datapath on live sam-node (egress credential injection and MCP tools/call across mesh)", func(t *testing.T) {
+ var h2cProtocols http.Protocols
+ h2cProtocols.SetUnencryptedHTTP2(true)
+ h2cClient := &http.Client{
+ Timeout: 10 * time.Second,
+ Transport: &http.Transport{
+ ForceAttemptHTTP2: true,
+ Protocols: &h2cProtocols,
+ },
+ }
+ extProcURL := "http://" + pepAPI + "/envoy.service.ext_proc.v3.ExternalProcessor/Process"
+ mcpBackendCalls.Store(0)
+
+ // Envoy-compatible Gateway proxy that runs the ExternalProcessor/Process h2c stream
+ // against pepAPI (including ModeOverride BUFFERED body inspection and HeaderMutation)
+ // before forwarding to the target backend or mesh dataplane.
+ gatewayProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var bodyBytes []byte
+ if r.Body != nil {
+ bodyBytes, _ = io.ReadAll(r.Body)
+ _ = r.Body.Close()
+ }
+
+ pr, pw := io.Pipe()
+ procReq, _ := http.NewRequestWithContext(r.Context(), http.MethodPost, extProcURL, pr)
+ procReq.Header.Set("Content-Type", "application/grpc+proto")
+ procReq.Header.Set("TE", "trailers")
+
+ respCh := make(chan *http.Response, 1)
+ errCh := make(chan error, 1)
+ go func() {
+ resp, err := h2cClient.Do(procReq)
+ if err != nil {
+ errCh <- err
+ return
+ }
+ respCh <- resp
+ }()
+
+ hdrs := []*corev3.HeaderValue{
+ {Key: ":method", Value: r.Method},
+ {Key: ":authority", Value: r.Host},
+ {Key: ":path", Value: r.URL.RequestURI()},
+ }
+ for k, vals := range r.Header {
+ hdrs = append(hdrs, &corev3.HeaderValue{Key: strings.ToLower(k), Value: strings.Join(vals, ", ")})
+ }
+ _ = writeExtProcFrame(pw, &extprocv3.ProcessingRequest{
+ Request: &extprocv3.ProcessingRequest_RequestHeaders{
+ RequestHeaders: &extprocv3.HttpHeaders{
+ EndOfStream: len(bodyBytes) == 0,
+ Headers: &corev3.HeaderMap{Headers: hdrs},
+ },
+ },
+ })
+ if len(bodyBytes) == 0 {
+ _ = pw.Close()
+ }
+
+ var procHTTPResp *http.Response
+ select {
+ case procHTTPResp = <-respCh:
+ case err := <-errCh:
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ defer func() { _ = procHTTPResp.Body.Close() }()
+
+ var procResp extprocv3.ProcessingResponse
+ if err := readExtProcFrame(procHTTPResp.Body, &procResp); err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ if imm := procResp.GetImmediateResponse(); imm != nil {
+ w.WriteHeader(int(imm.GetStatus().GetCode()))
+ _, _ = w.Write(imm.GetBody())
+ return
+ }
+
+ applyMuts := func(mut *extprocv3.HeaderMutation) {
+ if mut == nil {
+ return
+ }
+ for _, rem := range mut.GetRemoveHeaders() {
+ r.Header.Del(rem)
+ }
+ for _, sh := range mut.GetSetHeaders() {
+ k := sh.GetHeader().GetKey()
+ v := sh.GetHeader().GetValue()
+ if v == "" {
+ v = string(sh.GetHeader().GetRawValue())
+ }
+ r.Header.Set(k, v)
+ }
+ }
+ applyMuts(procResp.GetRequestHeaders().GetResponse().GetHeaderMutation())
+
+ if len(bodyBytes) > 0 && procResp.GetModeOverride().GetRequestBodyMode() == extprocv3http.ProcessingMode_BUFFERED {
+ _ = writeExtProcFrame(pw, &extprocv3.ProcessingRequest{
+ Request: &extprocv3.ProcessingRequest_RequestBody{
+ RequestBody: &extprocv3.HttpBody{
+ Body: bodyBytes,
+ EndOfStream: true,
+ },
+ },
+ })
+ _ = pw.Close()
+
+ var bodyProcResp extprocv3.ProcessingResponse
+ if err := readExtProcFrame(procHTTPResp.Body, &bodyProcResp); err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ if imm := bodyProcResp.GetImmediateResponse(); imm != nil {
+ w.WriteHeader(int(imm.GetStatus().GetCode()))
+ _, _ = w.Write(imm.GetBody())
+ return
+ }
+ applyMuts(bodyProcResp.GetRequestBody().GetResponse().GetHeaderMutation())
+ } else {
+ _ = pw.Close()
+ }
+
+ // Route /sam/... requests through caller node's real libp2p mesh proxy,
+ // and direct egress Host-routed requests to upstream.
+ targetBase := upstream.URL
+ if strings.HasPrefix(r.URL.Path, "/sam/") {
+ targetBase = "http://" + callerAPI
+ }
+ fwdReq, _ := http.NewRequestWithContext(r.Context(), r.Method, targetBase+r.URL.RequestURI(), strings.NewReader(string(bodyBytes)))
+ fwdReq.Header = r.Header.Clone()
+ fwdResp, err := client.Do(fwdReq)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ defer func() { _ = fwdResp.Body.Close() }()
+ w.WriteHeader(fwdResp.StatusCode)
+ _, _ = io.Copy(w, fwdResp.Body)
+ }))
+ defer gatewayProxy.Close()
+
+ // 1. Egress credential brokering datapath over Gateway ext_proc:
+ // Contractor sends contractorJWT with Host: api.github.com -> gateway ext_proc exchanges JWT,
+ // strips contractorJWT, injects Authorization: Bearer ghp_upstream_secret & X-Sam-Roles: contractor,
+ // and forwards to upstream.
+ contractorJWT := mintToken(map[string]any{"sub": "contractor-user", "email": "[email protected]"})
+ egressReq, _ := http.NewRequest(http.MethodGet, gatewayProxy.URL+"/repos/acme/public/readme", nil)
+ egressReq.Host = "api.github.com"
+ egressReq.Header.Set("Authorization", "Bearer "+contractorJWT)
+ egressResp, err := client.Do(egressReq)
+ if err != nil {
+ t.Fatalf("gateway ext_proc egress request: %v", err)
+ }
+ egressBody, _ := io.ReadAll(egressResp.Body)
+ _ = egressResp.Body.Close()
+ if egressResp.StatusCode != http.StatusOK {
+ t.Fatalf("gateway ext_proc egress status = %d, want 200: %s", egressResp.StatusCode, string(egressBody))
+ }
+ if !strings.Contains(string(egressBody), `"auth":"Bearer ghp_upstream_secret"`) || !strings.Contains(string(egressBody), `"roles":"contractor"`) {
+ t.Fatalf("expected upstream to receive injected secret and contractor role, got %s", string(egressBody))
+ }
+
+ // 2. MCP tools/call datapath over Gateway ext_proc AND across the libp2p mesh:
+ // Mint a Task Biscuit on caller node scoped to tool:get_pr on mcp://github.
+ mcpForm := url.Values{}
+ mcpForm.Set("grant_type", api.GrantTypeTokenExchange)
+ mcpForm.Add("resource", "mcp://github")
+ mcpForm.Set("scope", "tool:get_pr")
+ reqMCPTok, _ := http.NewRequest(http.MethodPost, "http://"+callerAPI+"/oauth/token", strings.NewReader(mcpForm.Encode()))
+ reqMCPTok.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ reqMCPTok.Header.Set("Authorization", "Bearer "+callerToken)
+ respMCPTok, err := client.Do(reqMCPTok)
+ if err != nil {
+ t.Fatalf("POST /oauth/token (mcp task biscuit): %v", err)
+ }
+ var mcpTokData map[string]any
+ _ = json.NewDecoder(respMCPTok.Body).Decode(&mcpTokData)
+ _ = respMCPTok.Body.Close()
+ mcpTaskBiscuit, _ := mcpTokData["access_token"].(string)
+ if mcpTaskBiscuit == "" {
+ t.Fatalf("expected mcpTaskBiscuit, got %v", mcpTokData)
+ }
+
+ mcpMeshPath := fmt.Sprintf("/sam/%s/mcp/github", pepPeer)
+
+ // 2a. Allowed MCP tool "get_pr" traverses:
+ // gatewayProxy -> ext_proc (RequestHeaders -> ModeOverride BUFFERED -> RequestBody) ->
+ // caller node (/sam/<pepPeer>/mcp/github) -> libp2p (/libp2p-http) -> pep node -> mcpBackend!
+ mcpAllowReq, _ := http.NewRequest(http.MethodPost, gatewayProxy.URL+mcpMeshPath,
+ strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"get_pr"}}`))
+ mcpAllowReq.Header.Set("Authorization", "Bearer "+mcpTaskBiscuit)
+ mcpAllowReq.Header.Set("Content-Type", "application/json")
+ mcpAllowResp, err := client.Do(mcpAllowReq)
+ if err != nil {
+ t.Fatalf("gateway ext_proc MCP allow: %v", err)
+ }
+ mcpAllowBody, _ := io.ReadAll(mcpAllowResp.Body)
+ _ = mcpAllowResp.Body.Close()
+ wantText := fmt.Sprintf("PR #42 from dev-user peer=%s biscuit=", callerPeer)
+ if mcpAllowResp.StatusCode != http.StatusOK || !strings.Contains(string(mcpAllowBody), wantText) {
+ t.Fatalf("expected 200 OK with %q, got %d: %s", wantText, mcpAllowResp.StatusCode, string(mcpAllowBody))
+ }
+ if mcpBackendCalls.Load() != 1 {
+ t.Fatalf("expected 1 mcpBackend call, got %d", mcpBackendCalls.Load())
+ }
+
+ // 2b. Unauthorized MCP tool "delete_repo" is blocked at the gateway by ext_proc ImmediateResponse(403)
+ // before ever entering the mesh or reaching mcpBackend.
+ mcpDenyReq, _ := http.NewRequest(http.MethodPost, gatewayProxy.URL+mcpMeshPath,
+ strings.NewReader(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"delete_repo"}}`))
+ mcpDenyReq.Header.Set("Authorization", "Bearer "+mcpTaskBiscuit)
+ mcpDenyReq.Header.Set("Content-Type", "application/json")
+ mcpDenyResp, err := client.Do(mcpDenyReq)
+ if err != nil {
+ t.Fatalf("gateway ext_proc MCP deny: %v", err)
+ }
+ _ = mcpDenyResp.Body.Close()
+ if mcpDenyResp.StatusCode != http.StatusForbidden {
+ t.Fatalf("expected 403 Forbidden for unauthorized MCP tool delete_repo at gateway, got %d", mcpDenyResp.StatusCode)
+ }
+ if mcpBackendCalls.Load() != 1 {
+ t.Fatalf("expected mcpBackendCalls to remain 1 after gateway blocked tool call, got %d", mcpBackendCalls.Load())
+ }
+
+ // 2c. Even if a caller bypasses the gateway proxy and sends "delete_repo" directly across
+ // the libp2p mesh to pep node, pep node's /libp2p-http ingress inspects the JSON-RPC body
+ // and enforces the Task Biscuit's tool:get_pr constraint with 403 Forbidden!
+ directMeshDenyReq, _ := http.NewRequest(http.MethodPost, "http://"+callerAPI+mcpMeshPath,
+ strings.NewReader(`{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"delete_repo"}}`))
+ directMeshDenyReq.Header.Set(api.HeaderSamAuthentication, "Bearer "+mcpTaskBiscuit)
+ directMeshDenyReq.Header.Set("Content-Type", "application/json")
+ directMeshDenyResp, err := client.Do(directMeshDenyReq)
+ if err != nil {
+ t.Fatalf("direct mesh MCP deny: %v", err)
+ }
+ _ = directMeshDenyResp.Body.Close()
+ if directMeshDenyResp.StatusCode != http.StatusForbidden {
+ t.Fatalf("expected 403 Forbidden from provider node on direct mesh call for unauthorized tool, got %d", directMeshDenyResp.StatusCode)
+ }
+ if mcpBackendCalls.Load() != 1 {
+ t.Fatalf("expected mcpBackendCalls to remain 1 after provider node blocked tool call, got %d", mcpBackendCalls.Load())
}
})
}