Skip to content

Agent Mesh

node: answer 502, not 403, for a provider that did not answer

Antonio Ojea7c03f3946beb
diff --git a/internal/node/a2a_service.go b/internal/node/a2a_service.go
index 5564d102..f639c9f6 100644
--- a/internal/node/a2a_service.go
+++ b/internal/node/a2a_service.go
@@ -108,7 +108,8 @@ func a2aEgressGate(node *AgentMeshNode, w http.ResponseWriter, r *http.Request,
}
if err := node.VerifyPeerLabels(r.Context(), pid, required); err != nil {
logger.Warnf("[A2A] label gate refused egress to %s: %v", route.peerID, err)
- http.Error(w, "Required labels not attested by provider", http.StatusForbidden)
+ text, status := labelGateStatus(err, "Required labels not attested by provider")
+ http.Error(w, text, status)
return r, false
}
}
diff --git a/internal/node/egress.go b/internal/node/egress.go
index 4cb2baee..4a437986 100644
--- a/internal/node/egress.go
+++ b/internal/node/egress.go
@@ -47,9 +47,10 @@ const DefaultSecretsDir = "/etc/agentmesh/secrets"
// the destination, refuses a request. A client can then tell a policy denial
// from a 403 the destination sent.
const (
- proxyStatusDenied = "http_request_denied"
- proxyStatusDestinationNotFound = "destination_not_found"
- proxyStatusConfigurationError = "proxy_configuration_error"
+ proxyStatusDenied = "http_request_denied"
+ proxyStatusDestinationNotFound = "destination_not_found"
+ proxyStatusDestinationUnreachable = "destination_unavailable"
+ proxyStatusConfigurationError = "proxy_configuration_error"
)
// refuse answers a request the node refuses itself, naming the reason in
diff --git a/internal/node/egress_tunnel.go b/internal/node/egress_tunnel.go
index 8204cc2b..1d5ced50 100644
--- a/internal/node/egress_tunnel.go
+++ b/internal/node/egress_tunnel.go
@@ -19,6 +19,7 @@ import (
"bytes"
"context"
"encoding/base64"
+ "errors"
"fmt"
"io"
"net"
@@ -419,6 +420,10 @@ func handleConnectTunnel(node *AgentMeshNode, w http.ResponseWriter, r *http.Req
refuse(w, http.StatusNotFound, fmt.Sprintf("no provider found for egress://%s", host), proxyStatusDestinationNotFound)
return
}
+ // A provider that was heard from and refused is a verdict; one that
+ // never answered is not, and when every provider is in that state the
+ // caller is told so rather than told it was denied.
+ var denied bool
for _, p := range providers {
targetPeer, decErr := peer.Decode(p.GetPeerId())
if decErr != nil {
@@ -426,11 +431,16 @@ func handleConnectTunnel(node *AgentMeshNode, w http.ResponseWriter, r *http.Req
}
if err := node.VerifyPeerLabels(r.Context(), targetPeer, requiredLabels); err != nil {
logger.Warnf("[Egress] CONNECT label gate refused egress to %s: %v", targetPeer, err)
+ denied = denied || !errors.Is(err, ErrProviderUnreachable)
continue
}
forwardConnectTunnelToPeer(node, w, r, targetPeer, host, port, identity)
return
}
+ if !denied {
+ refuse(w, http.StatusBadGateway, "Bad Gateway: no provider answered", proxyStatusDestinationUnreachable)
+ return
+ }
recordEgressDecision(host, egressOutcomeDeny)
refuse(w, http.StatusForbidden, "Required labels not attested by provider", proxyStatusDenied)
}
diff --git a/internal/node/labels_gate.go b/internal/node/labels_gate.go
index 5d30f22b..0f5164d3 100644
--- a/internal/node/labels_gate.go
+++ b/internal/node/labels_gate.go
@@ -17,7 +17,9 @@ package node
import (
"context"
"crypto/ed25519"
+ "errors"
"fmt"
+ "net/http"
"sort"
"strings"
"time"
@@ -90,6 +92,23 @@ const (
labelGateDialTimeout = 10 * time.Second
)
+// ErrProviderUnreachable marks a label-gate failure in which the provider
+// could not be reached or did not answer, as against one in which it
+// answered and its credential did not satisfy the requirement. The first
+// is the mesh's state and a caller may retry it; the second is a verdict.
+var ErrProviderUnreachable = errors.New("provider unreachable")
+
+// labelGateStatus is the HTTP answer for a label-gate failure, in the
+// order http.Error takes it: 502 when the provider was never heard from, so
+// a caller does not read a dead or busy destination as a denial of its
+// request, 403 with the given text for a verdict.
+func labelGateStatus(err error, denied string) (string, int) {
+ if errors.Is(err, ErrProviderUnreachable) {
+ return "Bad Gateway: destination did not answer", http.StatusBadGateway
+ }
+ return denied, http.StatusForbidden
+}
+
// labelGateKey builds a deterministic cache key from a required label set and
// the floor in force. The floor is part of the key because a verdict reached
// under one floor says nothing about another: keying on the caller's
@@ -155,7 +174,7 @@ func (n *AgentMeshNode) VerifyPeerLabels(ctx context.Context, peerID peer.ID, re
providerBiscuit, err := n.fetchPeerBiscuit(ctx, peerID)
if err != nil {
- return fmt.Errorf("provider %s labels unverifiable: %w", peerID, err)
+ return fmt.Errorf("provider %s labels unverifiable: %w: %w", peerID, ErrProviderUnreachable, err)
}
if err := n.checkPeerLabels(providerBiscuit, peerID, required); err != nil {
return err
diff --git a/internal/node/labels_gate_test.go b/internal/node/labels_gate_test.go
index 2cc2f69f..d2f30720 100644
--- a/internal/node/labels_gate_test.go
+++ b/internal/node/labels_gate_test.go
@@ -17,6 +17,8 @@ package node
import (
"context"
"crypto/ed25519"
+ "errors"
+ "net/http"
"testing"
"time"
@@ -249,6 +251,39 @@ func TestVerifyPeerLabelsDoesNotShortCircuit(t *testing.T) {
}
}
+// A provider that cannot be reached is not one that refused: the gate marks
+// the first so a caller is answered 502 and may retry, and a verdict on a
+// credential that was read stays a 403. On the testnet every caller of a
+// provider that was down, or too busy to answer in time, was told
+// "destination is not an enrolled peer".
+func TestLabelGateTellsUnreachableFromDenied(t *testing.T) {
+ node := &AgentMeshNode{BiscuitTimeout: 500 * time.Millisecond, nodeConfig: &NodeConfigComplete{}}
+ cache, err := lru.New[string, time.Time](8)
+ if err != nil {
+ t.Fatal(err)
+ }
+ node.peerLabelGate = cache
+
+ // No host, no identity: the biscuit fetch fails before any credential
+ // is read, which is the unreachable case.
+ err = node.VerifyPeerLabels(context.Background(), peer.ID("some-peer"), nil)
+ if !errors.Is(err, ErrProviderUnreachable) {
+ t.Fatalf("fetch failure: err = %v, want ErrProviderUnreachable", err)
+ }
+ if text, status := labelGateStatus(err, "denied"); status != http.StatusBadGateway || text == "denied" {
+ t.Errorf("unreachable: got %d %q, want 502", status, text)
+ }
+
+ // A credential that was read and found wanting is a verdict.
+ err = node.checkPeerLabels(nil, peer.ID("some-peer"), nil)
+ if err == nil || errors.Is(err, ErrProviderUnreachable) {
+ t.Fatalf("empty credential: err = %v, want a denial", err)
+ }
+ if text, status := labelGateStatus(err, "denied"); status != http.StatusForbidden || text != "denied" {
+ t.Errorf("denied: got %d %q, want 403 denied", status, text)
+ }
+}
+
// The cache key has to separate the caller's requirement from the floor, or a
// pass recorded under one floor could be replayed under another.
func TestLabelGateKeySeparatesFloorFromRequirement(t *testing.T) {
diff --git a/internal/node/sidecar.go b/internal/node/sidecar.go
index a8c73594..e99e7c98 100644
--- a/internal/node/sidecar.go
+++ b/internal/node/sidecar.go
@@ -809,13 +809,14 @@ func createEgressProxy(node *AgentMeshNode) http.Handler {
}
if err := node.VerifyPeerLabels(r.Context(), pid, requiredLabels); err != nil {
logger.Warnf("[Egress] refused egress to %s: %v", pid, err)
+ denied := "Forbidden: destination is not an enrolled peer"
if len(requiredLabels) > 0 {
- http.Error(w, "Required labels not attested by provider", http.StatusForbidden)
+ denied = "Required labels not attested by provider"
} else if len(node.egressFloor()) > 0 {
- http.Error(w, "Forbidden: provider does not attest the egress floor", http.StatusForbidden)
- } else {
- http.Error(w, "Forbidden: destination is not an enrolled peer", http.StatusForbidden)
+ denied = "Forbidden: provider does not attest the egress floor"
}
+ text, status := labelGateStatus(err, denied)
+ http.Error(w, text, status)
return
}