Skip to content

Agent Mesh

node: tell an authorizer run limit apart from a provider denial

HosniBelfekid696a18a84df
diff --git a/internal/node/labels_gate.go b/internal/node/labels_gate.go
index 6349ecee..c237c257 100644
--- a/internal/node/labels_gate.go
+++ b/internal/node/labels_gate.go
@@ -24,6 +24,7 @@ import (
"strings"
"time"
+ "github.com/eclipse-biscuit/biscuit-go/v2/datalog"
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/identity"
"github.com/libp2p/go-libp2p/core/network"
@@ -149,14 +150,36 @@ func reachFailure(err error) string {
// labelGateStatus is the HTTP answer for a label-gate failure, in the
// order http.Error takes it: 502 when the provider was never heard from, so
// a caller does not read a dead or busy destination as a denial of its
-// request, 403 with the given text for a verdict.
+// request, 504 or 500 when the authorizer stopped at a run limit before it
+// reached a verdict on the provider's credential, 403 with the given text
+// for a verdict.
func labelGateStatus(err error, denied string) (string, int) {
if errors.Is(err, ErrProviderUnreachable) {
return "Bad Gateway: destination did not answer", http.StatusBadGateway
}
+ if stopped, transient := authorizerLimit(err); stopped {
+ if transient {
+ return "Gateway Timeout: the provider's credential was not checked in time", http.StatusGatewayTimeout
+ }
+ return "Internal Server Error: the provider's credential exceeds the authorizer's limits", http.StatusInternalServerError
+ }
return denied, http.StatusForbidden
}
+// authorizerLimit reports whether err is an authorizer that stopped at one
+// of its run limits before it reached a verdict, and whether that limit is
+// time, the only one that depends on load: the facts and iterations limits
+// are reached again with the same credential and policy.
+func authorizerLimit(err error) (stopped, transient bool) {
+ switch {
+ case errors.Is(err, datalog.ErrWorldRunLimitTimeout):
+ return true, true
+ case errors.Is(err, datalog.ErrWorldRunLimitMaxFacts), errors.Is(err, datalog.ErrWorldRunLimitMaxIterations):
+ return true, false
+ }
+ return false, false
+}
+
// labelGateKey builds a deterministic cache key from a required label set and
// the floor in force. The floor is part of the key because a verdict reached
// under one floor says nothing about another: keying on the caller's
@@ -317,6 +340,9 @@ func (n *AgentMeshNode) checkProvider(providerBiscuit []byte, peerID peer.ID, re
meshRules := n.MeshPolicyRules
n.MeshPolicyMu.RUnlock()
if err := identity.RequirePublish(b, key, svcType, svcName, meshRules, n.BiscuitTimeout); err != nil {
+ if stopped, _ := authorizerLimit(err); stopped {
+ return fmt.Errorf("provider %s grant for %s undecided: %w", peerID, service, err)
+ }
return fmt.Errorf("%w: %w", ErrProviderNotGranted, err)
}
}
diff --git a/internal/node/labels_gate_test.go b/internal/node/labels_gate_test.go
index 16880099..49820fe6 100644
--- a/internal/node/labels_gate_test.go
+++ b/internal/node/labels_gate_test.go
@@ -23,6 +23,8 @@ import (
"testing"
"time"
+ "github.com/eclipse-biscuit/biscuit-go/v2"
+ "github.com/eclipse-biscuit/biscuit-go/v2/datalog"
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/identity"
lru "github.com/hashicorp/golang-lru/v2"
@@ -354,6 +356,58 @@ func TestLabelGateTellsUnreachableFromDenied(t *testing.T) {
}
}
+// authorizerFactLimitRules are mesh policy rules that derive more facts
+// from role(node) than the authorizer admits, so any check that adds them
+// stops at datalog.ErrWorldRunLimitMaxFacts before it reaches a verdict.
+func authorizerFactLimitRules(t *testing.T) []biscuit.Rule {
+ t.Helper()
+ texts := make([]string, 0, 1100)
+ for i := range 1100 {
+ texts = append(texts, fmt.Sprintf("limit_fact_%d(true) <- role(%q)", i, api.RoleNode))
+ }
+ rules, err := api.ParseDatalogRules(texts)
+ if err != nil {
+ t.Fatalf("ParseDatalogRules: %v", err)
+ }
+ return rules
+}
+
+// An authorizer that stops at a run limit reaches no verdict on the
+// provider's credential, so the caller is not told it was denied. Time
+// depends on load: 504, and the caller may retry. The facts and iterations
+// limits are reached again with the same credential and policy: 500.
+func TestLabelGateTellsAuthorizerLimitFromDenied(t *testing.T) {
+ cpPub, cpPriv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ providerPeer := peer.ID("provider-peer-id")
+ role := &api.PolicyRole{Name: api.RoleNode, AllowedPublish: []string{"mcp://calc"}}
+ granted, err := identity.MintBootstrapBiscuitToken(cpPriv, providerPeer, api.RoleNode, time.Now().Add(time.Hour), []*api.PolicyRole{role}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ trusted := []TrustedKey{{Key: cpPub, ReceivedAt: time.Now()}}
+
+ slow := &AgentMeshNode{trustedKeys: trusted, BiscuitTimeout: time.Nanosecond}
+ err = slow.checkProvider(granted, providerPeer, nil, "mcp://calc")
+ if !errors.Is(err, datalog.ErrWorldRunLimitTimeout) {
+ t.Fatalf("checkProvider with a 1ns timeout = %v, want an error wrapping %v", err, datalog.ErrWorldRunLimitTimeout)
+ }
+ if text, status := labelGateStatus(err, "denied"); status != http.StatusGatewayTimeout || text == "denied" {
+ t.Errorf("timeout: got %d %q, want 504", status, text)
+ }
+
+ full := &AgentMeshNode{trustedKeys: trusted, BiscuitTimeout: 500 * time.Millisecond, MeshPolicyRules: authorizerFactLimitRules(t)}
+ err = full.checkProvider(granted, providerPeer, nil, "mcp://calc")
+ if !errors.Is(err, datalog.ErrWorldRunLimitMaxFacts) || errors.Is(err, ErrProviderNotGranted) {
+ t.Fatalf("checkProvider over the facts limit = %v, want an error wrapping %v and not %v", err, datalog.ErrWorldRunLimitMaxFacts, ErrProviderNotGranted)
+ }
+ if text, status := labelGateStatus(err, "denied"); status != http.StatusInternalServerError || text == "denied" {
+ t.Errorf("facts limit: got %d %q, want 500", status, text)
+ }
+}
+
// The cache key has to separate the caller's requirement from the floor, or a
// pass recorded under one floor could be replayed under another.
func TestLabelGateKeySeparatesFloorFromRequirement(t *testing.T) {
diff --git a/internal/node/mcp.go b/internal/node/mcp.go
index cc4bc61e..ed7e606f 100644
--- a/internal/node/mcp.go
+++ b/internal/node/mcp.go
@@ -225,6 +225,12 @@ func (n *AgentMeshNode) CallMCPTool(ctx context.Context, targetPeer peer.ID, too
logger.Warnf("[MCP] Tool call denied, not retrying: %v", err)
return nil, err
}
+ // An authorizer that stopped at its facts or iterations limit stops
+ // there again on a retry; only its time limit depends on load.
+ if stopped, transient := authorizerLimit(err); stopped && !transient {
+ logger.Warnf("[MCP] Tool call stopped at an authorizer limit, not retrying: %v", err)
+ return nil, err
+ }
logger.Warnf("[MCP] Tool call failed, retrying in %v: %v", backoff, err)
timer := time.NewTimer(backoff)
select {
diff --git a/internal/node/mcp_handlers_test.go b/internal/node/mcp_handlers_test.go
index 65794ffa..e14e3a97 100644
--- a/internal/node/mcp_handlers_test.go
+++ b/internal/node/mcp_handlers_test.go
@@ -28,6 +28,7 @@ import (
"time"
"github.com/eclipse-biscuit/biscuit-go/v2"
+ "github.com/eclipse-biscuit/biscuit-go/v2/datalog"
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/identity"
"github.com/libp2p/go-libp2p/core/crypto"
@@ -652,6 +653,20 @@ func TestCallMCPTool_ProviderNotGrantedIsNotRetried(t *testing.T) {
if strings.Contains(err.Error(), "retries") {
t.Fatalf("CallMCPTool to an ungranted provider = %v, want one refusal with no retry", err)
}
+
+ // A publish check that stops at the facts limit is no verdict on the
+ // grant, and it is not retried either: the same credential and policy
+ // reach the limit again.
+ nodeA.MeshPolicyMu.Lock()
+ nodeA.MeshPolicyRules = authorizerFactLimitRules(t)
+ nodeA.MeshPolicyMu.Unlock()
+ _, err = nodeA.CallMCPTool(ctx, nodeB.Host.ID(), "mcp://code-reviewer/review_pr", map[string]any{}, nil)
+ if !errors.Is(err, datalog.ErrWorldRunLimitMaxFacts) || errors.Is(err, ErrProviderNotGranted) {
+ t.Fatalf("CallMCPTool over the facts limit = %v, want an error wrapping %v and not %v", err, datalog.ErrWorldRunLimitMaxFacts, ErrProviderNotGranted)
+ }
+ if strings.Contains(err.Error(), "retries") {
+ t.Fatalf("CallMCPTool over the facts limit = %v, want one refusal with no retry", err)
+ }
}
// The operator's egress floor gates MCP egress even when the caller supplies no