Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(egress): process-wide resolve semaphore + structured audit fields

高然bb9b48fa0c47
diff --git a/components/egress/pkg/credentialvault/resolve_handler.go b/components/egress/pkg/credentialvault/resolve_handler.go
index 006e907a0..79d5b48f5 100644
--- a/components/egress/pkg/credentialvault/resolve_handler.go
+++ b/components/egress/pkg/credentialvault/resolve_handler.go
@@ -16,6 +16,7 @@ package credentialvault
import (
"bytes"
+ "context"
"crypto/sha256"
"encoding/binary"
"encoding/json"
@@ -30,6 +31,8 @@ import (
"sync"
"time"
+ slogger "github.com/alibaba/opensandbox/internal/logger"
+
"github.com/alibaba/opensandbox/egress/pkg/constants"
"github.com/alibaba/opensandbox/egress/pkg/log"
"github.com/alibaba/opensandbox/egress/pkg/telemetry"
@@ -68,6 +71,12 @@ type ResolveHandler struct {
access ResolveAccessFunc
newClient func(endpoint string) (ResolverClient, error)
audit func(event ResolveAuditEvent)
+ // The handler semaphore is the process-wide resolve bound: past the
+ // client-cache cap each request builds its own client, and per-client
+ // semaphores would no longer bound anything. timeout bounds queueing
+ // and the call (T2a semantics).
+ timeout time.Duration
+ semaphore chan struct{}
mu sync.Mutex
clients map[string]ResolverClient
@@ -79,6 +88,8 @@ func NewResolveHandler(access ResolveAccessFunc) *ResolveHandler {
access: access,
newClient: NewResolverClientFromEnv,
audit: defaultResolveAudit,
+ timeout: ResolverTimeoutFromEnv(),
+ semaphore: make(chan struct{}, ResolverMaxConcurrencyFromEnv()),
clients: make(map[string]ResolverClient),
}
}
@@ -95,8 +106,15 @@ type ResolveAuditEvent struct {
}
func defaultResolveAudit(event ResolveAuditEvent) {
- log.Infof("credential_resolve_audit resolution=%s credential=%s binding=%s target_host=%s subject=%s duration_ms=%d",
- event.Resolution, event.Credential, event.Binding, event.TargetHost, event.Subject, event.Duration.Milliseconds())
+ log.Logger.With(
+ slogger.Field{Key: "opensandbox.event", Value: "egress.credential.resolve"},
+ slogger.Field{Key: "resolution", Value: event.Resolution},
+ slogger.Field{Key: "credential", Value: event.Credential},
+ slogger.Field{Key: "binding", Value: event.Binding},
+ slogger.Field{Key: "target.host", Value: event.TargetHost},
+ slogger.Field{Key: "subject", Value: event.Subject},
+ slogger.Field{Key: "duration_ms", Value: event.Duration.Milliseconds()},
+ ).Infof("credential resolve audit")
telemetry.RecordCredentialResolution(event.Resolution, event.Duration.Seconds())
}
@@ -168,7 +186,17 @@ func (h *ResolveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeResolveError(w, http.StatusServiceUnavailable, "resolver unavailable")
return
}
- decision, err := client.Resolve(r.Context(), ResolveInput{
+ resolveCtx, cancelResolve := context.WithTimeout(r.Context(), h.timeout)
+ defer cancelResolve()
+ select {
+ case h.semaphore <- struct{}{}:
+ defer func() { <-h.semaphore }()
+ case <-resolveCtx.Done():
+ outcome = telemetry.ResolutionOutcomeTimeout
+ writeResolveError(w, http.StatusGatewayTimeout, "resolver timeout")
+ return
+ }
+ decision, err := client.Resolve(resolveCtx, ResolveInput{
Subject: access.SandboxID,
Credential: capture.Credential,
Binding: document.Binding,
@@ -208,7 +236,8 @@ func (h *ResolveHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// maxCachedResolverClients bounds the never-evicted per-endpoint cache; past
-// the cap, clients are rebuilt per request (construction is parse-only).
+// the cap, clients are rebuilt per request (construction is parse-only). The
+// handler semaphore keeps the process-wide concurrency bound either way.
const maxCachedResolverClients = 64
// clientFor caches one ResolverClient per endpoint; the env never changes.
diff --git a/components/egress/pkg/credentialvault/resolve_handler_test.go b/components/egress/pkg/credentialvault/resolve_handler_test.go
index 235c0fc9c..179a822fe 100644
--- a/components/egress/pkg/credentialvault/resolve_handler_test.go
+++ b/components/egress/pkg/credentialvault/resolve_handler_test.go
@@ -21,14 +21,18 @@ import (
"fmt"
"net/http"
"net/http/httptest"
+ "os"
+ "path/filepath"
"strings"
"sync"
"testing"
"time"
+ slogger "github.com/alibaba/opensandbox/internal/logger"
"github.com/stretchr/testify/require"
"github.com/alibaba/opensandbox/egress/pkg/constants"
+ "github.com/alibaba/opensandbox/egress/pkg/log"
)
// fakeResolveClient is an in-process ResolverClient: records inputs, blocks
@@ -590,6 +594,96 @@ func TestResolveHandlerClientCacheBounded(t *testing.T) {
require.Equal(t, maxCachedResolverClients+1, h.newCalls, "cached endpoint must not rebuild")
}
+func TestResolveHandlerSemaphoreBoundedPastClientCache(t *testing.T) {
+ // Past the client-cache cap every request builds its own client, so only
+ // the handler semaphore keeps the process-wide concurrency bound.
+ t.Setenv(constants.EnvCredentialResolverMaxConcurrency, "1")
+ store := testResolverReadyStore()
+ _, createErr := store.Create(testExchangeVaultRequest(), testExchangePolicy(t))
+ require.NoError(t, createErr)
+
+ var mu sync.Mutex
+ inFlight, maxInFlight := 0, 0
+ slow := &slowCountingClient{mu: &mu, inFlight: &inFlight, max: &maxInFlight, delay: 150 * time.Millisecond}
+ handler := NewResolveHandler(func(*http.Request) (ResolveAccess, *ResolveAccessError) {
+ return ResolveAccess{SandboxID: "sbx-bounded", Store: store}, nil
+ })
+ handler.newClient = func(string) (ResolverClient, error) { return slow, nil }
+
+ done := make(chan int, 2)
+ for i := 0; i < 2; i++ {
+ go func() {
+ req := httptest.NewRequest(http.MethodPost, "/credential-vault/_resolve", strings.NewReader(validResolveBody()))
+ req.Header.Set("Content-Type", "application/json")
+ rec := httptest.NewRecorder()
+ handler.ServeHTTP(rec, req)
+ done <- rec.Code
+ }()
+ }
+ require.Equal(t, http.StatusOK, <-done)
+ require.Equal(t, http.StatusOK, <-done)
+
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, 1, *slow.max, "handler semaphore must serialize resolves across client instances")
+}
+
+// slowCountingClient tracks concurrent Resolve calls and answers after delay.
+type slowCountingClient struct {
+ mu *sync.Mutex
+ inFlight *int
+ max *int
+ delay time.Duration
+}
+
+func (c *slowCountingClient) Resolve(_ context.Context, _ ResolveInput) (*ResolverDecision, error) {
+ c.mu.Lock()
+ *c.inFlight++
+ if *c.inFlight > *c.max {
+ *c.max = *c.inFlight
+ }
+ c.mu.Unlock()
+ time.Sleep(c.delay)
+ c.mu.Lock()
+ *c.inFlight--
+ c.mu.Unlock()
+ return &ResolverDecision{Headers: map[string]string{}, Suppress: map[string]struct{}{}, Redactions: []string{}}, nil
+}
+
+func TestDefaultResolveAuditEmitsStructuredFields(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "audit.log")
+ captured := slogger.MustNew(slogger.Config{Level: "info", OutputPaths: []string{path}})
+ previous := log.Logger
+ log.Logger = captured
+ t.Cleanup(func() {
+ log.Logger = previous
+ _ = captured.Sync()
+ })
+
+ defaultResolveAudit(ResolveAuditEvent{
+ Resolution: "resolved",
+ Credential: "spi-signer",
+ Binding: "internal-api",
+ TargetHost: "api.internal.example.com",
+ Subject: "sbx-audit",
+ Duration: 3 * time.Millisecond,
+ })
+ require.NoError(t, captured.Sync())
+ encoded, err := os.ReadFile(path)
+ require.NoError(t, err)
+
+ var line map[string]any
+ require.NoError(t, json.Unmarshal(encoded, &line))
+ require.Equal(t, "egress.credential.resolve", line["opensandbox.event"])
+ require.Equal(t, "resolved", line["resolution"])
+ require.Equal(t, "spi-signer", line["credential"])
+ require.Equal(t, "internal-api", line["binding"])
+ require.Equal(t, "api.internal.example.com", line["target.host"])
+ require.Equal(t, "sbx-audit", line["subject"])
+ require.Contains(t, line, "duration_ms")
+ require.NotContains(t, string(encoded), "Bearer", "audit never carries values")
+}
+
func TestResolverConfigFromEnvWiring(t *testing.T) {
t.Setenv(constants.EnvCredentialResolverTimeout, "5")
t.Setenv(constants.EnvCredentialResolverAuth, "https://resolver.internal:8443=token-1")