Skip to content

Modern Docker Management, Designed for Everyone

fix: agent api token fallbacks and guards (#2568)

Kyle Mendellb37533af358e
diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.go
index 13219cd84..fa439fd2a 100644
--- a/backend/internal/services/environment_service.go
+++ b/backend/internal/services/environment_service.go
@@ -90,6 +90,7 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,
Where("access_token = ?", token).
First(&env).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
+ s.logEdgeTokenResolveMissInternal(ctx, token)
return "", errors.New("invalid agent token")
}
return "", fmt.Errorf("failed to resolve edge environment by token: %w", err)
@@ -99,6 +100,42 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,
return env.ID, nil
}
+// logEdgeTokenResolveMissInternal emits a debug log diagnosing why an agent
+// token failed to resolve to an edge environment. Counts existing edge
+// environments (by access_token presence) so operators can distinguish
+// "no edge envs configured" from "token does not match any configured env".
+// Token contents are never logged in full — only length and a short
+// fingerprint that cannot be reversed.
+func (s *EnvironmentService) logEdgeTokenResolveMissInternal(ctx context.Context, token string) {
+ if s == nil || s.db == nil {
+ return
+ }
+ if !slog.Default().Enabled(ctx, slog.LevelDebug) {
+ return
+ }
+
+ var totalEdgeEnvs int64
+ var edgeEnvsWithToken int64
+ totalEdgeEnvsErr := s.db.WithContext(ctx).Model(&models.Environment{}).Where("is_edge = ?", true).Count(&totalEdgeEnvs).Error
+ edgeEnvsWithTokenErr := s.db.WithContext(ctx).Model(&models.Environment{}).
+ Where("is_edge = ?", true).
+ Where("access_token IS NOT NULL AND access_token != ?", "").
+ Count(&edgeEnvsWithToken).Error
+
+ args := []any{
+ "token_length", len(token),
+ "token_fingerprint", remenv.RedactedTokenFingerprint(token),
+ }
+ if totalEdgeEnvsErr == nil {
+ args = append(args, "edge_envs_total", totalEdgeEnvs)
+ }
+ if edgeEnvsWithTokenErr == nil {
+ args = append(args, "edge_envs_with_access_token", edgeEnvsWithToken)
+ }
+
+ slog.DebugContext(ctx, "Edge agent token did not match any environment", args...)
+}
+
func (s *EnvironmentService) getCachedEnvironmentIDForTokenInternal(token string, now time.Time) (string, bool) {
if s == nil || token == "" {
return "", false
@@ -772,7 +809,12 @@ func (s *EnvironmentService) createEnvironmentEvent(ctx context.Context, envID,
}
func (s *EnvironmentService) RegenerateEnvironmentApiKey(ctx context.Context, envID string, newApiKeyID string, encryptedKey string, userID, username string, envName string) error {
- // Update environment with new API key and set to pending status
+ // Trim once at the boundary so the value persisted, the value cached,
+ // and the value returned by callers (which already TrimSpace before
+ // returning) all stay byte-identical. Any divergence here would surface
+ // as a 401 "invalid agent token" because lookup is direct equality.
+ encryptedKey = strings.TrimSpace(encryptedKey)
+
updates := map[string]any{
"api_key_id": newApiKeyID,
"access_token": encryptedKey,
diff --git a/backend/internal/services/environment_service_test.go b/backend/internal/services/environment_service_test.go
index 46bd69329..e21b3a56b 100644
--- a/backend/internal/services/environment_service_test.go
+++ b/backend/internal/services/environment_service_test.go
@@ -604,6 +604,67 @@ func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_CreatesHiddenChildAn
require.Len(t, apiKeysAfterReuse, 1)
}
+// TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd
+// pins the agent token round-trip: the token returned from the swarm node
+// agent provisioning flow must resolve back to the same environment via
+// ResolveEdgeEnvironmentByToken, and rotation must invalidate the previous
+// token while making the new one resolvable. This is the end-to-end gap that
+// would have caught the v1.18.1 "invalid agent token" bug if any silent
+// transformation (trim, encode, hash) were ever introduced between the
+// command-generation path and the poll-validation path.
+func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd(t *testing.T) {
+ ctx := context.Background()
+ db := setupEnvironmentServiceTestDB(t)
+ userService := NewUserService(db)
+ apiKeyService := NewApiKeyService(db, userService)
+ svc := NewEnvironmentService(db, nil, nil, nil, nil, apiKeyService)
+ user := createTestEnvironmentServiceUser(t, ctx, userService, "swarm-resolve-admin")
+
+ createdEnv, createdToken, err := svc.EnsureSwarmNodeAgentEnvironment(
+ ctx,
+ "manager-env-resolve",
+ "node-resolve-1234567890",
+ "resolve-host",
+ user.ID,
+ user.Username,
+ false,
+ )
+ require.NoError(t, err)
+ require.NotEmpty(t, createdToken)
+
+ resolvedID, err := svc.ResolveEdgeEnvironmentByToken(ctx, createdToken)
+ require.NoError(t, err)
+ require.Equal(t, createdEnv.ID, resolvedID)
+
+ // Whitespace normalization on the wire (e.g. a proxy adding a trailing
+ // newline) must still resolve.
+ resolvedID, err = svc.ResolveEdgeEnvironmentByToken(ctx, " "+createdToken+"\n")
+ require.NoError(t, err)
+ require.Equal(t, createdEnv.ID, resolvedID)
+
+ // Rotate the token: the new token must resolve and the old token must not.
+ rotatedEnv, rotatedToken, err := svc.EnsureSwarmNodeAgentEnvironment(
+ ctx,
+ "manager-env-resolve",
+ "node-resolve-1234567890",
+ "resolve-host",
+ user.ID,
+ user.Username,
+ true,
+ )
+ require.NoError(t, err)
+ require.Equal(t, createdEnv.ID, rotatedEnv.ID)
+ require.NotEqual(t, createdToken, rotatedToken)
+
+ resolvedID, err = svc.ResolveEdgeEnvironmentByToken(ctx, rotatedToken)
+ require.NoError(t, err)
+ require.Equal(t, createdEnv.ID, resolvedID)
+
+ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, createdToken)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "invalid agent token")
+}
+
func TestEnvironmentService_ListMethods_ExcludeHiddenEnvironments(t *testing.T) {
ctx := context.Background()
db := setupEnvironmentServiceTestDB(t)
diff --git a/backend/pkg/libarcane/edge/client_transport_grpc.go b/backend/pkg/libarcane/edge/client_transport_grpc.go
index 5b58ead99..1d0645372 100644
--- a/backend/pkg/libarcane/edge/client_transport_grpc.go
+++ b/backend/pkg/libarcane/edge/client_transport_grpc.go
@@ -65,6 +65,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {
streamCtx, streamCancel := context.WithCancel(metadata.NewOutgoingContext(ctx, metadata.Pairs(
strings.ToLower(HeaderAgentToken), c.cfg.AgentToken,
strings.ToLower(HeaderAPIKey), c.cfg.AgentToken,
+ strings.ToLower(HeaderAuthorization), "Bearer "+c.cfg.AgentToken,
)))
defer streamCancel()
diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.go
index aaef4a507..bc0f5dfc8 100644
--- a/backend/pkg/libarcane/edge/client_transport_poll.go
+++ b/backend/pkg/libarcane/edge/client_transport_poll.go
@@ -181,6 +181,7 @@ func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, pollURL st
req.Header.Set("Content-Type", "application/json")
req.Header.Set(HeaderAgentToken, c.cfg.AgentToken)
req.Header.Set(HeaderAPIKey, c.cfg.AgentToken)
+ req.Header.Set(HeaderAuthorization, "Bearer "+c.cfg.AgentToken)
resp, err := c.httpClient.Do(req)
if err != nil {
diff --git a/backend/pkg/libarcane/edge/client_transport_websocket.go b/backend/pkg/libarcane/edge/client_transport_websocket.go
index 1662309dd..76587a6ce 100644
--- a/backend/pkg/libarcane/edge/client_transport_websocket.go
+++ b/backend/pkg/libarcane/edge/client_transport_websocket.go
@@ -34,6 +34,7 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {
headers := http.Header{}
headers.Set(HeaderAgentToken, c.cfg.AgentToken)
headers.Set(HeaderAPIKey, c.cfg.AgentToken)
+ headers.Set(HeaderAuthorization, "Bearer "+c.cfg.AgentToken)
slog.DebugContext(ctx, "Dialing manager for websocket edge tunnel", "url", managerWSURL)
diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.go
index 0dcb3876f..b9a22b1ca 100644
--- a/backend/pkg/libarcane/edge/poll_control.go
+++ b/backend/pkg/libarcane/edge/poll_control.go
@@ -10,6 +10,7 @@ import (
"sync"
"time"
+ "github.com/getarcaneapp/arcane/backend/pkg/remenv"
"github.com/gin-gonic/gin"
)
@@ -274,19 +275,27 @@ func (s *TunnelServer) HandlePoll(c *gin.Context) {
// In proxy-terminated mTLS deployments, the client certificate is consumed
// by the TLS terminator before this request reaches Arcane. The token is
// still needed as the poll protocol's environment lookup claim.
- token := c.GetHeader(HeaderAgentToken)
- if token == "" {
- token = c.GetHeader(HeaderAPIKey)
- }
+ token, source := tokenFromHeadersWithSourceInternal(c.Request)
if token == "" {
slog.WarnContext(ctx, "Edge poll request without token")
c.JSON(http.StatusUnauthorized, gin.H{"error": "agent token required"})
return
}
+ if source != HeaderAgentToken {
+ slog.DebugContext(ctx, "Edge poll request authenticated via fallback header",
+ "source_header", source,
+ "token_length", len(token),
+ )
+ }
envID, err := s.resolveEnvironment(ctx, token)
if err != nil {
- slog.WarnContext(ctx, "Failed to resolve agent token for edge poll", "error", err)
+ slog.WarnContext(ctx, "Failed to resolve agent token for edge poll",
+ "error", err,
+ "source_header", source,
+ "token_length", len(token),
+ "token_fingerprint", remenv.RedactedTokenFingerprint(token),
+ )
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
return
}
diff --git a/backend/pkg/libarcane/edge/poll_control_test.go b/backend/pkg/libarcane/edge/poll_control_test.go
index 418636abc..a05633b1d 100644
--- a/backend/pkg/libarcane/edge/poll_control_test.go
+++ b/backend/pkg/libarcane/edge/poll_control_test.go
@@ -75,6 +75,107 @@ func TestTunnelServer_HandlePoll(t *testing.T) {
assert.Equal(t, EdgeTransportGRPC, resp.ActiveTransport)
}
+// TestTunnelServer_HandlePoll_AcceptsTokenFromAllSupportedHeaders pins that
+// every header form the agent sends — X-Arcane-Agent-Token, X-API-Key, and
+// Authorization: Bearer — is accepted by the manager. The Authorization
+// fallback specifically guards against reverse proxies (e.g. Cloudflare-style
+// access policies) that strip non-standard X- headers; without it, agents
+// behind such proxies log "invalid agent token" indefinitely.
+func TestTunnelServer_HandlePoll_AcceptsTokenFromAllSupportedHeaders(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ cases := []struct {
+ name string
+ setHeader func(req *http.Request)
+ expectCode int
+ expectError string
+ }{
+ {
+ name: "X-Arcane-Agent-Token",
+ setHeader: func(req *http.Request) {
+ req.Header.Set(HeaderAgentToken, "valid-token")
+ },
+ expectCode: http.StatusOK,
+ },
+ {
+ name: "X-API-Key",
+ setHeader: func(req *http.Request) {
+ req.Header.Set(HeaderAPIKey, "valid-token")
+ },
+ expectCode: http.StatusOK,
+ },
+ {
+ name: "Authorization Bearer",
+ setHeader: func(req *http.Request) {
+ req.Header.Set(HeaderAuthorization, "Bearer valid-token")
+ },
+ expectCode: http.StatusOK,
+ },
+ {
+ name: "Authorization bearer lowercase scheme",
+ setHeader: func(req *http.Request) {
+ req.Header.Set(HeaderAuthorization, "bearer valid-token")
+ },
+ expectCode: http.StatusOK,
+ },
+ {
+ name: "Authorization without Bearer scheme is rejected",
+ setHeader: func(req *http.Request) {
+ req.Header.Set(HeaderAuthorization, "valid-token")
+ },
+ expectCode: http.StatusUnauthorized,
+ expectError: "agent token required",
+ },
+ {
+ name: "Authorization Basic is rejected",
+ setHeader: func(req *http.Request) {
+ req.Header.Set(HeaderAuthorization, "Basic dmFsaWQtdG9rZW4=")
+ },
+ expectCode: http.StatusUnauthorized,
+ expectError: "agent token required",
+ },
+ {
+ name: "no headers is rejected",
+ setHeader: func(req *http.Request) {
+ },
+ expectCode: http.StatusUnauthorized,
+ expectError: "agent token required",
+ },
+ {
+ name: "wrong token is rejected",
+ setHeader: func(req *http.Request) {
+ req.Header.Set(HeaderAuthorization, "Bearer not-the-token")
+ },
+ expectCode: http.StatusUnauthorized,
+ expectError: "invalid agent token",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ server := NewTunnelServerWithRegistry(NewTunnelRegistry(), func(ctx context.Context, token string) (string, error) {
+ if token != "valid-token" {
+ return "", errors.New("invalid token")
+ }
+ return "env-headers", nil
+ }, nil)
+
+ router := gin.New()
+ router.POST("/api/tunnel/poll", server.HandlePoll)
+
+ req := httptest.NewRequest(http.MethodPost, "/api/tunnel/poll", bytes.NewBufferString(`{"transport":"poll"}`))
+ tc.setHeader(req)
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+
+ require.Equal(t, tc.expectCode, rec.Code, "body: %s", rec.Body.String())
+ if tc.expectError != "" {
+ assert.Contains(t, rec.Body.String(), tc.expectError)
+ }
+ })
+ }
+}
+
func TestTunnelServer_HandlePoll_AcceptsTokenAfterProxyTerminatedMTLS(t *testing.T) {
gin.SetMode(gin.TestMode)
diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.go
index eb8f37104..256ebb871 100644
--- a/backend/pkg/libarcane/edge/server.go
+++ b/backend/pkg/libarcane/edge/server.go
@@ -12,6 +12,7 @@ import (
"time"
tunnelpb "github.com/getarcaneapp/arcane/backend/pkg/libarcane/edge/proto/tunnel/v1"
+ "github.com/getarcaneapp/arcane/backend/pkg/remenv"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/gorilla/websocket"
@@ -299,7 +300,7 @@ func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMes
if !ok || envID == "" {
token := strings.TrimSpace(register.GetAgentToken())
if token == "" {
- token = tokenFromMetadata(ctx)
+ token = tokenFromMetadataInternal(ctx)
}
var resolveErr error
@@ -361,7 +362,7 @@ func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (st
return s.resolver(ctx, token)
}
-func tokenFromMetadata(ctx context.Context) string {
+func tokenFromMetadataInternal(ctx context.Context) string {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return ""
@@ -378,19 +379,36 @@ func tokenFromMetadata(ctx context.Context) string {
}
}
}
+ for _, value := range md.Get(strings.ToLower(HeaderAuthorization)) {
+ if token := remenv.ExtractBearerToken(value); token != "" {
+ return token
+ }
+ }
return ""
}
func tokenFromHeadersInternal(req *http.Request) string {
+ token, _ := tokenFromHeadersWithSourceInternal(req)
+ return token
+}
+
+// tokenFromHeadersWithSourceInternal returns the agent token along with the
+// canonical header name that supplied it. Falls back from X-Arcane-Agent-Token
+// to X-API-Key to "Authorization: Bearer <token>" so deployments behind
+// reverse proxies that strip custom X- headers can still authenticate.
+func tokenFromHeadersWithSourceInternal(req *http.Request) (string, string) {
if req == nil {
- return ""
+ return "", ""
}
for _, header := range []string{HeaderAgentToken, HeaderAPIKey} {
if token := strings.TrimSpace(req.Header.Get(header)); token != "" {
- return token
+ return token, header
}
}
- return ""
+ if token := remenv.ExtractBearerToken(req.Header.Get(HeaderAuthorization)); token != "" {
+ return token, HeaderAuthorization
+ }
+ return "", ""
}
func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx context.Context, tunnel *AgentTunnel) {
@@ -585,7 +603,7 @@ func (s *TunnelServer) authStreamInterceptorInternal(ctx context.Context) grpc.S
return handler(srv, ss)
}
- token := tokenFromMetadata(ss.Context())
+ token := tokenFromMetadataInternal(ss.Context())
envID, err := s.resolveEnvironment(ss.Context(), token)
if err != nil {
return status.Error(codes.Unauthenticated, "invalid agent token")
diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.go
index 1c6f74c0d..da25d028c 100644
--- a/backend/pkg/libarcane/edge/server_test.go
+++ b/backend/pkg/libarcane/edge/server_test.go
@@ -526,7 +526,7 @@ func TestTokenFromMetadata(t *testing.T) {
strings.ToLower(HeaderAPIKey), "api-token",
))
- assert.Equal(t, "agent-token", tokenFromMetadata(ctx))
+ assert.Equal(t, "agent-token", tokenFromMetadataInternal(ctx))
})
t.Run("falls back to api key", func(t *testing.T) {
@@ -534,11 +534,11 @@ func TestTokenFromMetadata(t *testing.T) {
strings.ToLower(HeaderAPIKey), "api-token",
))
- assert.Equal(t, "api-token", tokenFromMetadata(ctx))
+ assert.Equal(t, "api-token", tokenFromMetadataInternal(ctx))
})
t.Run("returns empty when metadata missing", func(t *testing.T) {
- assert.Equal(t, "", tokenFromMetadata(context.Background()))
+ assert.Equal(t, "", tokenFromMetadataInternal(context.Background()))
})
}
diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.go
index 4aef3cdc6..99cdd2a3a 100644
--- a/backend/pkg/libarcane/edge/tls.go
+++ b/backend/pkg/libarcane/edge/tls.go
@@ -339,6 +339,7 @@ func enrollAgentMTLSAssetsInternal(ctx context.Context, cfg *Config, assetsDir,
}
req.Header.Set(HeaderAgentToken, cfg.AgentToken)
req.Header.Set(HeaderAPIKey, cfg.AgentToken)
+ req.Header.Set(HeaderAuthorization, "Bearer "+cfg.AgentToken)
resp, err := httpClient.Do(req) //nolint:gosec // intentional request to configured manager endpoint
if err != nil {
diff --git a/backend/pkg/remenv/client.go b/backend/pkg/remenv/client.go
index ec2bb1bcc..7a151e868 100644
--- a/backend/pkg/remenv/client.go
+++ b/backend/pkg/remenv/client.go
@@ -12,10 +12,37 @@ import (
)
const (
- HeaderAPIKey = "X-API-Key" // #nosec G101: header name, not a credential
- HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential
+ HeaderAPIKey = "X-API-Key" // #nosec G101: header name, not a credential
+ HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential
+ HeaderAuthorization = "Authorization"
+ bearerScheme = "Bearer "
)
+// ExtractBearerToken returns the token portion of an "Authorization: Bearer <token>"
+// header value. It is case-insensitive on the scheme and trims surrounding whitespace.
+// Returns an empty string if the header is empty or does not use the Bearer scheme.
+func ExtractBearerToken(headerValue string) string {
+ headerValue = strings.TrimSpace(headerValue)
+ if headerValue == "" {
+ return ""
+ }
+ if len(headerValue) < len(bearerScheme) || !strings.EqualFold(headerValue[:len(bearerScheme)], bearerScheme) {
+ return ""
+ }
+ return strings.TrimSpace(headerValue[len(bearerScheme):])
+}
+
+// RedactedTokenFingerprint returns a short, redacted identifier for a token
+// suitable for debug logs. It is intentionally not reversible and never
+// reveals the full secret.
+func RedactedTokenFingerprint(token string) string {
+ token = strings.TrimSpace(token)
+ if len(token) <= 10 {
+ return "***"
+ }
+ return token[:6] + "..." + token[len(token)-4:]
+}
+
type Request struct {
EnvironmentID string
IsEdge bool
@@ -222,6 +249,14 @@ func (e *DecodeError) Unwrap() error {
return e.Err
}
+// ApplyAgentTokenHeaders sets the agent token in both X- header forms on the
+// given http.Header. It deliberately does NOT set the Authorization header:
+// these helpers are used by the manager when proxying user requests through
+// to remote environments, and the Authorization header in that case carries
+// the calling user's bearer or session token. Direct agent->manager call
+// sites (poll, websocket dial, grpc dial, mTLS enroll) set
+// `Authorization: Bearer <token>` themselves so the token survives reverse
+// proxies that strip non-standard X- headers.
func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {
if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {
return
@@ -231,6 +266,8 @@ func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {
headers.Set(HeaderAPIKey, *accessToken)
}
+// ApplyAgentTokenHeaderMap mirrors ApplyAgentTokenHeaders for map-based
+// header carriers. Same rationale: do not write Authorization here.
func ApplyAgentTokenHeaderMap(headers map[string]string, accessToken *string) {
if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {
return
diff --git a/backend/pkg/remenv/client_test.go b/backend/pkg/remenv/client_test.go
index da1bc2ae3..b7228d0f5 100644
--- a/backend/pkg/remenv/client_test.go
+++ b/backend/pkg/remenv/client_test.go
@@ -27,6 +27,13 @@ func TestApplyAgentTokenHeaders(t *testing.T) {
require.Equal(t, token, headerMap[HeaderAgentToken])
}
+func TestRedactedTokenFingerprint(t *testing.T) {
+ require.Equal(t, "***", RedactedTokenFingerprint(""))
+ require.Equal(t, "***", RedactedTokenFingerprint("short"))
+ require.Equal(t, "***", RedactedTokenFingerprint("1234567890"))
+ require.Equal(t, "123456...cdef", RedactedTokenFingerprint(" 1234567890abcdef "))
+}
+
func TestClientDo_DirectHTTPSuccess(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodPost, r.Method)