Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: agent api token fallbacks and guards (#2568)
Kyle Mendellb37533af358e
diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.goindex 13219cd84..fa439fd2a 100644--- a/backend/internal/services/environment_service.go+++ b/backend/internal/services/environment_service.go@@ -90,6 +90,7 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,Where("access_token = ?", token).First(&env).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {+ s.logEdgeTokenResolveMissInternal(ctx, token)return "", errors.New("invalid agent token")}return "", fmt.Errorf("failed to resolve edge environment by token: %w", err)@@ -99,6 +100,42 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,return env.ID, nil}+// logEdgeTokenResolveMissInternal emits a debug log diagnosing why an agent+// token failed to resolve to an edge environment. Counts existing edge+// environments (by access_token presence) so operators can distinguish+// "no edge envs configured" from "token does not match any configured env".+// Token contents are never logged in full — only length and a short+// fingerprint that cannot be reversed.+func (s *EnvironmentService) logEdgeTokenResolveMissInternal(ctx context.Context, token string) {+ if s == nil || s.db == nil {+ return+ }+ if !slog.Default().Enabled(ctx, slog.LevelDebug) {+ return+ }++ var totalEdgeEnvs int64+ var edgeEnvsWithToken int64+ totalEdgeEnvsErr := s.db.WithContext(ctx).Model(&models.Environment{}).Where("is_edge = ?", true).Count(&totalEdgeEnvs).Error+ edgeEnvsWithTokenErr := s.db.WithContext(ctx).Model(&models.Environment{}).+ Where("is_edge = ?", true).+ Where("access_token IS NOT NULL AND access_token != ?", "").+ Count(&edgeEnvsWithToken).Error++ args := []any{+ "token_length", len(token),+ "token_fingerprint", remenv.RedactedTokenFingerprint(token),+ }+ if totalEdgeEnvsErr == nil {+ args = append(args, "edge_envs_total", totalEdgeEnvs)+ }+ if edgeEnvsWithTokenErr == nil {+ args = append(args, "edge_envs_with_access_token", edgeEnvsWithToken)+ }++ slog.DebugContext(ctx, "Edge agent token did not match any environment", args...)+}+func (s *EnvironmentService) getCachedEnvironmentIDForTokenInternal(token string, now time.Time) (string, bool) {if s == nil || token == "" {return "", false@@ -772,7 +809,12 @@ func (s *EnvironmentService) createEnvironmentEvent(ctx context.Context, envID,}func (s *EnvironmentService) RegenerateEnvironmentApiKey(ctx context.Context, envID string, newApiKeyID string, encryptedKey string, userID, username string, envName string) error {- // Update environment with new API key and set to pending status+ // Trim once at the boundary so the value persisted, the value cached,+ // and the value returned by callers (which already TrimSpace before+ // returning) all stay byte-identical. Any divergence here would surface+ // as a 401 "invalid agent token" because lookup is direct equality.+ encryptedKey = strings.TrimSpace(encryptedKey)+updates := map[string]any{"api_key_id": newApiKeyID,"access_token": encryptedKey,diff --git a/backend/internal/services/environment_service_test.go b/backend/internal/services/environment_service_test.goindex 46bd69329..e21b3a56b 100644--- a/backend/internal/services/environment_service_test.go+++ b/backend/internal/services/environment_service_test.go@@ -604,6 +604,67 @@ func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_CreatesHiddenChildAnrequire.Len(t, apiKeysAfterReuse, 1)}+// TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd+// pins the agent token round-trip: the token returned from the swarm node+// agent provisioning flow must resolve back to the same environment via+// ResolveEdgeEnvironmentByToken, and rotation must invalidate the previous+// token while making the new one resolvable. This is the end-to-end gap that+// would have caught the v1.18.1 "invalid agent token" bug if any silent+// transformation (trim, encode, hash) were ever introduced between the+// command-generation path and the poll-validation path.+func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd(t *testing.T) {+ ctx := context.Background()+ db := setupEnvironmentServiceTestDB(t)+ userService := NewUserService(db)+ apiKeyService := NewApiKeyService(db, userService)+ svc := NewEnvironmentService(db, nil, nil, nil, nil, apiKeyService)+ user := createTestEnvironmentServiceUser(t, ctx, userService, "swarm-resolve-admin")++ createdEnv, createdToken, err := svc.EnsureSwarmNodeAgentEnvironment(+ ctx,+ "manager-env-resolve",+ "node-resolve-1234567890",+ "resolve-host",+ user.ID,+ user.Username,+ false,+ )+ require.NoError(t, err)+ require.NotEmpty(t, createdToken)++ resolvedID, err := svc.ResolveEdgeEnvironmentByToken(ctx, createdToken)+ require.NoError(t, err)+ require.Equal(t, createdEnv.ID, resolvedID)++ // Whitespace normalization on the wire (e.g. a proxy adding a trailing+ // newline) must still resolve.+ resolvedID, err = svc.ResolveEdgeEnvironmentByToken(ctx, " "+createdToken+"\n")+ require.NoError(t, err)+ require.Equal(t, createdEnv.ID, resolvedID)++ // Rotate the token: the new token must resolve and the old token must not.+ rotatedEnv, rotatedToken, err := svc.EnsureSwarmNodeAgentEnvironment(+ ctx,+ "manager-env-resolve",+ "node-resolve-1234567890",+ "resolve-host",+ user.ID,+ user.Username,+ true,+ )+ require.NoError(t, err)+ require.Equal(t, createdEnv.ID, rotatedEnv.ID)+ require.NotEqual(t, createdToken, rotatedToken)++ resolvedID, err = svc.ResolveEdgeEnvironmentByToken(ctx, rotatedToken)+ require.NoError(t, err)+ require.Equal(t, createdEnv.ID, resolvedID)++ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, createdToken)+ require.Error(t, err)+ require.Contains(t, err.Error(), "invalid agent token")+}+func TestEnvironmentService_ListMethods_ExcludeHiddenEnvironments(t *testing.T) {ctx := context.Background()db := setupEnvironmentServiceTestDB(t)diff --git a/backend/pkg/libarcane/edge/client_transport_grpc.go b/backend/pkg/libarcane/edge/client_transport_grpc.goindex 5b58ead99..1d0645372 100644--- a/backend/pkg/libarcane/edge/client_transport_grpc.go+++ b/backend/pkg/libarcane/edge/client_transport_grpc.go@@ -65,6 +65,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {streamCtx, streamCancel := context.WithCancel(metadata.NewOutgoingContext(ctx, metadata.Pairs(strings.ToLower(HeaderAgentToken), c.cfg.AgentToken,strings.ToLower(HeaderAPIKey), c.cfg.AgentToken,+ strings.ToLower(HeaderAuthorization), "Bearer "+c.cfg.AgentToken,)))defer streamCancel()diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.goindex aaef4a507..bc0f5dfc8 100644--- a/backend/pkg/libarcane/edge/client_transport_poll.go+++ b/backend/pkg/libarcane/edge/client_transport_poll.go@@ -181,6 +181,7 @@ func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, pollURL streq.Header.Set("Content-Type", "application/json")req.Header.Set(HeaderAgentToken, c.cfg.AgentToken)req.Header.Set(HeaderAPIKey, c.cfg.AgentToken)+ req.Header.Set(HeaderAuthorization, "Bearer "+c.cfg.AgentToken)resp, err := c.httpClient.Do(req)if err != nil {diff --git a/backend/pkg/libarcane/edge/client_transport_websocket.go b/backend/pkg/libarcane/edge/client_transport_websocket.goindex 1662309dd..76587a6ce 100644--- a/backend/pkg/libarcane/edge/client_transport_websocket.go+++ b/backend/pkg/libarcane/edge/client_transport_websocket.go@@ -34,6 +34,7 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {headers := http.Header{}headers.Set(HeaderAgentToken, c.cfg.AgentToken)headers.Set(HeaderAPIKey, c.cfg.AgentToken)+ headers.Set(HeaderAuthorization, "Bearer "+c.cfg.AgentToken)slog.DebugContext(ctx, "Dialing manager for websocket edge tunnel", "url", managerWSURL)diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.goindex 0dcb3876f..b9a22b1ca 100644--- a/backend/pkg/libarcane/edge/poll_control.go+++ b/backend/pkg/libarcane/edge/poll_control.go@@ -10,6 +10,7 @@ import ("sync""time"+ "github.com/getarcaneapp/arcane/backend/pkg/remenv""github.com/gin-gonic/gin")@@ -274,19 +275,27 @@ func (s *TunnelServer) HandlePoll(c *gin.Context) {// In proxy-terminated mTLS deployments, the client certificate is consumed// by the TLS terminator before this request reaches Arcane. The token is// still needed as the poll protocol's environment lookup claim.- token := c.GetHeader(HeaderAgentToken)- if token == "" {- token = c.GetHeader(HeaderAPIKey)- }+ token, source := tokenFromHeadersWithSourceInternal(c.Request)if token == "" {slog.WarnContext(ctx, "Edge poll request without token")c.JSON(http.StatusUnauthorized, gin.H{"error": "agent token required"})return}+ if source != HeaderAgentToken {+ slog.DebugContext(ctx, "Edge poll request authenticated via fallback header",+ "source_header", source,+ "token_length", len(token),+ )+ }envID, err := s.resolveEnvironment(ctx, token)if err != nil {- slog.WarnContext(ctx, "Failed to resolve agent token for edge poll", "error", err)+ slog.WarnContext(ctx, "Failed to resolve agent token for edge poll",+ "error", err,+ "source_header", source,+ "token_length", len(token),+ "token_fingerprint", remenv.RedactedTokenFingerprint(token),+ )c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})return}diff --git a/backend/pkg/libarcane/edge/poll_control_test.go b/backend/pkg/libarcane/edge/poll_control_test.goindex 418636abc..a05633b1d 100644--- a/backend/pkg/libarcane/edge/poll_control_test.go+++ b/backend/pkg/libarcane/edge/poll_control_test.go@@ -75,6 +75,107 @@ func TestTunnelServer_HandlePoll(t *testing.T) {assert.Equal(t, EdgeTransportGRPC, resp.ActiveTransport)}+// TestTunnelServer_HandlePoll_AcceptsTokenFromAllSupportedHeaders pins that+// every header form the agent sends — X-Arcane-Agent-Token, X-API-Key, and+// Authorization: Bearer — is accepted by the manager. The Authorization+// fallback specifically guards against reverse proxies (e.g. Cloudflare-style+// access policies) that strip non-standard X- headers; without it, agents+// behind such proxies log "invalid agent token" indefinitely.+func TestTunnelServer_HandlePoll_AcceptsTokenFromAllSupportedHeaders(t *testing.T) {+ gin.SetMode(gin.TestMode)++ cases := []struct {+ name string+ setHeader func(req *http.Request)+ expectCode int+ expectError string+ }{+ {+ name: "X-Arcane-Agent-Token",+ setHeader: func(req *http.Request) {+ req.Header.Set(HeaderAgentToken, "valid-token")+ },+ expectCode: http.StatusOK,+ },+ {+ name: "X-API-Key",+ setHeader: func(req *http.Request) {+ req.Header.Set(HeaderAPIKey, "valid-token")+ },+ expectCode: http.StatusOK,+ },+ {+ name: "Authorization Bearer",+ setHeader: func(req *http.Request) {+ req.Header.Set(HeaderAuthorization, "Bearer valid-token")+ },+ expectCode: http.StatusOK,+ },+ {+ name: "Authorization bearer lowercase scheme",+ setHeader: func(req *http.Request) {+ req.Header.Set(HeaderAuthorization, "bearer valid-token")+ },+ expectCode: http.StatusOK,+ },+ {+ name: "Authorization without Bearer scheme is rejected",+ setHeader: func(req *http.Request) {+ req.Header.Set(HeaderAuthorization, "valid-token")+ },+ expectCode: http.StatusUnauthorized,+ expectError: "agent token required",+ },+ {+ name: "Authorization Basic is rejected",+ setHeader: func(req *http.Request) {+ req.Header.Set(HeaderAuthorization, "Basic dmFsaWQtdG9rZW4=")+ },+ expectCode: http.StatusUnauthorized,+ expectError: "agent token required",+ },+ {+ name: "no headers is rejected",+ setHeader: func(req *http.Request) {+ },+ expectCode: http.StatusUnauthorized,+ expectError: "agent token required",+ },+ {+ name: "wrong token is rejected",+ setHeader: func(req *http.Request) {+ req.Header.Set(HeaderAuthorization, "Bearer not-the-token")+ },+ expectCode: http.StatusUnauthorized,+ expectError: "invalid agent token",+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ server := NewTunnelServerWithRegistry(NewTunnelRegistry(), func(ctx context.Context, token string) (string, error) {+ if token != "valid-token" {+ return "", errors.New("invalid token")+ }+ return "env-headers", nil+ }, nil)++ router := gin.New()+ router.POST("/api/tunnel/poll", server.HandlePoll)++ req := httptest.NewRequest(http.MethodPost, "/api/tunnel/poll", bytes.NewBufferString(`{"transport":"poll"}`))+ tc.setHeader(req)+ rec := httptest.NewRecorder()+ router.ServeHTTP(rec, req)++ require.Equal(t, tc.expectCode, rec.Code, "body: %s", rec.Body.String())+ if tc.expectError != "" {+ assert.Contains(t, rec.Body.String(), tc.expectError)+ }+ })+ }+}+func TestTunnelServer_HandlePoll_AcceptsTokenAfterProxyTerminatedMTLS(t *testing.T) {gin.SetMode(gin.TestMode)diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.goindex eb8f37104..256ebb871 100644--- a/backend/pkg/libarcane/edge/server.go+++ b/backend/pkg/libarcane/edge/server.go@@ -12,6 +12,7 @@ import ("time"tunnelpb "github.com/getarcaneapp/arcane/backend/pkg/libarcane/edge/proto/tunnel/v1"+ "github.com/getarcaneapp/arcane/backend/pkg/remenv""github.com/gin-gonic/gin""github.com/google/uuid""github.com/gorilla/websocket"@@ -299,7 +300,7 @@ func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMesif !ok || envID == "" {token := strings.TrimSpace(register.GetAgentToken())if token == "" {- token = tokenFromMetadata(ctx)+ token = tokenFromMetadataInternal(ctx)}var resolveErr error@@ -361,7 +362,7 @@ func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (streturn s.resolver(ctx, token)}-func tokenFromMetadata(ctx context.Context) string {+func tokenFromMetadataInternal(ctx context.Context) string {md, ok := metadata.FromIncomingContext(ctx)if !ok {return ""@@ -378,19 +379,36 @@ func tokenFromMetadata(ctx context.Context) string {}}}+ for _, value := range md.Get(strings.ToLower(HeaderAuthorization)) {+ if token := remenv.ExtractBearerToken(value); token != "" {+ return token+ }+ }return ""}func tokenFromHeadersInternal(req *http.Request) string {+ token, _ := tokenFromHeadersWithSourceInternal(req)+ return token+}++// tokenFromHeadersWithSourceInternal returns the agent token along with the+// canonical header name that supplied it. Falls back from X-Arcane-Agent-Token+// to X-API-Key to "Authorization: Bearer <token>" so deployments behind+// reverse proxies that strip custom X- headers can still authenticate.+func tokenFromHeadersWithSourceInternal(req *http.Request) (string, string) {if req == nil {- return ""+ return "", ""}for _, header := range []string{HeaderAgentToken, HeaderAPIKey} {if token := strings.TrimSpace(req.Header.Get(header)); token != "" {- return token+ return token, header}}- return ""+ if token := remenv.ExtractBearerToken(req.Header.Get(HeaderAuthorization)); token != "" {+ return token, HeaderAuthorization+ }+ return "", ""}func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx context.Context, tunnel *AgentTunnel) {@@ -585,7 +603,7 @@ func (s *TunnelServer) authStreamInterceptorInternal(ctx context.Context) grpc.Sreturn handler(srv, ss)}- token := tokenFromMetadata(ss.Context())+ token := tokenFromMetadataInternal(ss.Context())envID, err := s.resolveEnvironment(ss.Context(), token)if err != nil {return status.Error(codes.Unauthenticated, "invalid agent token")diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.goindex 1c6f74c0d..da25d028c 100644--- a/backend/pkg/libarcane/edge/server_test.go+++ b/backend/pkg/libarcane/edge/server_test.go@@ -526,7 +526,7 @@ func TestTokenFromMetadata(t *testing.T) {strings.ToLower(HeaderAPIKey), "api-token",))- assert.Equal(t, "agent-token", tokenFromMetadata(ctx))+ assert.Equal(t, "agent-token", tokenFromMetadataInternal(ctx))})t.Run("falls back to api key", func(t *testing.T) {@@ -534,11 +534,11 @@ func TestTokenFromMetadata(t *testing.T) {strings.ToLower(HeaderAPIKey), "api-token",))- assert.Equal(t, "api-token", tokenFromMetadata(ctx))+ assert.Equal(t, "api-token", tokenFromMetadataInternal(ctx))})t.Run("returns empty when metadata missing", func(t *testing.T) {- assert.Equal(t, "", tokenFromMetadata(context.Background()))+ assert.Equal(t, "", tokenFromMetadataInternal(context.Background()))})}diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.goindex 4aef3cdc6..99cdd2a3a 100644--- a/backend/pkg/libarcane/edge/tls.go+++ b/backend/pkg/libarcane/edge/tls.go@@ -339,6 +339,7 @@ func enrollAgentMTLSAssetsInternal(ctx context.Context, cfg *Config, assetsDir,}req.Header.Set(HeaderAgentToken, cfg.AgentToken)req.Header.Set(HeaderAPIKey, cfg.AgentToken)+ req.Header.Set(HeaderAuthorization, "Bearer "+cfg.AgentToken)resp, err := httpClient.Do(req) //nolint:gosec // intentional request to configured manager endpointif err != nil {diff --git a/backend/pkg/remenv/client.go b/backend/pkg/remenv/client.goindex ec2bb1bcc..7a151e868 100644--- a/backend/pkg/remenv/client.go+++ b/backend/pkg/remenv/client.go@@ -12,10 +12,37 @@ import ()const (- HeaderAPIKey = "X-API-Key" // #nosec G101: header name, not a credential- HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential+ HeaderAPIKey = "X-API-Key" // #nosec G101: header name, not a credential+ HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential+ HeaderAuthorization = "Authorization"+ bearerScheme = "Bearer ")+// ExtractBearerToken returns the token portion of an "Authorization: Bearer <token>"+// header value. It is case-insensitive on the scheme and trims surrounding whitespace.+// Returns an empty string if the header is empty or does not use the Bearer scheme.+func ExtractBearerToken(headerValue string) string {+ headerValue = strings.TrimSpace(headerValue)+ if headerValue == "" {+ return ""+ }+ if len(headerValue) < len(bearerScheme) || !strings.EqualFold(headerValue[:len(bearerScheme)], bearerScheme) {+ return ""+ }+ return strings.TrimSpace(headerValue[len(bearerScheme):])+}++// RedactedTokenFingerprint returns a short, redacted identifier for a token+// suitable for debug logs. It is intentionally not reversible and never+// reveals the full secret.+func RedactedTokenFingerprint(token string) string {+ token = strings.TrimSpace(token)+ if len(token) <= 10 {+ return "***"+ }+ return token[:6] + "..." + token[len(token)-4:]+}+type Request struct {EnvironmentID stringIsEdge bool@@ -222,6 +249,14 @@ func (e *DecodeError) Unwrap() error {return e.Err}+// ApplyAgentTokenHeaders sets the agent token in both X- header forms on the+// given http.Header. It deliberately does NOT set the Authorization header:+// these helpers are used by the manager when proxying user requests through+// to remote environments, and the Authorization header in that case carries+// the calling user's bearer or session token. Direct agent->manager call+// sites (poll, websocket dial, grpc dial, mTLS enroll) set+// `Authorization: Bearer <token>` themselves so the token survives reverse+// proxies that strip non-standard X- headers.func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {return@@ -231,6 +266,8 @@ func ApplyAgentTokenHeaders(headers http.Header, accessToken *string) {headers.Set(HeaderAPIKey, *accessToken)}+// ApplyAgentTokenHeaderMap mirrors ApplyAgentTokenHeaders for map-based+// header carriers. Same rationale: do not write Authorization here.func ApplyAgentTokenHeaderMap(headers map[string]string, accessToken *string) {if headers == nil || accessToken == nil || strings.TrimSpace(*accessToken) == "" {returndiff --git a/backend/pkg/remenv/client_test.go b/backend/pkg/remenv/client_test.goindex da1bc2ae3..b7228d0f5 100644--- a/backend/pkg/remenv/client_test.go+++ b/backend/pkg/remenv/client_test.go@@ -27,6 +27,13 @@ func TestApplyAgentTokenHeaders(t *testing.T) {require.Equal(t, token, headerMap[HeaderAgentToken])}+func TestRedactedTokenFingerprint(t *testing.T) {+ require.Equal(t, "***", RedactedTokenFingerprint(""))+ require.Equal(t, "***", RedactedTokenFingerprint("short"))+ require.Equal(t, "***", RedactedTokenFingerprint("1234567890"))+ require.Equal(t, "123456...cdef", RedactedTokenFingerprint(" 1234567890abcdef "))+}+func TestClientDo_DirectHTTPSuccess(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {require.Equal(t, http.MethodPost, r.Method)