Skip to content

Modern Docker Management, Designed for Everyone

fix: remove double verification of mTLS certificates (#2505)

Kyle Mendell22b647aab42d
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index d82d7ceb2..3be65feb5 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -4,7 +4,6 @@ import (
"context"
"net/http"
"net/http/httptest"
- "os"
"testing"
"github.com/getarcaneapp/arcane/backend/internal/config"
@@ -159,6 +158,5 @@ func TestPrepareServerTLSInternal_AllowsExternalMTLSTermination(t *testing.T) {
assert.Equal(t, "required", edgeCfg.EdgeMTLSMode)
require.FileExists(t, edgeCfg.EdgeMTLSCAFile)
require.FileExists(t, assetsDir+"/ca.crt")
- _, statErr := os.Stat(assetsDir + "/ca.key")
- require.NoError(t, statErr)
+ require.FileExists(t, assetsDir+"/ca.key")
}
diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.go
index b6e55d7c2..0dcb3876f 100644
--- a/backend/pkg/libarcane/edge/poll_control.go
+++ b/backend/pkg/libarcane/edge/poll_control.go
@@ -266,17 +266,14 @@ func (s *TunnelServer) pollStatusInternal(envID string) TunnelPollResponse {
func (s *TunnelServer) HandlePoll(c *gin.Context) {
ctx := c.Request.Context()
- if err := s.requireClientCertificateInternal(c.Request); err != nil {
- slog.WarnContext(ctx, "Rejected edge poll request without required client certificate", "error", err)
- c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
- return
- }
-
if _, err := decodeTunnelPollRequestInternal(c); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid poll payload"})
return
}
+ // In proxy-terminated mTLS deployments, the client certificate is consumed
+ // by the TLS terminator before this request reaches Arcane. The token is
+ // still needed as the poll protocol's environment lookup claim.
token := c.GetHeader(HeaderAgentToken)
if token == "" {
token = c.GetHeader(HeaderAPIKey)
@@ -293,7 +290,7 @@ func (s *TunnelServer) HandlePoll(c *gin.Context) {
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
return
}
- if err := s.requireCertificateIdentityInternal(c.Request.TLS, envID); err != nil {
+ if err := s.requireRequestCertificateIdentityInternal(c.Request, envID); err != nil {
slog.WarnContext(ctx, "Rejected edge poll request with mismatched client certificate", "environment_id", envID, "error", err)
c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
return
diff --git a/backend/pkg/libarcane/edge/poll_control_test.go b/backend/pkg/libarcane/edge/poll_control_test.go
index 6f7055609..418636abc 100644
--- a/backend/pkg/libarcane/edge/poll_control_test.go
+++ b/backend/pkg/libarcane/edge/poll_control_test.go
@@ -75,6 +75,34 @@ func TestTunnelServer_HandlePoll(t *testing.T) {
assert.Equal(t, EdgeTransportGRPC, resp.ActiveTransport)
}
+func TestTunnelServer_HandlePoll_AcceptsTokenAfterProxyTerminatedMTLS(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ server := NewTunnelServerWithRegistry(NewTunnelRegistry(), func(ctx context.Context, token string) (string, error) {
+ if token != "valid-token" {
+ return "", errors.New("invalid token")
+ }
+ return "env-proxy-mtls", nil
+ }, nil)
+ server.SetConfig(&Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ AppURL: "https://manager.example.com",
+ })
+
+ router := gin.New()
+ router.POST("/api/tunnel/poll", server.HandlePoll)
+
+ req := httptest.NewRequest(http.MethodPost, "/api/tunnel/poll", bytes.NewBufferString(`{"transport":"poll"}`))
+ req.Header.Set(HeaderAgentToken, "valid-token")
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+
+ require.Equal(t, http.StatusOK, rec.Code)
+ var resp TunnelPollResponse
+ require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
+ assert.Equal(t, TunnelStatusIdle, resp.Status)
+}
+
func TestPollRuntimeRegistryGetExpiresStaleState(t *testing.T) {
r := NewPollRuntimeRegistry()
now := time.Now()
diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.go
index 5895686db..eb8f37104 100644
--- a/backend/pkg/libarcane/edge/server.go
+++ b/backend/pkg/libarcane/edge/server.go
@@ -136,12 +136,6 @@ func (s *TunnelServer) HandleConnect(c *gin.Context) {
ctx := c.Request.Context()
callbackCtx := context.WithoutCancel(ctx)
- if err := s.requireClientCertificateInternal(c.Request); err != nil {
- slog.WarnContext(ctx, "Rejected websocket edge tunnel without required client certificate", "error", err)
- c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
- return
- }
-
// Upgrade to WebSocket.
conn, err := tunnelUpgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
@@ -165,9 +159,9 @@ func (s *TunnelServer) HandleConnect(c *gin.Context) {
token := tokenFromHeadersInternal(c.Request)
if token == "" {
// Header auth can be unavailable after the WebSocket upgrade path, so
- // the register message remains a fallback token source. Treat it only
- // as an auth claim; mTLS identity is verified separately below against
- // the environment resolved from this token.
+ // the register message remains a fallback environment lookup claim.
+ // In proxy-terminated mTLS deployments, the client certificate is the
+ // auth credential and is consumed before the request reaches Arcane.
token = strings.TrimSpace(firstMsg.AgentToken)
}
if token == "" {
@@ -184,7 +178,7 @@ func (s *TunnelServer) HandleConnect(c *gin.Context) {
_ = tunnelConn.Close()
return
}
- if err := s.requireCertificateIdentityInternal(c.Request.TLS, envID); err != nil {
+ if err := s.requireRequestCertificateIdentityInternal(c.Request, envID); err != nil {
slog.WarnContext(ctx, "Rejected websocket edge tunnel with mismatched client certificate", "environment_id", envID, "error", err)
_ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Accepted: false, Error: "client certificate does not match environment"})
_ = tunnelConn.Close()
@@ -272,6 +266,8 @@ func (s *TunnelServer) HandleMTLSEnroll(c *gin.Context) {
}
if assets.Reenrolled {
slog.WarnContext(ctx, "Edge mTLS certificate assets re-enrolled", "environment_id", envID, "remote_addr", c.ClientIP(), "cert_issued", assets.CertIssued)
+ } else {
+ slog.InfoContext(ctx, "Edge mTLS certificate assets enrolled", "environment_id", envID, "remote_addr", c.ClientIP(), "cert_issued", assets.CertIssued)
}
if s.enrollmentCallback != nil {
@@ -589,10 +585,6 @@ func (s *TunnelServer) authStreamInterceptorInternal(ctx context.Context) grpc.S
return handler(srv, ss)
}
- if err := s.requireClientCertificateFromContextInternal(ss.Context()); err != nil {
- return status.Error(codes.Unauthenticated, err.Error())
- }
-
token := tokenFromMetadata(ss.Context())
envID, err := s.resolveEnvironment(ss.Context(), token)
if err != nil {
@@ -668,35 +660,18 @@ func resolvedEnvironmentIDFromContextInternal(ctx context.Context) (string, bool
return envID, true
}
-func (s *TunnelServer) requireClientCertificateInternal(req *http.Request) error {
- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) != EdgeMTLSModeRequired {
- return nil
- }
- if req != nil && req.TLS != nil && hasVerifiedPeerCertificateInternal(req.TLS) {
- return nil
- }
- return errors.New("verified client certificate required")
-}
-
-func (s *TunnelServer) requireClientCertificateFromContextInternal(ctx context.Context) error {
- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) != EdgeMTLSModeRequired {
+func (s *TunnelServer) requireCertificateIdentityInternal(state *tls.ConnectionState, envID string) error {
+ if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeDisabled {
return nil
}
-
- p, ok := peer.FromContext(ctx)
- if ok {
- if tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo); ok && hasVerifiedPeerCertificateInternal(&tlsInfo.State) {
- return nil
- }
- }
- return errors.New("verified client certificate required")
+ return verifiedPeerCertificateEnvironmentIDMatchesInternal(state, envID, edgeMTLSTrustDomainInternal(s.cfg))
}
-func (s *TunnelServer) requireCertificateIdentityInternal(state *tls.ConnectionState, envID string) error {
- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeDisabled {
+func (s *TunnelServer) requireRequestCertificateIdentityInternal(req *http.Request, envID string) error {
+ if req == nil || req.TLS == nil || !hasVerifiedPeerCertificateInternal(req.TLS) {
return nil
}
- return verifiedPeerCertificateEnvironmentIDMatchesInternal(state, envID, edgeMTLSTrustDomainInternal(s.cfg))
+ return s.requireCertificateIdentityInternal(req.TLS, envID)
}
func (s *TunnelServer) requireCertificateIdentityFromContextInternal(ctx context.Context, envID string) error {
diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.go
index 1daf9bce9..1c6f74c0d 100644
--- a/backend/pkg/libarcane/edge/server_test.go
+++ b/backend/pkg/libarcane/edge/server_test.go
@@ -170,6 +170,55 @@ func TestTunnelServer_HandleConnect(t *testing.T) {
reg.Unregister("env-connected")
}
+func TestTunnelServer_HandleConnect_AcceptsTokenAfterProxyTerminatedMTLS(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+
+ server := NewTunnelServer(func(ctx context.Context, token string) (string, error) {
+ if token == "valid-token" {
+ return "env-proxy-mtls", nil
+ }
+ return "", errors.New("invalid token")
+ }, nil)
+ server.SetConfig(&Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ AppURL: "https://manager.example.com",
+ })
+
+ router := gin.New()
+ router.GET("/connect", server.HandleConnect)
+
+ ts := httptest.NewServer(router)
+ defer ts.Close()
+
+ url := "ws" + strings.TrimPrefix(ts.URL, "http") + "/connect"
+ headers := http.Header{}
+ headers.Set(HeaderAgentToken, "valid-token")
+
+ conn, resp, err := websocket.DefaultDialer.Dial(url, headers)
+ require.NoError(t, err)
+ if resp != nil {
+ defer func() { _ = resp.Body.Close() }()
+ }
+ defer func() { _ = conn.Close() }()
+ require.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode)
+
+ err = conn.WriteJSON(&TunnelMessage{
+ Type: MessageTypeRegister,
+ AgentToken: "valid-token",
+ AgentInstance: "agent-proxy-mtls",
+ })
+ require.NoError(t, err)
+
+ var registerResp TunnelMessage
+ err = conn.ReadJSON(&registerResp)
+ require.NoError(t, err)
+ require.Equal(t, MessageTypeRegisterResponse, registerResp.Type)
+ require.True(t, registerResp.Accepted)
+ require.Equal(t, "token", registerResp.SecurityMode)
+
+ GetRegistry().Unregister("env-proxy-mtls")
+}
+
func TestTunnelServer_HandleConnect_InvalidToken(t *testing.T) {
gin.SetMode(gin.TestMode)
diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.go
index 2984e77ae..4aef3cdc6 100644
--- a/backend/pkg/libarcane/edge/tls.go
+++ b/backend/pkg/libarcane/edge/tls.go
@@ -88,11 +88,12 @@ func BuildManagerServerTLSConfig(cfg *Config) (*tls.Config, error) {
}
// ClientAuth is intentionally VerifyClientCertIfGiven even when
- // EdgeMTLSMode is "required". Enforcement of "required" mode is done
- // per-request at the application layer (see TunnelServer.requireClientCertificate*
- // in server.go) so that the mTLS enrollment endpoint, which agents must
- // reach before they own a client certificate, remains accessible. If the
- // handshake were set to RequireAndVerifyClientCert, bootstrap would fail.
+ // EdgeMTLSMode is "required". Enforcement of certificate identity is done
+ // per-request at the application layer so that the mTLS enrollment endpoint,
+ // which agents must reach before they own a client certificate, remains
+ // accessible. In proxy-terminated deployments the TLS state is not visible
+ // here; identity is enforced by the upstream proxy.
+ // If the handshake were set to RequireAndVerifyClientCert, bootstrap would fail.
// TODO: reload ClientCAs when CA rotation support is added.
return &tls.Config{
MinVersion: tls.VersionTLS12,
@@ -465,11 +466,14 @@ func ValidateManagerMTLSConfig(cfg *Config) error {
}
if strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {
- return fmt.Errorf("EDGE_MTLS_MODE=%s requires EDGE_MTLS_CA_FILE on the manager", mode)
+ return nil
}
- _, err := BuildManagerServerTLSConfig(cfg)
- return err
+ _, err := loadCertPoolInternal(cfg.EdgeMTLSCAFile)
+ if err != nil {
+ return fmt.Errorf("failed to load edge mTLS CA file: %w", err)
+ }
+ return nil
}
func loadCertPoolInternal(caFile string) (*x509.CertPool, error) {
@@ -582,17 +586,19 @@ func setAgentMTLSAssetPathsInternal(cfg *Config, assetsDir string) {
}
func requestSecurityModeInternal(req *http.Request) string {
- if req == nil || req.TLS == nil {
- return "token"
- }
-
- if hasVerifiedPeerCertificateInternal(req.TLS) {
+ if hasVerifiedEdgeMTLSRequestInternal(req) {
return "mtls"
}
-
return "token"
}
+func hasVerifiedEdgeMTLSRequestInternal(req *http.Request) bool {
+ if req == nil {
+ return false
+ }
+ return req.TLS != nil && hasVerifiedPeerCertificateInternal(req.TLS)
+}
+
func grpcContextSecurityModeInternal(pctx peer.Peer) string {
if tlsInfo, ok := pctx.AuthInfo.(credentials.TLSInfo); ok && hasVerifiedPeerCertificateInternal(&tlsInfo.State) {
return "mtls"
diff --git a/backend/pkg/libarcane/edge/tls_test.go b/backend/pkg/libarcane/edge/tls_test.go
index 08728fbd7..babe5ee41 100644
--- a/backend/pkg/libarcane/edge/tls_test.go
+++ b/backend/pkg/libarcane/edge/tls_test.go
@@ -281,6 +281,10 @@ func TestTunnelServerRequireCertificateIdentityInternal_RejectsWrongEnvironmentU
}
func TestValidateManagerMTLSConfig_DoesNotRequireArcaneTLSTermination(t *testing.T) {
+ require.NoError(t, ValidateManagerMTLSConfig(&Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ }))
+
assetsDir := t.TempDir()
caPath, _, _, err := ensureManagerCAInternal(context.Background(), assetsDir)
require.NoError(t, err)
@@ -315,38 +319,88 @@ func TestValidateManagerMTLSConfig_RejectsMissingOrMalformedCAFile(t *testing.T)
})
}
-func TestTunnelServerRequiredMTLS_RejectsRequestWithoutTLSState(t *testing.T) {
+func TestTunnelServerRequiredMTLS_AllowsHTTPRequestsWithoutVisibleTLSState(t *testing.T) {
+ server := NewTunnelServer(nil, nil)
+ server.SetConfig(&Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ AppURL: "https://manager.example.com",
+ })
+
+ req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", nil)
+ require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))
+}
+
+func TestTunnelServerRequiredMTLS_DetectsOnlyDirectTLSForRequestSecurityMode(t *testing.T) {
server := NewTunnelServer(nil, nil)
server.SetConfig(&Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
AppURL: "https://manager.example.com",
})
+ cert := &x509.Certificate{}
req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", nil)
- require.ErrorContains(t, server.requireClientCertificateInternal(req), "verified client certificate required")
- require.NoError(t, server.requireCertificateIdentityInternal(nil, "env-a"))
+ req.TLS = &tls.ConnectionState{
+ PeerCertificates: []*x509.Certificate{cert},
+ VerifiedChains: [][]*x509.Certificate{{cert}},
+ }
+ require.Equal(t, "mtls", requestSecurityModeInternal(req))
+
+ req = httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", nil)
+ req.Header.Set("X-SSL-Client-Verify", "SUCCESS")
+ require.Equal(t, "token", requestSecurityModeInternal(req))
+ require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))
+}
+
+func TestTunnelServerRequiredMTLS_IgnoresProxyVerificationHeaders(t *testing.T) {
+ tests := []struct {
+ name string
+ value string
+ }{
+ {name: "empty", value: ""},
+ {name: "success", value: "SUCCESS"},
+ {name: "true", value: "true"},
+ {name: "none", value: "NONE"},
+ {name: "failed", value: "FAILED"},
+ {name: "unknown", value: "probably"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", nil)
+ req.Header.Set("X-SSL-Client-Verify", tt.value)
+
+ require.Equal(t, "token", requestSecurityModeInternal(req))
+ })
+ }
}
-func TestTunnelServerRequiredMTLS_RejectsGRPCContextsWithoutVerifiedPeerCertificate(t *testing.T) {
+func TestTunnelServerRequiredMTLS_AllowsGRPCContextsWithoutVisibleTLSState(t *testing.T) {
server := NewTunnelServer(nil, nil)
server.SetConfig(&Config{EdgeMTLSMode: EdgeMTLSModeRequired})
t.Run("missing peer", func(t *testing.T) {
- require.ErrorContains(t, server.requireClientCertificateFromContextInternal(context.Background()), "verified client certificate required")
+ require.NoError(t, server.requireCertificateIdentityFromContextInternal(context.Background(), "env-a"))
})
t.Run("non tls peer", func(t *testing.T) {
ctx := peer.NewContext(context.Background(), &peer.Peer{AuthInfo: testAuthInfo{}})
- require.ErrorContains(t, server.requireClientCertificateFromContextInternal(ctx), "verified client certificate required")
+ require.NoError(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"))
})
- t.Run("verified tls peer", func(t *testing.T) {
- cert := &x509.Certificate{}
+ t.Run("verified tls peer checks identity", func(t *testing.T) {
+ uriSAN, err := url.Parse("spiffe://manager.example.com/edge/env-a")
+ require.NoError(t, err)
+ cert := &x509.Certificate{URIs: []*url.URL{uriSAN}}
ctx := peer.NewContext(context.Background(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{
PeerCertificates: []*x509.Certificate{cert},
VerifiedChains: [][]*x509.Certificate{{cert}},
}}})
- require.NoError(t, server.requireClientCertificateFromContextInternal(ctx))
+ server.SetConfig(&Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ AppURL: "https://manager.example.com",
+ })
+ require.NoError(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"))
+ require.Error(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-b"))
})
}