Skip to content

Modern Docker Management, Designed for Everyone

perf: stream edge responses, pool proxy connections, and clean up edge package (#4354)

Kyle Mendelldf8049171e4e
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index e8b129086..295704c20 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -66,6 +66,7 @@ func Bootstrap(ctx context.Context) error {
slog.SetDefault(slog.New(logs.NewSlogHandler(slog.Default().Handler(), ws.LogBroadcaster())))
database.SetGormLogger(BuildGormLogger(cfg))
slog.InfoContext(ctx, "Arcane is starting...", "version", config.Version)
+ startup.ApplyMemoryLimit(ctx)
slog.InfoContext(ctx, "Arcane Identity Configuration", "puid", os.Getuid(), "pgid", os.Getgid())
appCtx, cancelApp := context.WithCancel(ctx)
diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.go
index 5b2e61687..0b4952f81 100644
--- a/backend/internal/bootstrap/edge_bootstrap.go
+++ b/backend/internal/bootstrap/edge_bootstrap.go
@@ -50,7 +50,7 @@ func registerEdgeTunnelRoutes(
}
server := edge.NewTunnelServerWithRegistry(registry, resolver, statusCallback)
- server.SetConfig(&edge.Config{
+ server.Config = &edge.Config{
EdgeMTLSMode: cfg.EdgeMTLSMode,
EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,
EdgeMTLSCertFile: cfg.EdgeMTLSCertFile,
@@ -59,8 +59,8 @@ func registerEdgeTunnelRoutes(
EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,
AppURL: cfg.GetAppURL(),
ManagerApiUrl: cfg.ManagerApiUrl,
- })
- server.SetEnvironmentNameResolver(func(ctx context.Context, envID string) (string, error) {
+ }
+ server.NameResolver = func(ctx context.Context, envID string) (string, error) {
env, err := environmentService.GetEnvironmentByID(ctx, envID)
if err != nil {
return "", err
@@ -69,9 +69,9 @@ func registerEdgeTunnelRoutes(
return "", nil
}
return env.Name, nil
- })
- server.SetEventCallback(eventCallback)
- server.SetEnrollmentCallback(func(ctx context.Context, envID, remoteAddr string, certIssued, caGenerated, reenrolled bool) {
+ }
+ server.EventCallback = eventCallback
+ server.EnrollmentCallback = func(ctx context.Context, envID, remoteAddr string, certIssued, caGenerated, reenrolled bool) {
if eventService == nil {
return
}
@@ -93,7 +93,7 @@ func registerEdgeTunnelRoutes(
Metadata: database.JSON{"remoteAddr": remoteAddr, "reenrollment": reenrolled},
})
createEdgeMTLSIssueEventsInternal(ctx, eventService, envIDCopy, envNameCopy, remoteAddr, certIssued, caGenerated, reenrolled)
- })
+ }
var stopCleanup func(context.Context) error
lifecycle.Append(fx.Hook{
OnStart: func(context.Context) error {
diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.go
index 10d390bbe..5d4ab342a 100644
--- a/backend/internal/bootstrap/server_bootstrap.go
+++ b/backend/internal/bootstrap/server_bootstrap.go
@@ -47,7 +47,7 @@ func NewHTTPServer(lc fx.Lifecycle, p HTTPServerParams) (*http.Server, error) {
return nil, err
}
if p.TunnelServer != nil {
- p.TunnelServer.SetConfig(edgeCfg)
+ p.TunnelServer.Config = edgeCfg
}
httpHandler, grpcServer := configureTunnelServerInternal(p.AppCtx, p.Config, p.Router, p.TunnelServer, listenAddr)
diff --git a/backend/internal/build/service.go b/backend/internal/build/service.go
index bfd43cb92..d1a358885 100644
--- a/backend/internal/build/service.go
+++ b/backend/internal/build/service.go
@@ -390,7 +390,8 @@ func (s *BuildService) cloneGitContextInternal(ctx context.Context, repositoryUR
}
if s.gitRepository != nil && s.gitRepository.Client != nil {
- return s.gitRepository.Clone(ctx, repositoryURL, ref, authConfig)
+ // Builds may run git describe or count commits, so they keep full history.
+ return s.gitRepository.Clone(ctx, repositoryURL, ref, authConfig, 0)
}
return "", errors.New("git repository service not available")
diff --git a/backend/internal/gitops/children/sync/service.go b/backend/internal/gitops/children/sync/service.go
index ee25a239a..17a5593d6 100644
--- a/backend/internal/gitops/children/sync/service.go
+++ b/backend/internal/gitops/children/sync/service.go
@@ -107,31 +107,42 @@ func (s *Service) DirectoryProject(ctx context.Context, sync *projectpkg.GitOpsS
// ReconcileRevision confirms an interrupted run when the record already holds
// the source revision that run pinned.
func ReconcileRevision(ctx context.Context, previous scheduler.Run, record *projectpkg.GitOpsSync, outcome scheduler.Outcome) (scheduler.Outcome, error) {
- for _, target := range previous.Outcome.Targets {
- if target.ID != record.ID || len(target.RecoveryData) == 0 {
- continue
- }
- var revision syncRevisionInternal
- if unmarshalErr := json.Unmarshal(target.RecoveryData, &revision); unmarshalErr != nil {
- return outcome, unmarshalErr
- }
- if revision.Commit != "" && record.LastSyncCommit != nil && *record.LastSyncCommit == revision.Commit && record.LastSyncStatus != nil && *record.LastSyncStatus == "success" {
- target.Status = scheduler.Succeeded
- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
- return outcome, progressErr
- }
- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
- }
+ target, commit, err := recordedRevision(previous, record.ID)
+ if err != nil {
+ return outcome, err
+ }
+ if commit == "" || kit.FromPtr(record.LastSyncCommit) != commit || kit.FromPtr(record.LastSyncStatus) != "success" {
+ return outcome, nil
}
- return outcome, nil
+ target.Status = scheduler.Succeeded
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return outcome, progressErr
+ }
+ return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil
}
-// preparedSyncSource captures the repository data needed by the sync execution
-// paths after the source repository has been cloned and validated.
-type syncRevisionInternal struct {
+// recordedRevision returns the target and source revision an earlier run recorded for syncID.
+func recordedRevision(run scheduler.Run, syncID string) (scheduler.TargetOutcome, string, error) {
+ idx := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {
+ return target.ID == syncID && len(target.RecoveryData) > 0
+ })
+ if idx < 0 {
+ return scheduler.TargetOutcome{}, "", nil
+ }
+ var revision syncRevision
+ if err := json.Unmarshal(run.Outcome.Targets[idx].RecoveryData, &revision); err != nil {
+ return scheduler.TargetOutcome{}, "", err
+ }
+ return run.Outcome.Targets[idx], revision.Commit, nil
+}
+
+// syncRevision is the recovery data a sync run records so a retry replays the same source revision.
+type syncRevision struct {
Commit string `json:"commit"`
}
+// preparedSyncSource captures the repository data needed by the sync execution
+// paths after the source repository has been cloned and validated.
type preparedSyncSource struct {
repoPath string
commitHash string
@@ -177,35 +188,56 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOps
return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to get authentication config", err.Error())
}
- repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig)
+ // A retry replays the revision its interrupted predecessor recorded, which needs full history.
+ pinnedCommit := ""
+ if previous, ok := jobcontext.Run(ctx); ok {
+ if _, pinnedCommit, err = recordedRevision(previous, sync.ID); err != nil {
+ return nil, err
+ }
+ }
+ repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig, kit.Ternary(pinnedCommit == "", 1, 0))
if err != nil {
return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to clone repository", err.Error())
}
+ source := &preparedSyncSource{repoPath: repoPath}
- commitHash, err := s.repoService.GetCurrentCommit(ctx, repoPath)
+ source.commitHash, err = s.repoService.GetCurrentCommit(ctx, repoPath)
if err != nil {
slog.WarnContext(ctx, "Failed to get commit hash", "error", err)
- commitHash = ""
}
-
- commitHash, err = pinSyncRevisionInternal(ctx, sync.ID, repoPath, commitHash)
+ if pinnedCommit != "" && pinnedCommit != source.commitHash {
+ repo, openErr := git.PlainOpen(repoPath)
+ if openErr != nil {
+ return source, openErr
+ }
+ tree, treeErr := repo.Worktree()
+ if treeErr != nil {
+ return source, treeErr
+ }
+ if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(pinnedCommit)}); checkoutErr != nil {
+ return source, checkoutErr
+ }
+ source.commitHash = pinnedCommit
+ }
+ if source.commitHash == "" {
+ return source, errors.New("GitOps source revision is unavailable")
+ }
+ revision, err := json.Marshal(syncRevision{Commit: source.commitHash})
if err != nil {
- return &preparedSyncSource{repoPath: repoPath}, err
+ return source, err
+ }
+ progress := scheduler.TargetOutcome{ResourceType: "gitops_sync", ID: sync.ID, Status: scheduler.Running, RecoveryData: revision}
+ if progressErr := jobcontext.Progress(ctx, progress); progressErr != nil {
+ return source, progressErr
}
+
if !s.repoService.FileExists(ctx, repoPath, sync.ComposePath) {
errMsg := "compose file not found: " + sync.ComposePath
- return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, "Compose file not found at "+sync.ComposePath, errMsg)
+ return source, s.failSync(ctx, sync.ID, result, sync, actor, "Compose file not found at "+sync.ComposePath, errMsg)
}
-
- composeContent, err := s.repoService.ReadFile(ctx, repoPath, sync.ComposePath)
+ source.composeContent, err = s.repoService.ReadFile(ctx, repoPath, sync.ComposePath)
if err != nil {
- return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())
- }
-
- source := &preparedSyncSource{
- repoPath: repoPath,
- commitHash: commitHash,
- composeContent: composeContent,
+ return source, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())
}
envPath := filepath.Join(filepath.Dir(sync.ComposePath), ".env")
@@ -248,45 +280,6 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOps
return source, nil
}
-func pinSyncRevisionInternal(ctx context.Context, syncID, repoPath, commitHash string) (string, error) {
- if previous, ok := jobcontext.Run(ctx); ok {
- for _, target := range previous.Outcome.Targets {
- if target.ID != syncID || len(target.RecoveryData) == 0 {
- continue
- }
- var revision syncRevisionInternal
- if err := json.Unmarshal(target.RecoveryData, &revision); err != nil {
- return "", err
- }
- if revision.Commit != "" && revision.Commit != commitHash {
- repository, err := git.PlainOpen(repoPath)
- if err != nil {
- return "", err
- }
- tree, err := repository.Worktree()
- if err != nil {
- return "", err
- }
- if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(revision.Commit)}); checkoutErr != nil {
- return "", checkoutErr
- }
- commitHash = revision.Commit
- }
- }
- }
- revision, err := json.Marshal(syncRevisionInternal{Commit: commitHash})
- if err != nil {
- return "", err
- }
- if commitHash == "" {
- return "", errors.New("GitOps source revision is unavailable")
- }
- if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "gitops_sync", ID: syncID, Status: scheduler.Running, RecoveryData: revision}); progressErr != nil {
- return "", progressErr
- }
- return commitHash, nil
-}
-
// performDirectorySync runs the directory-sync path and only triggers a
// redeploy when an already running project's synced contents changed.
func (
diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.go
index 62e549939..a30501164 100644
--- a/backend/internal/gitops/service.go
+++ b/backend/internal/gitops/service.go
@@ -1171,7 +1171,7 @@ func (s *GitOpsSyncService) BrowseFiles(ctx context.Context, environmentID, id,
}
// Clone the repository
- repoPath, err := s.repoService.Clone(browseCtx, repository.URL, syncRecord.Branch, authConfig)
+ repoPath, err := s.repoService.Clone(browseCtx, repository.URL, syncRecord.Branch, authConfig, 1)
if err != nil {
return nil, fmt.Errorf("failed to clone repository: %w", err)
}
diff --git a/backend/internal/gitops/service_test.go b/backend/internal/gitops/service_test.go
index c54c78ed8..341ca4e2a 100644
--- a/backend/internal/gitops/service_test.go
+++ b/backend/internal/gitops/service_test.go
@@ -1197,7 +1197,7 @@ func (e *backupTestEnvInternal) reloadInternal(t *testing.T, id string) *project
// checkoutRemoteInternal clones the backup branch so the pushed tree can be inspected.
func (e *backupTestEnvInternal) checkoutRemoteInternal(t *testing.T) string {
t.Helper()
- repoPath, err := e.remote.Clone(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"})
+ repoPath, err := e.remote.Clone(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"}, 0)
require.NoError(t, err)
t.Cleanup(func() { _ = e.remote.Cleanup(repoPath) })
return repoPath
diff --git a/backend/internal/gitrepo/service.go b/backend/internal/gitrepo/service.go
index 57bcd9aab..926a17906 100644
--- a/backend/internal/gitrepo/service.go
+++ b/backend/internal/gitrepo/service.go
@@ -540,7 +540,7 @@ func (s *GitRepositoryService) BrowseFiles(ctx context.Context, id, branch, path
}
// Clone the repository
- repoPath, err := s.Clone(ctx, repository.URL, branch, authConfig)
+ repoPath, err := s.Clone(ctx, repository.URL, branch, authConfig, 1)
if err != nil {
return nil, fmt.Errorf("failed to clone repository: %w", err)
}
diff --git a/backend/internal/image/children/attestations/service.go b/backend/internal/image/children/attestations/service.go
index 487b7a06a..aab316a77 100644
--- a/backend/internal/image/children/attestations/service.go
+++ b/backend/internal/image/children/attestations/service.go
@@ -12,6 +12,7 @@ import (
"log/slog"
"net/http"
"strings"
+ "sync"
"github.com/containerd/platforms"
"github.com/getarcaneapp/arcane/types/v2/image"
@@ -28,6 +29,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth"
)
+// zstdDecoder is shared because DecodeAll is safe for concurrent use and a decoder is costly to build.
+var zstdDecoder = sync.OnceValues(func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })
+
// Service reads in-toto attestations attached to local images and registry references.
type Service struct {
dockerClient func(ctx context.Context) (*client.Client, error)
@@ -502,12 +506,11 @@ func decompressAttestationStatementInternal(data []byte) ([]byte, error) {
defer func() { _ = reader.Close() }()
return io.ReadAll(reader)
case len(data) >= 4 && data[0] == 0x28 && data[1] == 0xb5 && data[2] == 0x2f && data[3] == 0xfd:
- reader, err := zstd.NewReader(nil)
+ decoder, err := zstdDecoder()
if err != nil {
return nil, err
}
- defer reader.Close()
- return reader.DecodeAll(data, nil)
+ return decoder.DecodeAll(data, nil)
default:
return data, nil
}
diff --git a/backend/internal/middleware/environment.go b/backend/internal/middleware/environment.go
index b7b200a94..d24466ae3 100644
--- a/backend/internal/middleware/environment.go
+++ b/backend/internal/middleware/environment.go
@@ -19,6 +19,7 @@ import (
"github.com/getarcaneapp/arcane/types/v2/container"
"github.com/getarcaneapp/arcane/types/v2/gitops"
+ httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
"github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/labstack/echo/v5"
@@ -107,7 +108,7 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(
paramName: paramName,
resolver: resolver,
authValidator: authValidator,
- httpClient: &http.Client{Timeout: proxyTimeout},
+ httpClient: httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second}),
registry: registry,
matcher: matcher,
checkOrigin: checkOrigin,
@@ -774,8 +775,7 @@ func (m *EnvironmentMiddleware) createProxyRequest(c *echo.Context, target strin
ContentLength: contentLength,
}).WithContext(srcReq.Context())
- skip := edge.GetSkipHeaders()
- edge.CopyRequestHeaders(srcReq.Header, req.Header, skip)
+ edge.CopyRequestHeaders(srcReq.Header, req.Header)
edge.SetAuthHeader(req, c)
edge.SetAgentToken(req, accessToken)
edge.SetForwardedHeaders(req, c.RealIP(), srcReq.Host)
diff --git a/backend/internal/project/children/sync/service.go b/backend/internal/project/children/sync/service.go
index a60e5bb8f..6365c9e85 100644
--- a/backend/internal/project/children/sync/service.go
+++ b/backend/internal/project/children/sync/service.go
@@ -243,19 +243,37 @@ func ApplyGitSyncEnv(ctx context.Context, projectPath, projectsDirectory string,
return update.State.DirectContent, kit.FromPtr(update.EffectiveContent), nil
}
-// LoadComposeMetadata loads a discovered project's compose file once and
-// returns the service count plus compose-go's effective project name.
-func LoadComposeMetadata(ctx context.Context, dirPath, dirName, projectsDirectory string, autoInjectEnv bool, pathMapper *projects.PathMapper) (project.ComposeIdentity, error) {
+// LoadComposeMetadata returns a discovered project's service count plus
+// compose-go's effective project name. Results are cached by dirPath until a
+// compose, include or env file changes.
+func LoadComposeMetadata(
+ ctx context.Context,
+ cache project.ComposeCache[project.ComposeIdentity],
+ dirPath, dirName, projectsDirectory string,
+ autoInjectEnv bool,
+ pathMapper *projects.PathMapper,
+) (project.ComposeIdentity, error) {
normName := projects.NormalizeProjectName(dirName)
meta := project.ComposeIdentity{
ResolvedProjectName: normName,
}
+ composeFile, err := projects.DetectComposeFile(ctx, projectsDirectory, dirPath)
+ if err != nil {
+ return meta, err
+ }
+ fingerprint := fmt.Sprintf("%q|%q|%q|%t|%#v", composeFile, normName, projectsDirectory, autoInjectEnv, pathMapper)
+ if cached, ok := cache.Get(dirPath, fingerprint); ok {
+ return cached, nil
+ }
+
// Load unnamed first so COMPOSE_PROJECT_NAME from .env wins; fall back to
// the normalized directory name when that fails.
- proj, _, err := projects.LoadComposeProjectFromDir(ctx, dirPath, "", projectsDirectory, autoInjectEnv, pathMapper)
+ dependencies := project.ComposeDependencies{}
+ proj, err := projects.LoadComposeProject(ctx, composeFile, "", projectsDirectory, autoInjectEnv, pathMapper, nil, nil, false, &dependencies, nil, nil)
if err != nil {
- proj, _, err = projects.LoadComposeProjectFromDir(ctx, dirPath, normName, projectsDirectory, autoInjectEnv, pathMapper)
+ dependencies = project.ComposeDependencies{}
+ proj, err = projects.LoadComposeProject(ctx, composeFile, normName, projectsDirectory, autoInjectEnv, pathMapper, nil, nil, false, &dependencies, nil, nil)
if err != nil {
return meta, err
}
@@ -273,5 +291,10 @@ func LoadComposeMetadata(ctx context.Context, dirPath, dirName, projectsDirector
meta.ComposeProjectName = new(proj.Name)
}
+ if composeFiles, envFiles, ok := projects.ComposeCacheDependencies(ctx, proj, dependencies, composeFile, projectsDirectory, autoInjectEnv); ok {
+ if setErr := cache.Set(dirPath, fingerprint, dirPath, projectsDirectory, composeFile, composeFiles, envFiles, meta); setErr != nil {
+ slog.DebugContext(ctx, "failed to cache compose metadata", "path", dirPath, "error", setErr)
+ }
+ }
return meta, nil
}
diff --git a/backend/internal/project/service.go b/backend/internal/project/service.go
index da4753576..7efcf7816 100644
--- a/backend/internal/project/service.go
+++ b/backend/internal/project/service.go
@@ -115,6 +115,9 @@ type ProjectService struct {
// projects, a gitops_syncs lookup — per project, on every list request.
// Entries are validated by compose/include/env file mtimes rather than a TTL.
metaCache projecttypes.ComposeCache[projects.ArcaneComposeMetadata]
+ // composeIdentities caches filesystem-sync compose identities by project path,
+ // so a sync only re-parses projects whose compose or env files changed.
+ composeIdentities projecttypes.ComposeCache[projecttypes.ComposeIdentity]
// FilesChanged fires with the project ID after project files are saved
// through Arcane, so Git backups can react without polling.
@@ -313,6 +316,7 @@ func NewProjectService(
config: cfg,
parsedCompose: projects.NewParsedComposeCache(),
metaCache: projects.NewComposeCache[projects.ArcaneComposeMetadata](1024, nil),
+ composeIdentities: projects.NewComposeCache[projecttypes.ComposeIdentity](2048, nil),
FilesChanged: concurrency.NewSignal[string](),
}
s.initChildren(kvService, buildService)
@@ -719,7 +723,7 @@ func (s *ProjectService) refreshComposeProjectName(ctx context.Context, proj *Pr
if err == nil {
dirName := cmp.Or(mo.PointerToOption(proj.DirName).OrEmpty(), proj.Name)
autoInjectEnv := kit.ParseOrDefault(cfg.AutoInjectEnv.Value, false, strconv.ParseBool)
- meta, err = projectsync.LoadComposeMetadata(ctx, proj.Path, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))
+ meta, err = projectsync.LoadComposeMetadata(ctx, s.composeIdentities, proj.Path, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))
}
if err != nil {
if errors.Is(err, common.ErrProjectEnvUnreadable) {
@@ -2615,7 +2619,7 @@ func (s *ProjectService) upsertProjectForDir(ctx context.Context, dirName, dirPa
projectsDirectory, serviceCountErr := projects.GetProjectsDirectory(ctx, cfg.ProjectsDirectory.Value)
if serviceCountErr == nil {
autoInjectEnv := kit.ParseOrDefault(cfg.AutoInjectEnv.Value, false, strconv.ParseBool)
- composeMetadata, serviceCountErr = projectsync.LoadComposeMetadata(ctx, dirPath, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))
+ composeMetadata, serviceCountErr = projectsync.LoadComposeMetadata(ctx, s.composeIdentities, dirPath, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))
}
serviceCountLogLevel := kit.Ternary(errors.Is(serviceCountErr, common.ErrProjectEnvUnreadable), slog.LevelDebug, slog.LevelWarn)
diff --git a/backend/internal/vulnerability/children/ignores/service.go b/backend/internal/vulnerability/children/ignores/service.go
index 1dd31d7cd..cf9c5d7a0 100644
--- a/backend/internal/vulnerability/children/ignores/service.go
+++ b/backend/internal/vulnerability/children/ignores/service.go
@@ -31,6 +31,8 @@ type Records struct {
DeleteIDs func(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error)
// Page counts the ignore records of envID and lists one ordered page of them.
Page func(ctx context.Context, envID string, order func(*gorm.DB) *gorm.DB, offset, limit int) ([]vulnerability.IgnoredVulnerability, int64, error)
+ // Changed runs after an ignore change commits.
+ Changed func()
}
// Service records which vulnerability findings are ignored per environment.
@@ -119,6 +121,7 @@ func (s *Service) IgnoreVulnerability(ctx context.Context, envID string, payload
if createIgnoreErr != nil {
return nil, fmt.Errorf("failed to create ignore record: %w", createIgnoreErr)
}
+ s.records.Changed()
slog.InfoContext(ctx, "vulnerability ignored",
"vulnerabilityId", payload.VulnerabilityID,
@@ -146,6 +149,7 @@ func (s *Service) UnignoreVulnerability(ctx context.Context, envID, ignoreID str
if deleteErr := s.records.DeleteIdentity(ctx, ignore); deleteErr != nil {
return fmt.Errorf("failed to delete ignore record: %w", deleteErr)
}
+ s.records.Changed()
slog.InfoContext(ctx, "vulnerability unignored", "ignoreId", ignoreID)
return nil
@@ -197,6 +201,7 @@ func (s *Service) BulkIgnoreVulnerabilities(ctx context.Context, envID string, f
if err != nil {
return 0, err
}
+ s.records.Changed()
slog.InfoContext(ctx, "vulnerabilities ignored in bulk", "environmentId", envID, "affected", len(ignores))
return len(ignores), nil
@@ -228,6 +233,7 @@ func (s *Service) BulkUnignoreVulnerabilities(ctx context.Context, envID string,
if err != nil {
return 0, err
}
+ s.records.Changed()
slog.InfoContext(ctx, "vulnerabilities unignored in bulk", "environmentId", envID, "affected", affected)
return int(affected), nil
diff --git a/backend/internal/vulnerability/children/query/service.go b/backend/internal/vulnerability/children/query/service.go
index f51685e22..1dd27faa1 100644
--- a/backend/internal/vulnerability/children/query/service.go
+++ b/backend/internal/vulnerability/children/query/service.go
@@ -15,6 +15,8 @@ import (
"sort"
"strconv"
"strings"
+ "sync"
+ "sync/atomic"
"time"
"github.com/getarcaneapp/arcane/types/v2"
@@ -23,6 +25,7 @@ import (
"github.com/moby/moby/api/types/container"
"github.com/samber/mo"
"go.getarcane.app/kit/pkg"
+ "golang.org/x/sync/singleflight"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -40,6 +43,11 @@ const (
scansTableInternal = "vulnerability_scans"
scanItemsTableInternal = "vulnerability_scan_items"
threatIntelTableInternal = "vulnerability_threat_intel"
+
+ // riskOverviewCacheTTL bounds staleness from changes that do not invalidate
+ // the cache, such as containers starting or threat intel refreshing.
+ riskOverviewCacheTTL = 5 * time.Second
+ riskOverviewBuildTimeout = 30 * time.Second
)
// Service answers vulnerability result, listing, export, and risk queries.
@@ -52,6 +60,18 @@ type Service struct {
refreshSummary func(ctx context.Context, summary *vulnerability.ScanSummary)
threatIntelStatus func(ctx context.Context) vulnerability.ThreatIntelStatus
filterIgnored func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error)
+
+ // riskFlight and riskCache share one risk overview build per environment;
+ // riskGen is bumped by InvalidateRiskOverview to drop cached overviews.
+ riskFlight singleflight.Group
+ riskCache sync.Map // envID -> *riskOverviewCacheEntryInternal
+ riskGen atomic.Uint64
+}
+
+type riskOverviewCacheEntryInternal struct {
+ overview *vulnerability.RiskOverview
+ builtAt time.Time
+ gen uint64
}
func NewService(
@@ -1035,11 +1055,64 @@ func (s *Service) riskNowInternal() time.Time {
}
// GetRiskOverview returns the environment patch priority with its trend, drivers, prioritized images and top findings.
+// Concurrent callers share one build, and the result is reused briefly. Callers must not mutate it.
func (s *Service) GetRiskOverview(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {
if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {
return nil, common.ErrFeatureDisabled
}
+ if overview, ok := s.cachedRiskOverviewInternal(envID); ok {
+ return overview, nil
+ }
+
+ // Keying the build by generation keeps a request that follows an invalidation off an older build.
+ gen := s.riskGen.Load()
+ results := s.riskFlight.DoChan(envID+":"+strconv.FormatUint(gen, 10), func() (any, error) {
+ // A caller that queued behind the winner finds the fresh entry here.
+ if overview, ok := s.cachedRiskOverviewInternal(envID); ok {
+ return overview, nil
+ }
+ buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), riskOverviewBuildTimeout)
+ defer cancel()
+ overview, buildErr := s.buildRiskOverviewInternal(buildCtx, envID)
+ if buildErr != nil {
+ return nil, buildErr
+ }
+ s.riskCache.Store(envID, &riskOverviewCacheEntryInternal{overview: overview, builtAt: time.Now(), gen: gen})
+ return overview, nil
+ })
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case result := <-results:
+ if result.Err != nil {
+ return nil, result.Err
+ }
+ overview, ok := result.Val.(*vulnerability.RiskOverview)
+ if !ok {
+ return nil, errors.New("risk overview cache returned unexpected type")
+ }
+ return overview, nil
+ }
+}
+
+// InvalidateRiskOverview drops cached risk overviews. Call it after a scan or ignore change commits.
+func (s *Service) InvalidateRiskOverview() {
+ s.riskGen.Add(1)
+}
+
+func (s *Service) cachedRiskOverviewInternal(envID string) (*vulnerability.RiskOverview, bool) {
+ value, ok := s.riskCache.Load(envID)
+ if !ok {
+ return nil, false
+ }
+ entry, ok := value.(*riskOverviewCacheEntryInternal)
+ if !ok || entry.gen != s.riskGen.Load() || time.Since(entry.builtAt) >= riskOverviewCacheTTL {
+ return nil, false
+ }
+ return entry.overview, true
+}
+func (s *Service) buildRiskOverviewInternal(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {
overview, err := s.computeRiskInternal(ctx, envID)
if err != nil {
return nil, err
diff --git a/backend/internal/vulnerability/service.go b/backend/internal/vulnerability/service.go
index 488bf67a2..10d83d071 100644
--- a/backend/internal/vulnerability/service.go
+++ b/backend/internal/vulnerability/service.go
@@ -92,6 +92,7 @@ func NewVulnerabilityService(
DeleteIdentity: s.deleteIgnoreIdentityInternal,
DeleteIDs: deleteIgnoreIDsInternal,
Page: s.ignorePageInternal,
+ Changed: s.query.InvalidateRiskOverview,
}, s.query.ActiveItemsQuery)
s.scan = scan.NewService(scan.Dependencies{
DB: db,
@@ -109,9 +110,16 @@ func NewVulnerabilityService(
Events: eventService,
Notifications: notificationService,
Activity: activityService,
- SaveResult: s.store.SaveResult,
- EnsureSummary: store.EnsureSummary,
- FilterIgnored: s.ignores.FilterForImage,
+ // The risk cache is cleared only once the scan's transaction has committed.
+ SaveResult: func(ctx context.Context, result *vulnerability.ScanResult) error {
+ if err := s.store.SaveResult(ctx, result); err != nil {
+ return err
+ }
+ s.query.InvalidateRiskOverview()
+ return nil
+ },
+ EnsureSummary: store.EnsureSummary,
+ FilterIgnored: s.ignores.FilterForImage,
})
return s
}
@@ -303,7 +311,11 @@ func (s *VulnerabilityService) saveRecordStatusInternal(ctx context.Context, res
}
func (s *VulnerabilityService) deleteRecordsInternal(ctx context.Context, imageIDs []string) error {
- return s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error
+ if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {
+ return err
+ }
+ s.query.InvalidateRiskOverview()
+ return nil
}
func (s *VulnerabilityService) deleteReportsInternal(ctx context.Context, imageIDs []string) error {
diff --git a/backend/pkg/gitutil/git.go b/backend/pkg/gitutil/git.go
index 19a60f338..c1ca6a8ae 100644
--- a/backend/pkg/gitutil/git.go
+++ b/backend/pkg/gitutil/git.go
@@ -319,8 +319,9 @@ func getKnownHostsPathsInternal(
}
}
-// Clone clones a repository to a temporary directory
-func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig) (string, error) {
+// Clone clones a repository to a temporary directory. A depth of 0 fetches the
+// full history; read-only callers pass 1 to fetch only the branch tip.
+func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig, depth int) (string, error) {
if _, hasDeadline := ctx.Deadline(); !hasDeadline {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, 5*time.Minute)
@@ -362,6 +363,7 @@ func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig)
cloneOptions := &git.CloneOptions{
URL: url,
Progress: nil,
+ Depth: depth,
}
if authMethod != nil {
@@ -374,6 +376,13 @@ func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig)
}
_, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)
+ if err != nil && depth > 0 && strings.Contains(err.Error(), "shallow") {
+ // Servers without shallow support, go-git's own included, get a full clone.
+ cloneOptions.Depth = 0
+ if err = os.RemoveAll(tmpDir); err == nil {
+ _, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)
+ }
+ }
if err != nil {
_ = os.RemoveAll(tmpDir)
return "", fmt.Errorf("failed to clone repository: %w", err)
@@ -639,7 +648,7 @@ func (c *Client) TestConnection(ctx context.Context, url, branch string, auth Au
return err
}
- tmpDir, err := c.Clone(ctx, url, branch, auth)
+ tmpDir, err := c.Clone(ctx, url, branch, auth, 1)
if err != nil {
return err
}
diff --git a/backend/pkg/gitutil/git_write.go b/backend/pkg/gitutil/git_write.go
index caed9309c..c05d5859d 100644
--- a/backend/pkg/gitutil/git_write.go
+++ b/backend/pkg/gitutil/git_write.go
@@ -10,6 +10,7 @@ import (
"os"
"path"
"path/filepath"
+ "slices"
"sort"
"strings"
"time"
@@ -18,7 +19,6 @@ import (
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
- "github.com/go-git/go-git/v5/plumbing/format/diff"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
"go.getarcane.app/acfs"
@@ -215,7 +215,7 @@ func (c *Client) CheckoutForWrite(ctx context.Context, url, branch string, auth
return nil, err
}
- repoPath, err := c.Clone(ctx, normalized, branch, auth)
+ repoPath, err := c.Clone(ctx, normalized, branch, auth, 0)
if err == nil {
repo, openErr := git.PlainOpen(repoPath)
if openErr != nil {
@@ -269,7 +269,7 @@ func (c *Client) initEmptyCheckoutInternal(url, branch string) (*WriteCheckout,
}
func (c *Client) checkoutNewBranchInternal(ctx context.Context, url, branch string, auth AuthConfig) (*WriteCheckout, error) {
- repoPath, err := c.Clone(ctx, url, "", auth)
+ repoPath, err := c.Clone(ctx, url, "", auth, 0)
if err != nil {
return nil, err
}
@@ -448,10 +448,16 @@ func (c *Client) DirectoryHistory(ctx context.Context, repoPath, directory strin
}
return nil, fmt.Errorf("failed to iterate history: %w", nextErr)
}
- files, nextErr := commitDirectoryFilesInternal(commit, prefix)
+ changes, nextErr := commitChangesInternal(ctx, commit, object.DefaultDiffTreeOptions)
if nextErr != nil {
return nil, nextErr
}
+ var files []string
+ for _, change := range changes {
+ if _, relative, ok := changePathInDirectoryInternal(change, prefix); ok {
+ files = append(files, relative)
+ }
+ }
entries = append(entries, historyEntryInternal(commit, files))
if limit > 0 && len(entries) >= limit {
break
@@ -481,22 +487,31 @@ func (c *Client) CommitDiff(ctx context.Context, repoPath, commitHash, directory
return HistoryEntry{}, nil, fmt.Errorf("commit not found: %w", err)
}
prefix := directoryPrefixInternal(directory)
- patch, err := commitPatchInternal(commit)
+ // Files are listed rename-aware; each file's patch comes from the plain diff.
+ listed, err := commitChangesInternal(ctx, commit, object.DefaultDiffTreeOptions)
+ if err != nil {
+ return HistoryEntry{}, nil, err
+ }
+ plain, err := commitChangesInternal(ctx, commit, nil)
if err != nil {
return HistoryEntry{}, nil, err
}
var files []string
var diffs []FileDiff
- for _, filePatch := range patch.FilePatches() {
- name, relative, ok := patchPathInDirectoryInternal(filePatch, prefix)
+ for _, change := range listed {
+ name, relative, ok := changePathInDirectoryInternal(change, prefix)
if !ok {
continue
}
files = append(files, relative)
- single, commitFilePatchErr := commitFilePatchInternal(commit, name)
- if commitFilePatchErr != nil {
- return HistoryEntry{}, nil, commitFilePatchErr
+ single := ""
+ if idx := slices.IndexFunc(plain, func(c *object.Change) bool { return c.From.Name == name || c.To.Name == name }); idx >= 0 {
+ patch, patchErr := plain[idx].Patch()
+ if patchErr != nil {
+ return HistoryEntry{}, nil, fmt.Errorf("failed to build patch for %s: %w", name, patchErr)
+ }
+ single = patch.String()
}
diffs = append(diffs, FileDiff{Path: relative, Patch: single})
}
@@ -523,7 +538,9 @@ func historyEntryInternal(commit *object.Commit, files []string) HistoryEntry {
}
}
-func commitPatchInternal(commit *object.Commit) (*object.Patch, error) {
+// commitChangesInternal diffs a commit against its first parent. Nil opts
+// disables rename detection.
+func commitChangesInternal(ctx context.Context, commit *object.Commit, opts *object.DiffTreeOptions) (object.Changes, error) {
var parentTree *object.Tree
if commit.NumParents() > 0 {
parent, err := commit.Parent(0)
@@ -539,71 +556,23 @@ func commitPatchInternal(commit *object.Commit) (*object.Patch, error) {
if err != nil {
return nil, fmt.Errorf("failed to load commit tree: %w", err)
}
- patch, err := parentTree.Patch(tree)
+ changes, err := object.DiffTreeWithOptions(ctx, parentTree, tree, opts)
if err != nil {
return nil, fmt.Errorf("failed to diff commit: %w", err)
}
- return patch, nil
+ return changes, nil
}
-func commitDirectoryFilesInternal(commit *object.Commit, prefix string) ([]string, error) {
- patch, err := commitPatchInternal(commit)
- if err != nil {
- return nil, err
- }
- var files []string
- for _, filePatch := range patch.FilePatches() {
- if _, relative, ok := patchPathInDirectoryInternal(filePatch, prefix); ok {
- files = append(files, relative)
- }
- }
- return files, nil
-}
-
-// patchPathInDirectoryInternal picks the side of a file patch that lives under
+// changePathInDirectoryInternal picks the side of a change that lives under
// prefix, so renames into or out of the directory are still reported.
-func patchPathInDirectoryInternal(filePatch diff.FilePatch, prefix string) (string, string, bool) {
- from, to := filePatch.Files()
- for _, file := range []diff.File{to, from} {
- if file == nil {
+func changePathInDirectoryInternal(change *object.Change, prefix string) (string, string, bool) {
+ for _, name := range []string{change.To.Name, change.From.Name} {
+ if name == "" {
continue
}
- if relative, ok := strings.CutPrefix(file.Path(), prefix); ok {
- return file.Path(), relative, true
+ if relative, ok := strings.CutPrefix(name, prefix); ok {
+ return name, relative, true
}
}
return "", "", false
}
-
-func commitFilePatchInternal(commit *object.Commit, name string) (string, error) {
- var parentTree *object.Tree
- if commit.NumParents() > 0 {
- parent, err := commit.Parent(0)
- if err != nil {
- return "", fmt.Errorf("failed to load parent commit: %w", err)
- }
- parentTree, err = parent.Tree()
- if err != nil {
- return "", fmt.Errorf("failed to load parent tree: %w", err)
- }
- }
- tree, err := commit.Tree()
- if err != nil {
- return "", fmt.Errorf("failed to load commit tree: %w", err)
- }
- changes, err := object.DiffTree(parentTree, tree)
- if err != nil {
- return "", fmt.Errorf("failed to diff commit: %w", err)
- }
- for _, change := range changes {
- if change.From.Name != name && change.To.Name != name {
- continue
- }
- patch, patchErr := change.Patch()
- if patchErr != nil {
- return "", fmt.Errorf("failed to build patch for %s: %w", name, patchErr)
- }
- return patch.String(), nil
- }
- return "", nil
-}
diff --git a/backend/pkg/gitutil/git_write_test.go b/backend/pkg/gitutil/git_write_test.go
index 47c30099c..c1ca8a37f 100644
--- a/backend/pkg/gitutil/git_write_test.go
+++ b/backend/pkg/gitutil/git_write_test.go
@@ -117,7 +117,7 @@ func TestCheckoutForWrite_CreatesMissingBranchFromDefault(t *testing.T) {
require.Equal(t, head, remoteHead)
verify := NewClient(t.TempDir())
- repoPath, err := verify.Clone(ctx, url, "feature", noAuthInternal())
+ repoPath, err := verify.Clone(ctx, url, "feature", noAuthInternal(), 0)
require.NoError(t, err)
t.Cleanup(func() { _ = verify.Cleanup(repoPath) })
assert.FileExists(t, filepath.Join(repoPath, "unrelated", "keep.txt"))
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.go
index a601fe99c..2c50f2795 100644
--- a/backend/pkg/libarcane/edge/client.go
+++ b/backend/pkg/libarcane/edge/client.go
@@ -10,6 +10,7 @@ import (
"log/slog"
"net"
"net/http"
+ "slices"
"strings"
"sync"
"time"
@@ -403,7 +404,7 @@ func (c *TunnelClient) grpcFailureStreakInternal() int {
func (c *TunnelClient) registerMessageInternal() *TunnelMessage {
capabilities := AdvertisedEdgeCommands()
- capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity)
+ capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity, tunnelCapabilityCommandCredit)
registration, _ := c.registration.Load()
return &TunnelMessage{
Type: MessageTypeRegister,
@@ -457,7 +458,10 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn Tunne
if !result.msg.Accepted {
return nil, fmt.Errorf("manager rejected tunnel registration: %s", result.msg.Error)
}
- c.registration.Store(clientRegistrationInternal{sessionID: result.msg.SessionID})
+ c.registration.Store(clientRegistrationInternal{
+ sessionID: result.msg.SessionID,
+ commandCredit: slices.Contains(result.msg.Capabilities, tunnelCapabilityCommandCreditGrant),
+ })
return result.msg, nil
}
}
@@ -583,6 +587,16 @@ func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, w
c.handleStreamClose(ctx, msg)
case MessageTypeCancelRequest:
slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)
+ case MessageTypeCommandCredit:
+ if value, ok := c.commandRecorders.Load(msg.ID); ok {
+ if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {
+ recorder.inFlight.Add(-msg.Credit)
+ select {
+ case recorder.credited <- struct{}{}:
+ default:
+ }
+ }
+ }
case MessageTypeResponse, MessageTypeHeartbeat, MessageTypeStreamEnd, MessageTypeEvent, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete:
slog.DebugContext(ctx, "Ignoring message type on agent", "type", msg.Type)
case MessageTypeHeartbeatAck:
@@ -708,7 +722,20 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConn
return
}
- recorder := newCommandResponseRecorderInternal(msg.ID, msg.Command, conn)
+ recorder := &commandResponseRecorder{
+ commandID: msg.ID,
+ commandName: msg.Command,
+ conn: conn,
+ headers: make(http.Header),
+ statusCode: http.StatusOK,
+ ctx: reqCtx,
+ credited: make(chan struct{}, 1),
+ }
+ if registration, _ := c.registration.Load(); registration.commandCredit {
+ recorder.window = commandCreditWindow
+ c.commandRecorders.Store(msg.ID, recorder)
+ defer c.commandRecorders.Delete(msg.ID)
+ }
c.handler.ServeHTTP(recorder, req)
if closeErr := recorder.Close(); closeErr != nil {
slog.WarnContext(reqCtx, "Failed to finalize command response", "id", msg.ID, "command", msg.Command, "error", closeErr)
@@ -1188,16 +1215,6 @@ func (c *TunnelClient) sendCommandCompleteInternal(conn TunnelConnection, comman
})
}
-func newCommandResponseRecorderInternal(commandID, commandName string, conn TunnelConnection) *commandResponseRecorder {
- return &commandResponseRecorder{
- commandID: commandID,
- commandName: commandName,
- conn: conn,
- headers: make(http.Header),
- statusCode: http.StatusOK,
- }
-}
-
func (r *commandResponseRecorder) Header() http.Header {
return r.headers
}
@@ -1207,6 +1224,9 @@ func (r *commandResponseRecorder) Write(b []byte) (int, error) {
defer r.mu.Unlock()
originalLen := len(b)
+ if r.flushErr != nil {
+ return 0, r.flushErr
+ }
if len(b) == 0 {
return 0, nil
}
@@ -1225,16 +1245,9 @@ func (r *commandResponseRecorder) Write(b []byte) (int, error) {
if len(chunk) > defaultCommandChunkSize {
chunk = chunk[:defaultCommandChunkSize]
}
- if err := r.conn.Send(&TunnelMessage{
- ID: r.commandID,
- Type: MessageTypeCommandOutput,
- Body: append([]byte(nil), chunk...),
- Sequence: r.sequence,
- Command: r.commandName,
- }); err != nil {
+ if err := r.sendOutputLocked(chunk); err != nil {
return 0, err
}
- r.sequence++
b = b[len(chunk):]
}
@@ -1252,7 +1265,15 @@ func (r *commandResponseRecorder) Flush() {
r.mu.Lock()
defer r.mu.Unlock()
r.streaming = true
- _ = r.flushBufferLocked()
+ if r.flushErr != nil {
+ return
+ }
+ // An empty first flush still sends the status and headers so the manager can start the response.
+ if r.sequence == 0 && r.buffer.Len() == 0 {
+ r.flushErr = r.sendOutputLocked(nil)
+ return
+ }
+ r.flushErr = r.flushBufferLocked()
}
func (r *commandResponseRecorder) Close() error {
@@ -1261,6 +1282,9 @@ func (r *commandResponseRecorder) Close() error {
if r.closed {
return nil
}
+ if r.flushErr != nil {
+ return r.flushErr
+ }
if !r.streaming {
err := r.conn.Send(&TunnelMessage{
@@ -1301,17 +1325,40 @@ func (r *commandResponseRecorder) flushBufferLocked() error {
if r.buffer.Len() == 0 {
return nil
}
- if err := r.conn.Send(&TunnelMessage{
+ if err := r.sendOutputLocked(r.buffer.Bytes()); err != nil {
+ return err
+ }
+ r.buffer.Reset()
+ return nil
+}
+
+// sendOutputLocked sends one output chunk, first waiting for manager credit when the window is
+// full. The first chunk also carries the status and headers so the manager can stream the response.
+func (r *commandResponseRecorder) sendOutputLocked(chunk []byte) error {
+ for r.window > 0 && r.inFlight.Load() > 0 && r.inFlight.Load()+int64(len(chunk)) > r.window {
+ select {
+ case <-r.credited:
+ case <-r.ctx.Done():
+ return r.ctx.Err()
+ }
+ }
+ r.inFlight.Add(int64(len(chunk)))
+
+ msg := &TunnelMessage{
ID: r.commandID,
Type: MessageTypeCommandOutput,
- Body: append([]byte(nil), r.buffer.Bytes()...),
+ Body: append([]byte(nil), chunk...),
Sequence: r.sequence,
Command: r.commandName,
- }); err != nil {
+ }
+ if r.sequence == 0 {
+ msg.Status = r.statusCode
+ msg.Headers = flattenResponseHeadersInternal(r.headers)
+ }
+ if err := r.conn.Send(msg); err != nil {
return err
}
r.sequence++
- r.buffer.Reset()
return nil
}
@@ -1439,7 +1486,7 @@ func (r *streamingResponseRecorder) writeHeaderLocked(statusCode int) error {
respHeaders[k] = v[0]
}
}
- respHeaders["X-Arcane-Tunnel-Stream"] = "1"
+ respHeaders[tunnelStreamHeader] = "1"
if err := r.conn.Send(&TunnelMessage{
ID: r.requestID,
@@ -1498,7 +1545,7 @@ func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) err
}
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
if strings.HasPrefix(strings.ToLower(managerWSURL), "wss://") {
- tlsConfig, err := buildManagerClientTLSConfigInternal(c.cfg)
+ tlsConfig, err := buildManagerClientTLSConfig(c.cfg)
if err != nil {
return fmt.Errorf("failed to configure edge websocket TLS: %w", err)
}
diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.go
index 9da64aaf9..436b6039a 100644
--- a/backend/pkg/libarcane/edge/client_test.go
+++ b/backend/pkg/libarcane/edge/client_test.go
@@ -2274,7 +2274,7 @@ func (f *fakeTunnelConn) Send(msg *TunnelMessage) error {
}
copyMsg := *msg
if msg.Headers != nil {
- copyMsg.Headers = cloneHeaderMap(msg.Headers)
+ copyMsg.Headers = maps.Clone(msg.Headers)
}
if msg.Body != nil {
copyMsg.Body = append([]byte(nil), msg.Body...)
diff --git a/backend/pkg/libarcane/edge/client_transport_grpc.go b/backend/pkg/libarcane/edge/client_transport_grpc.go
index 6544c4ea5..5f8f7ea85 100644
--- a/backend/pkg/libarcane/edge/client_transport_grpc.go
+++ b/backend/pkg/libarcane/edge/client_transport_grpc.go
@@ -50,7 +50,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {
}
if c.useTLSForManagerGRPC() {
- tlsConfig, err := buildManagerClientTLSConfigInternal(c.cfg)
+ tlsConfig, err := buildManagerClientTLSConfig(c.cfg)
if err != nil {
return fmt.Errorf("failed to configure edge gRPC TLS: %w", err)
}
diff --git a/backend/pkg/libarcane/edge/command_client.go b/backend/pkg/libarcane/edge/command_client.go
index 298a1edc6..3b1ff9e12 100644
--- a/backend/pkg/libarcane/edge/command_client.go
+++ b/backend/pkg/libarcane/edge/command_client.go
@@ -4,6 +4,8 @@ import (
"context"
"errors"
"fmt"
+ "log/slog"
+ "math"
"net/http"
"slices"
"time"
@@ -17,7 +19,13 @@ const (
// manager->agent, cancel_request agent->manager) instead of the legacy
// re-encoded forms.
tunnelCapabilityProtoParity = "proto-parity-v1"
- bodyTransferMetadataKey = "body_transfer_id"
+ // Credit-based flow control for command output keeps at most commandCreditWindow bytes
+ // unconsumed by the manager. Agents offer it; only the manager sends the grant, which older
+ // managers that echo agent capabilities cannot produce.
+ tunnelCapabilityCommandCredit = "command-credit-v1"
+ tunnelCapabilityCommandCreditGrant = "command-credit-grant-v1"
+ commandCreditWindow = 8 << 20
+ bodyTransferMetadataKey = "body_transfer_id"
)
func NewCommandClient() *CommandClient {
@@ -101,8 +109,20 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C
}
}
- status, headers, body, err := collectCommandResponseInternal(ctx, tunnel, pending, req.Method)
+ // Output the collector has consumed is credited back, so the agent cannot outrun a slow reader.
+ var credit func(int)
+ if slices.Contains(tunnel.Capabilities, tunnelCapabilityCommandCredit) {
+ credit = func(consumed int) {
+ tunnel.grantCommandCredit(ctx, requestID, int64(consumed))
+ }
+ }
+
+ status, headers, body, err := collectCommandResponseInternal(ctx, tunnel, pending, req.Method, req.Output, credit)
if err != nil {
+ if credit != nil {
+ // Release an agent still waiting to send output that nobody will read.
+ credit(math.MaxInt32)
+ }
return nil, err
}
@@ -156,6 +176,42 @@ func (c *CommandClient) OpenStream(ctx context.Context, tunnel *AgentTunnel, req
var DefaultCommandClient = NewCommandClient()
+// grantCommandCredit queues credit for a command without blocking. A per-tunnel sender coalesces
+// queued credits and sends them, so a stalled connection never holds up the response reader.
+func (t *AgentTunnel) grantCommandCredit(ctx context.Context, commandID string, consumed int64) {
+ t.creditOnce.Do(func() {
+ t.credits = make(map[string]int64)
+ t.creditSignal = make(chan struct{}, 1)
+ logCtx := context.WithoutCancel(ctx)
+ go func() {
+ for {
+ select {
+ case <-t.done:
+ return
+ case <-t.creditSignal:
+ }
+ t.creditMu.Lock()
+ queued := t.credits
+ t.credits = make(map[string]int64)
+ t.creditMu.Unlock()
+ for id, credit := range queued {
+ if err := t.Conn.Send(&TunnelMessage{ID: id, Type: MessageTypeCommandCredit, Credit: credit}); err != nil {
+ slog.DebugContext(logCtx, "Failed to send edge command credit", "id", id, "error", err)
+ }
+ }
+ }
+ }()
+ })
+
+ t.creditMu.Lock()
+ t.credits[commandID] += consumed
+ t.creditMu.Unlock()
+ select {
+ case t.creditSignal <- struct{}{}:
+ default:
+ }
+}
+
func validateConnectedTunnelInternal(tunnel *AgentTunnel) error {
if tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {
return errors.New("edge tunnel is not connected")
diff --git a/backend/pkg/libarcane/edge/commands_test.go b/backend/pkg/libarcane/edge/commands_test.go
index 6d3dc5c2a..a1cda8158 100644
--- a/backend/pkg/libarcane/edge/commands_test.go
+++ b/backend/pkg/libarcane/edge/commands_test.go
@@ -125,7 +125,7 @@ func TestCollectCommandResponse(t *testing.T) {
pending.ResponseCh <- &TunnelMessage{ID: "cmd-1", Type: MessageTypeCommandComplete, Status: 200, Headers: map[string]string{"Content-Type": "text/plain"}, Body: []byte("world")}
require.NoError(t, tunnel.CloseWithReason(""))
- status, headers, body, err := collectCommandResponseInternal(t.Context(), tunnel, pending, "")
+ status, headers, body, err := collectCommandResponseInternal(t.Context(), tunnel, pending, "", nil, nil)
require.NoError(t, err)
require.Equal(t, 200, status)
require.Equal(t, "text/plain", headers["Content-Type"])
diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.go
index 89c55e331..79e38de68 100644
--- a/backend/pkg/libarcane/edge/poll_control.go
+++ b/backend/pkg/libarcane/edge/poll_control.go
@@ -237,7 +237,7 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {
// In proxy-terminated mTLS deployments, the client certificate is consumed
// by the TLS terminator before this request reaches Arcane. The token is
// still needed as the poll protocol's environment lookup claim.
- token, source := tokenFromHeadersWithSourceInternal(req)
+ token, source := agentToken(req.Header.Values)
if token == "" {
slog.WarnContext(ctx, "Edge poll request without token")
return c.JSON(http.StatusUnauthorized, map[string]any{"error": "agent token required"})
@@ -259,9 +259,9 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {
)
return c.JSON(http.StatusUnauthorized, map[string]any{"error": "invalid agent token"})
}
- if requireRequestCertificateIdentityErr := s.requireRequestCertificateIdentityInternal(req, envID); requireRequestCertificateIdentityErr != nil {
- slog.WarnContext(ctx, "Rejected edge poll request with mismatched client certificate", "environmentId", envID, "error", requireRequestCertificateIdentityErr)
- return c.JSON(http.StatusUnauthorized, map[string]any{"error": requireRequestCertificateIdentityErr.Error()})
+ if identityErr := s.requireCertificateIdentity(req.TLS, envID); identityErr != nil {
+ slog.WarnContext(ctx, "Rejected edge poll request with mismatched client certificate", "environmentId", envID, "error", identityErr)
+ return c.JSON(http.StatusUnauthorized, map[string]any{"error": identityErr.Error()})
}
pollInterval := DefaultTunnelPollInterval
diff --git a/backend/pkg/libarcane/edge/poll_control_test.go b/backend/pkg/libarcane/edge/poll_control_test.go
index 93d2a58fa..45acc1542 100644
--- a/backend/pkg/libarcane/edge/poll_control_test.go
+++ b/backend/pkg/libarcane/edge/poll_control_test.go
@@ -179,10 +179,10 @@ func TestTunnelServer_HandlePoll_AcceptsTokenAfterProxyTerminatedMTLS(t *testing
}
return "env-proxy-mtls", nil
}, nil)
- server.SetConfig(&Config{
+ server.Config = &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
AppURL: "https://manager.example.com",
- })
+ }
router := echo.New()
router.POST("/api/tunnel/poll", server.HandlePoll)
diff --git a/backend/pkg/libarcane/edge/proxy.go b/backend/pkg/libarcane/edge/proxy.go
index 0b88db0ef..5053a6bfc 100644
--- a/backend/pkg/libarcane/edge/proxy.go
+++ b/backend/pkg/libarcane/edge/proxy.go
@@ -15,13 +15,17 @@ import (
)
const (
- // DefaultProxyTimeout is the default timeout for proxied requests
- DefaultProxyTimeout = 5 * time.Minute
- // DefaultTunnelAcquirePollEvery is how frequently the manager checks for a
- // newly activated edge tunnel while waiting for poll mode to connect.
+ DefaultProxyTimeout = 5 * time.Minute
DefaultTunnelAcquirePollEvery = 100 * time.Millisecond
+ tunnelStreamHeader = "X-Arcane-Tunnel-Stream"
)
+// browserSecurityHeaders are browser-enforced headers that must not cross the edge tunnel.
+var browserSecurityHeaders = map[string]struct{}{
+ "Origin": {}, "Referer": {}, "Cookie": {}, "Access-Control-Request-Method": {},
+ "Access-Control-Request-Headers": {}, "Sec-Fetch-Mode": {}, "Sec-Fetch-Site": {}, "Sec-Fetch-Dest": {},
+}
+
// DefaultTunnelAcquireTimeout returns a poll-aware wait timeout for acquiring
// an on-demand edge tunnel.
func DefaultTunnelAcquireTimeout() time.Duration {
@@ -58,20 +62,30 @@ func registerPendingRequestInternal(tunnel *AgentTunnel, requestID string) (*Pen
return pending, nil
}
-func collectCommandResponseInternal(ctx context.Context, tunnel *AgentTunnel, pending *PendingRequest, method string) (int, map[string]string, []byte, error) {
- state := &grpcResponseState{}
+// collectCommandResponseInternal waits for a command response. When out is set
+// the response is written to it as it arrives and the returned body is nil.
+// credit, when set, returns flow-control credit for each consumed output chunk.
+func collectCommandResponseInternal(
+ ctx context.Context,
+ tunnel *AgentTunnel,
+ pending *PendingRequest,
+ method string,
+ out http.ResponseWriter,
+ credit func(consumed int),
+) (int, map[string]string, []byte, error) {
+ state := &grpcResponseState{out: out, credit: credit}
for {
select {
case <-ctx.Done():
return 0, nil, nil, ctx.Err()
case <-tunnel.done:
- if done, status, headers, body, err := state.drainTerminalResponseInternal(pending.ResponseCh, method); done {
+ if done, status, headers, body, err := state.drainTerminalResponseInternal(ctx, pending, method); done {
return status, headers, body, err
}
return 0, nil, nil, fmt.Errorf("edge tunnel closed while waiting for response: %w", ErrTunnelConnectionClosed)
case err := <-pending.failureCh:
- if done, status, headers, body, drainTerminalResponseErr := state.drainTerminalResponseInternal(pending.ResponseCh, method); done {
+ if done, status, headers, body, drainTerminalResponseErr := state.drainTerminalResponseInternal(ctx, pending, method); done {
return status, headers, body, drainTerminalResponseErr
}
return 0, nil, nil, err
@@ -86,53 +100,71 @@ func collectCommandResponseInternal(ctx context.Context, tunnel *AgentTunnel, pe
}
}
+// handleTunnelMessageInternal applies one response message. A failed write to the output
+// ends the response with that error.
func (s *grpcResponseState) handleTunnelMessageInternal(method string, incoming *TunnelMessage) (bool, int, map[string]string, []byte, error) {
if incoming == nil {
return false, 0, nil, nil, nil
}
+ var done bool
+ var status int
+ var headers map[string]string
+ var body []byte
+ var err error
switch incoming.Type {
case MessageTypeResponse:
- done, status, headers, body := s.handleResponse(method, incoming)
- return done, status, headers, body, nil
+ done, status, headers, body = s.handleResponse(method, incoming)
case MessageTypeCommandOutput, MessageTypeStreamData, MessageTypeFileChunk:
s.handleStreamData(incoming)
+ if incoming.Type == MessageTypeCommandOutput && s.credit != nil && len(incoming.Body) > 0 && s.outErr == nil {
+ s.credit(len(incoming.Body))
+ }
case MessageTypeCommandComplete:
- status, headers, body, err := s.handleCommandComplete(incoming)
- return true, status, headers, body, err
+ done = true
+ status, headers, body, err = s.handleCommandComplete(incoming)
case MessageTypeStreamEnd:
- done, status, headers, body := s.handleStreamEnd()
- return done, status, headers, body, nil
- case MessageTypeRequest,
- MessageTypeHeartbeat,
- MessageTypeHeartbeatAck,
- MessageTypeWebSocketStart,
- MessageTypeWebSocketData,
- MessageTypeWebSocketClose,
- MessageTypeRegister,
- MessageTypeRegisterResponse,
- MessageTypeEvent,
- MessageTypeCommandRequest,
- MessageTypeCommandAck,
- MessageTypeStreamOpen,
- MessageTypeStreamClose,
- MessageTypeCancelRequest:
- }
- return false, 0, nil, nil, nil
+ done, status, headers, body = s.handleStreamEnd()
+ }
+ if s.outErr != nil {
+ return true, s.status, nil, nil, fmt.Errorf("failed to write proxied response: %w", s.outErr)
+ }
+ return done, status, headers, body, err
}
-func (s *grpcResponseState) drainTerminalResponseInternal(respCh <-chan *TunnelMessage, method string) (bool, int, map[string]string, []byte, error) {
+// drainTerminalResponseInternal applies messages already received for pending, including any still
+// queued behind a full channel, until the response completes or nothing more is queued.
+func (s *grpcResponseState) drainTerminalResponseInternal(ctx context.Context, pending *PendingRequest, method string) (bool, int, map[string]string, []byte, error) {
for {
+ if err := ctx.Err(); err != nil {
+ return true, 0, nil, nil, err
+ }
+ // Checking idle before reading ensures every queued delivery is already in the channel.
+ pending.mu.Lock()
+ idle := pending.failed || (!pending.draining && len(pending.backlog) == 0)
+ pending.mu.Unlock()
+
+ var incoming *TunnelMessage
+ ok := true
select {
- case incoming, ok := <-respCh:
- if !ok {
- return true, 0, nil, nil, fmt.Errorf("edge tunnel response channel closed before a response was received: %w", ErrTunnelConnectionClosed)
+ case incoming, ok = <-pending.ResponseCh:
+ default:
+ if idle {
+ return false, 0, nil, nil, nil
}
- if done, status, headers, body, err := s.handleTunnelMessageInternal(method, incoming); done {
- return true, status, headers, body, err
+ select {
+ case incoming, ok = <-pending.ResponseCh:
+ case <-ctx.Done():
+ return true, 0, nil, nil, ctx.Err()
+ case <-time.After(pendingDeliveryTimeout):
+ return false, 0, nil, nil, nil
}
- default:
- return false, 0, nil, nil, nil
+ }
+ if !ok {
+ return true, 0, nil, nil, fmt.Errorf("edge tunnel response channel closed before a response was received: %w", ErrTunnelConnectionClosed)
+ }
+ if done, status, headers, body, err := s.handleTunnelMessageInternal(method, incoming); done {
+ return true, status, headers, body, err
}
}
}
@@ -144,36 +176,40 @@ func (s *grpcResponseState) handleResponse(method string, incoming *TunnelMessag
s.respHeaders = incoming.Headers
}
- if s.respHeaders["X-Arcane-Tunnel-Stream"] == "1" {
- if len(incoming.Body) > 0 {
- s.respBody.Write(incoming.Body)
- }
+ if s.respHeaders[tunnelStreamHeader] == "1" {
+ s.commitInternal()
+ s.writeBodyInternal(incoming.Body)
return false, 0, nil, nil
}
if len(incoming.Body) > 0 {
- s.respBody.Write(incoming.Body)
- return true, s.status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes()
+ s.writeBodyInternal(incoming.Body)
+ return s.finishInternal()
}
if method == http.MethodHead || s.status == http.StatusNoContent || s.status == http.StatusNotModified {
- return true, s.status, stripInternalTunnelHeaders(s.respHeaders), nil
+ return s.finishInternal()
}
return false, 0, nil, nil
}
func (s *grpcResponseState) handleStreamData(incoming *TunnelMessage) {
- if len(incoming.Body) > 0 {
- s.respBody.Write(incoming.Body)
+ // Agents that support streaming put the status and headers on the first output chunk.
+ if !s.gotResponse && incoming.Type == MessageTypeCommandOutput && incoming.Status != 0 {
+ s.gotResponse = true
+ s.status = incoming.Status
+ s.respHeaders = incoming.Headers
+ s.commitInternal()
}
+ s.writeBodyInternal(incoming.Body)
}
func (s *grpcResponseState) handleStreamEnd() (bool, int, map[string]string, []byte) {
if !s.gotResponse {
return false, 0, nil, nil
}
- return true, s.status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes()
+ return s.finishInternal()
}
func (s *grpcResponseState) handleCommandComplete(incoming *TunnelMessage) (int, map[string]string, []byte, error) {
@@ -182,13 +218,59 @@ func (s *grpcResponseState) handleCommandComplete(incoming *TunnelMessage) (int,
s.status = incoming.Status
s.respHeaders = incoming.Headers
}
- if len(incoming.Body) > 0 {
- s.respBody.Write(incoming.Body)
- }
+ s.writeBodyInternal(incoming.Body)
if incoming.Error != "" && incoming.Status >= http.StatusBadRequest {
return incoming.Status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes(), errors.New(incoming.Error)
}
- return incoming.Status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes(), nil
+ _, _, headers, body := s.finishInternal()
+ return incoming.Status, headers, body, nil
+}
+
+// commitInternal writes the status, headers and any buffered body to out.
+func (s *grpcResponseState) commitInternal() {
+ if s.out == nil || s.committed {
+ return
+ }
+ s.committed = true
+ header := s.out.Header()
+ for k, v := range s.respHeaders {
+ if !isHopByHopHeader(k) && http.CanonicalHeaderKey(k) != tunnelStreamHeader {
+ header.Set(k, v)
+ }
+ }
+ s.out.WriteHeader(s.status)
+ s.writeBodyInternal(s.respBody.Bytes())
+ s.respBody.Reset()
+}
+
+// writeBodyInternal buffers body until the response is committed, then writes and flushes it.
+// The first output error is kept and stops later writes.
+func (s *grpcResponseState) writeBodyInternal(body []byte) {
+ if !s.committed {
+ s.respBody.Write(body)
+ return
+ }
+ if s.outErr != nil {
+ return
+ }
+ if len(body) > 0 {
+ if _, s.outErr = s.out.Write(body); s.outErr != nil {
+ return
+ }
+ }
+ if flushErr := http.NewResponseController(s.out).Flush(); flushErr != nil && !errors.Is(flushErr, http.ErrNotSupported) {
+ s.outErr = flushErr
+ }
+}
+
+// finishInternal completes the response, committing it to out when set.
+func (s *grpcResponseState) finishInternal() (bool, int, map[string]string, []byte) {
+ headers := stripInternalTunnelHeaders(s.respHeaders)
+ if s.out != nil {
+ s.commitInternal()
+ return true, s.status, headers, nil
+ }
+ return true, s.status, headers, s.respBody.Bytes()
}
// ProxyHTTPRequest is a helper that proxies an echo context through a tunnel
@@ -227,13 +309,25 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) e
"bodyLength", len(body),
)
- status, respHeaders, respBody, err := ProxyRequest(proxyCtx, tunnel, req.Method, targetPath, req.URL.RawQuery, headers, body)
+ _, err := DefaultCommandClient.Execute(proxyCtx, tunnel, &CommandRequest{
+ Method: req.Method,
+ Path: targetPath,
+ Query: req.URL.RawQuery,
+ Headers: headers,
+ Body: body,
+ Output: c.Response(),
+ })
if err != nil {
slog.ErrorContext(ctx, "Edge tunnel proxy failed",
"environmentId", tunnel.EnvironmentID,
"error", err,
)
+ // Headers are already sent, so abort the connection rather than end a truncated
+ // response as if it were complete. Echo's recover middleware re-panics this sentinel.
+ if resp, unwrapErr := echo.UnwrapResponse(c.Response()); unwrapErr == nil && resp.Committed {
+ panic(http.ErrAbortHandler)
+ }
if proxyCtx.Err() != nil {
return c.JSON(http.StatusGatewayTimeout, map[string]any{"error": "request timed out"})
}
@@ -241,28 +335,13 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) e
return c.JSON(http.StatusBadGateway, map[string]any{"error": "failed to proxy request through tunnel"})
}
- for k, v := range respHeaders {
- if !isHopByHopHeader(k) {
- c.Response().Header().Set(k, v)
- }
- }
-
- return c.Blob(status, respHeaders["Content-Type"], respBody)
+ return nil
}
// isHopByHopHeader returns true if the header should not be forwarded
func isHopByHopHeader(header string) bool {
- hopByHop := map[string]bool{
- "Connection": true,
- "Keep-Alive": true,
- "Proxy-Authenticate": true,
- "Proxy-Authorization": true,
- "Te": true,
- "Trailers": true,
- "Transfer-Encoding": true,
- "Upgrade": true,
- }
- return hopByHop[http.CanonicalHeaderKey(header)]
+ _, ok := hopByHopHeaders[http.CanonicalHeaderKey(header)]
+ return ok
}
// isBrowserSecurityHeader returns true for headers that are browser-enforced
@@ -271,17 +350,8 @@ func isHopByHopHeader(header string) bool {
// its allowed origins, returning 403. The agent authenticates via
// X-Arcane-Agent-Token instead of browser cookies/origin checks.
func isBrowserSecurityHeader(header string) bool {
- browserHeaders := map[string]bool{
- "Origin": true,
- "Referer": true,
- "Cookie": true,
- "Access-Control-Request-Method": true,
- "Access-Control-Request-Headers": true,
- "Sec-Fetch-Mode": true,
- "Sec-Fetch-Site": true,
- "Sec-Fetch-Dest": true,
- }
- return browserHeaders[http.CanonicalHeaderKey(header)]
+ _, ok := browserSecurityHeaders[http.CanonicalHeaderKey(header)]
+ return ok
}
func stripInternalTunnelHeaders(headers map[string]string) map[string]string {
@@ -290,7 +360,7 @@ func stripInternalTunnelHeaders(headers map[string]string) map[string]string {
}
cleaned := make(map[string]string, len(headers))
for k, v := range headers {
- if http.CanonicalHeaderKey(k) == "X-Arcane-Tunnel-Stream" {
+ if http.CanonicalHeaderKey(k) == tunnelStreamHeader {
continue
}
cleaned[k] = v
diff --git a/backend/pkg/libarcane/edge/remenv.go b/backend/pkg/libarcane/edge/remenv.go
index 3d8f03057..81c4d6a76 100644
--- a/backend/pkg/libarcane/edge/remenv.go
+++ b/backend/pkg/libarcane/edge/remenv.go
@@ -2,6 +2,7 @@ package edge
import (
"io"
+ "maps"
"net/http"
"strings"
@@ -12,20 +13,33 @@ import (
)
const (
- HeaderAPIKey = "X-Api-Key" // #nosec G101: header name, not a credential
- HeaderAuthorization = "Authorization"
- HeaderCookie = "Cookie"
- HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential
- HeaderUpgrade = "Upgrade"
- HeaderConnection = "Connection"
-
+ HeaderAPIKey = "X-Api-Key" // #nosec G101: header name, not a credential
+ HeaderAuthorization = "Authorization"
+ HeaderCookie = "Cookie"
+ HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential
+ HeaderUpgrade = "Upgrade"
+ HeaderConnection = "Connection"
ConnectionUpgradeToken = "upgrade"
)
-func CopyRequestHeaders(from, to http.Header, skip map[string]struct{}) {
+var (
+ // hopByHopHeaders is shared and read-only; BuildHopByHopHeaders returns a private copy.
+ hopByHopHeaders = map[string]struct{}{
+ "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {}, "Proxy-Authorization": {},
+ "Te": {}, "Trailers": {}, "Trailer": {}, "Transfer-Encoding": {}, "Upgrade": {},
+ }
+ skipHeaders = map[string]struct{}{
+ "Host": {}, "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {},
+ "Proxy-Authorization": {}, "Te": {}, "Trailer": {}, "Transfer-Encoding": {},
+ "Upgrade": {}, "Content-Length": {}, "Accept-Encoding": {}, "Origin": {}, "Referer": {},
+ "Access-Control-Request-Method": {}, "Access-Control-Request-Headers": {}, "Cookie": {},
+ }
+)
+
+func CopyRequestHeaders(from, to http.Header) {
for k, vs := range from {
ck := http.CanonicalHeaderKey(k)
- if _, ok := skip[ck]; ok || ck == http.CanonicalHeaderKey(HeaderAuthorization) || ck == http.CanonicalHeaderKey(HeaderAPIKey) {
+ if _, ok := skipHeaders[ck]; ok || ck == http.CanonicalHeaderKey(HeaderAuthorization) || ck == http.CanonicalHeaderKey(HeaderAPIKey) {
continue
}
for _, v := range vs {
@@ -122,23 +136,8 @@ func SetForwardedHeaders(req *http.Request, clientIP, host string) {
req.Header.Set("X-Forwarded-Host", host)
}
-func GetHopByHopHeaders() map[string]struct{} {
- return map[string]struct{}{
- http.CanonicalHeaderKey("Connection"): {},
- http.CanonicalHeaderKey("Keep-Alive"): {},
- http.CanonicalHeaderKey("Proxy-Authenticate"): {},
- http.CanonicalHeaderKey("Proxy-Authorization"): {},
- http.CanonicalHeaderKey("TE"): {},
- http.CanonicalHeaderKey("Trailers"): {},
- http.CanonicalHeaderKey("Trailer"): {},
- http.CanonicalHeaderKey("Transfer-Encoding"): {},
- http.CanonicalHeaderKey("Upgrade"): {},
- }
-}
-
func BuildHopByHopHeaders(respHeader http.Header) map[string]struct{} {
- hop := GetHopByHopHeaders()
-
+ hop := maps.Clone(hopByHopHeaders)
for _, connVal := range respHeader.Values("Connection") {
for token := range strings.SplitSeq(connVal, ",") {
if t := strings.TrimSpace(token); t != "" {
@@ -161,12 +160,3 @@ func CopyResponseHeaders(from, to http.Header, hop map[string]struct{}) {
}
}
}
-
-func GetSkipHeaders() map[string]struct{} {
- return map[string]struct{}{
- "Host": {}, "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {},
- "Proxy-Authorization": {}, "Te": {}, "Trailer": {}, "Transfer-Encoding": {},
- "Upgrade": {}, "Content-Length": {}, "Accept-Encoding": {}, "Origin": {}, "Referer": {},
- "Access-Control-Request-Method": {}, "Access-Control-Request-Headers": {}, "Cookie": {},
- }
-}
diff --git a/backend/pkg/libarcane/edge/remenv_test.go b/backend/pkg/libarcane/edge/remenv_test.go
index 0a1f86ce5..21f44b8ce 100644
--- a/backend/pkg/libarcane/edge/remenv_test.go
+++ b/backend/pkg/libarcane/edge/remenv_test.go
@@ -76,7 +76,6 @@ func (w *noFlushResponseWriter) Write(b []byte) (int, error) {
func (w *noFlushResponseWriter) WriteHeader(statusCode int) { w.status = statusCode }
func TestCopyRequestHeaders_SkipsExpectedHeaders(t *testing.T) {
- skip := GetSkipHeaders()
from := http.Header{}
from.Add("X-Test", "a")
from.Add("X-Test", "b")
@@ -87,7 +86,7 @@ func TestCopyRequestHeaders_SkipsExpectedHeaders(t *testing.T) {
from.Set("Transfer-Encoding", "chunked")
to := http.Header{}
- CopyRequestHeaders(from, to, skip)
+ CopyRequestHeaders(from, to)
require.Equal(t, []string{"a", "b"}, to.Values("X-Test"))
require.Empty(t, to.Get(HeaderAuthorization))
@@ -151,7 +150,7 @@ func TestSetForwardedHeaders(t *testing.T) {
}
func TestGetHopByHopHeaders_IncludesStandardHeaders(t *testing.T) {
- h := GetHopByHopHeaders()
+ h := BuildHopByHopHeaders(http.Header{})
_, ok := h[http.CanonicalHeaderKey("Connection")]
require.True(t, ok)
_, ok = h[http.CanonicalHeaderKey("Transfer-Encoding")]
@@ -189,7 +188,7 @@ func TestCopyResponseHeaders_SkipsHopByHopAndConnectionNamedHeaders(t *testing.T
}
func TestGetSkipHeaders_ContainsExpectedEntries(t *testing.T) {
- skip := GetSkipHeaders()
+ skip := skipHeaders
require.Contains(t, skip, "Host")
require.Contains(t, skip, "Connection")
require.Contains(t, skip, "Transfer-Encoding")
diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.go
index 6ae5b4e8f..f8f9469a4 100644
--- a/backend/pkg/libarcane/edge/server.go
+++ b/backend/pkg/libarcane/edge/server.go
@@ -1,13 +1,17 @@
package edge
import (
+ "cmp"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
+ "io/fs"
"log/slog"
"net/http"
+ "os"
+ "path/filepath"
"slices"
"strings"
"time"
@@ -15,7 +19,7 @@ import (
certgen "github.com/getarcaneapp/arcane/cli/v2/pkg/generate"
"github.com/labstack/echo/v5"
- "github.com/samber/mo"
+ "go.getarcane.app/acfs/atomic"
kit "go.getarcane.app/kit/pkg"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@@ -27,19 +31,20 @@ import (
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)
const (
// TunnelStaleTimeout is how long before a tunnel is considered stale.
TunnelStaleTimeout = 2 * time.Minute
- // tunnelStaleSweepInterval is how often stale tunnels are reaped. The sweep
- // is a backstop behind read-deadline liveness, so it runs well under
- // TunnelStaleTimeout to keep the dead-but-selectable window short.
+ // tunnelStaleSweepInterval backs up read-deadline liveness, so it runs well under TunnelStaleTimeout.
tunnelStaleSweepInterval = time.Minute
- // streamDeliveryTimeout bounds per-message delivery wait to a pending consumer.
- // This prevents silent data loss while avoiding indefinite blocking.
- streamDeliveryTimeout = 5 * time.Second
+ // pendingDeliveryTimeout is how long one queued message may wait for a slow consumer.
+ pendingDeliveryTimeout = 5 * time.Second
+ // maxPendingBacklogBytes and maxPendingBacklogMessages bound how far a consumer may fall behind.
+ maxPendingBacklogBytes = 32 << 20
+ maxPendingBacklogMessages = 4096
)
// EnvironmentResolver resolves an agent token to an environment ID.
@@ -55,11 +60,8 @@ type StatusUpdateCallback func(ctx context.Context, environmentID string, connec
// EventCallback is called when an edge agent publishes an event.
type EventCallback func(ctx context.Context, environmentID string, event *TunnelEvent) error
-// EnrollmentCallback is called after a successful manager-side mTLS enrollment
-// of an edge agent, allowing the host to record audit events or metrics.
-// remoteAddr is the client socket address as seen by the manager.
-// reenrolled is true when an environment that had already enrolled receives
-// assets again after the enrollment cooldown.
+// EnrollmentCallback is called after a successful manager-side edge mTLS enrollment.
+// reenrolled is true when the environment had already enrolled before the cooldown.
type EnrollmentCallback func(ctx context.Context, environmentID, remoteAddr string, certIssued, caGenerated, reenrolled bool)
// NewTunnelServerWithRegistry creates a new tunnel server using an injected tunnel registry.
@@ -76,46 +78,66 @@ func NewTunnelServerWithRegistry(registry *TunnelRegistry, resolver EnvironmentR
}
}
-// SetEnvironmentNameResolver configures environment name lookup for manager-generated assets.
-func (s *TunnelServer) SetEnvironmentNameResolver(resolver EnvironmentNameResolver) {
- if s == nil {
- return
+// GRPCServerOptions returns the receive limit and stream interceptors for the tunnel service.
+// It is served through ServeHTTP, so transport keepalive options belong on net/http.
+func (s *TunnelServer) GRPCServerOptions() []grpc.ServerOption {
+ recovery := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {
+ defer func() {
+ if panicErr := utils.PanicToError(recover()); panicErr != nil {
+ slog.ErrorContext(ss.Context(), "panic in gRPC tunnel stream", "method", info.FullMethod, "error", panicErr)
+ err = status.Error(codes.Internal, "internal tunnel error")
+ }
+ }()
+ return handler(srv, ss)
}
- s.nameResolver = resolver
-}
-// GRPCServerOptions returns the receive guardrail and stream interceptor chain
-// used by the tunnel service. Arcane serves this server through ServeHTTP, so
-// transport-level server keepalive and MaxConcurrentStreams options would be
-// inert here; those belong on net/http unless the server moves to grpc.Serve.
-func (s *TunnelServer) GRPCServerOptions() []grpc.ServerOption {
- return []grpc.ServerOption{
- grpc.MaxRecvMsgSize(maxGRPCTunnelMessageSize),
- grpc.ChainStreamInterceptor(
- s.recoveryStreamInterceptorInternal(),
- s.loggingStreamInterceptorInternal(),
- s.authStreamInterceptorInternal(),
- ),
+ logging := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+ start := time.Now()
+ err := handler(srv, ss)
+ duration := time.Since(start)
+ switch {
+ case err == nil:
+ slog.DebugContext(ss.Context(), "gRPC stream completed", "method", info.FullMethod, "duration", duration)
+ case isExpectedReceiveError(err):
+ slog.DebugContext(ss.Context(), "gRPC stream closed", "method", info.FullMethod, "duration", duration, "error", err)
+ default:
+ slog.WarnContext(ss.Context(), "gRPC stream failed", "method", info.FullMethod, "duration", duration, "error", err)
+ }
+ return err
}
-}
-// SetEventCallback configures the manager callback invoked for agent events.
-func (s *TunnelServer) SetEventCallback(callback EventCallback) {
- s.eventCallback = callback
-}
+ auth := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+ // TunnelService only exposes Connect; keep this gate aligned with tunnel.proto.
+ if info.FullMethod != tunnelpb.TunnelService_Connect_FullMethodName {
+ return handler(srv, ss)
+ }
-// SetEnrollmentCallback configures the manager callback invoked after
-// a successful edge mTLS enrollment. Passing nil clears the callback.
-func (s *TunnelServer) SetEnrollmentCallback(callback EnrollmentCallback) {
- if s == nil {
- return
+ ctx := ss.Context()
+ md, _ := metadata.FromIncomingContext(ctx)
+ token, _ := agentToken(md.Get)
+ envID, err := s.resolveEnvironment(ctx, token)
+ if err != nil {
+ return status.Error(codes.Unauthenticated, "invalid agent token")
+ }
+
+ var state *tls.ConnectionState
+ if p, ok := peer.FromContext(ctx); ok {
+ if tlsInfo, isTLS := p.AuthInfo.(credentials.TLSInfo); isTLS {
+ state = &tlsInfo.State
+ }
+ }
+ if identityErr := s.requireCertificateIdentity(state, envID); identityErr != nil {
+ return status.Error(codes.Unauthenticated, identityErr.Error())
+ }
+
+ identity := streamIdentity{environmentID: envID, securityMode: kit.Ternary(hasVerifiedPeerCertificate(state), "mtls", "token")}
+ return handler(srv, &contextualServerStream{ServerStream: ss, ctx: context.WithValue(ctx, streamIdentityKey{}, identity)})
}
- s.enrollmentCallback = callback
-}
-// SetConfig attaches edge tunnel runtime config to the manager-side server.
-func (s *TunnelServer) SetConfig(cfg *Config) {
- s.cfg = cfg
+ return []grpc.ServerOption{
+ grpc.MaxRecvMsgSize(maxGRPCTunnelMessageSize),
+ grpc.ChainStreamInterceptor(recovery, logging, auth),
+ }
}
// HandleConnect is the WebSocket handler for edge agent connections.
@@ -123,10 +145,8 @@ func (s *TunnelServer) SetConfig(cfg *Config) {
func (s *TunnelServer) HandleConnect(c *echo.Context) error {
req := c.Request()
ctx := req.Context()
- callbackCtx := context.WithoutCancel(ctx)
- // Upgrade to WebSocket. Agents authenticate with tokens/mTLS, not browser
- // cookies, so no Origin check is needed here.
+ // Agents authenticate with tokens or mTLS rather than browser cookies, so no Origin check is needed.
conn, err := wshub.Accept(c.Response(), req, nil)
if err != nil {
slog.ErrorContext(ctx, "Failed to upgrade edge tunnel connection", "error", err)
@@ -134,67 +154,60 @@ func (s *TunnelServer) HandleConnect(c *echo.Context) error {
}
tunnelConn := NewTunnelConn(conn)
- firstMsg, err := tunnelConn.Receive()
- if err != nil {
- slog.WarnContext(ctx, "Failed to receive websocket edge tunnel registration", "error", err)
+ reject := func(reason string) error {
+ if reason != "" {
+ _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Error: reason})
+ }
_ = tunnelConn.Close()
return nil
}
- if firstMsg == nil || firstMsg.Type != MessageTypeRegister {
+
+ registerMsg, err := tunnelConn.Receive()
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to receive websocket edge tunnel registration", "error", err)
+ return reject("")
+ }
+ if registerMsg == nil || registerMsg.Type != MessageTypeRegister {
slog.WarnContext(ctx, "Websocket edge tunnel missing register message")
- _ = tunnelConn.Close()
- return nil
+ return reject("")
}
- token := tokenFromHeadersInternal(req)
- if token == "" {
- // Header auth can be unavailable after the WebSocket upgrade path, so
- // the register message remains a fallback environment lookup claim.
- // In proxy-terminated mTLS deployments, the client certificate is the
- // auth credential and is consumed before the request reaches Arcane.
- token = strings.TrimSpace(firstMsg.AgentToken)
- }
+ // Headers can be lost on the upgrade path, and proxy-terminated mTLS consumes the client
+ // certificate, so the register message's token is the fallback environment claim.
+ token, _ := agentToken(req.Header.Values)
+ token = cmp.Or(token, strings.TrimSpace(registerMsg.AgentToken))
if token == "" {
slog.WarnContext(ctx, "Edge tunnel connection attempt without token")
- _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Accepted: false, Error: "agent token required"})
- _ = tunnelConn.Close()
- return nil
+ return reject("agent token required")
}
envID, err := s.resolveEnvironment(ctx, token)
if err != nil {
slog.WarnContext(ctx, "Failed to resolve agent token", "error", err)
- _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Accepted: false, Error: "invalid agent token"})
- _ = tunnelConn.Close()
- return nil
+ return reject("invalid agent token")
}
- if requireRequestCertificateIdentityErr := s.requireRequestCertificateIdentityInternal(req, envID); requireRequestCertificateIdentityErr != nil {
- slog.WarnContext(ctx, "Rejected websocket edge tunnel with mismatched client certificate", "environmentId", envID, "error", requireRequestCertificateIdentityErr)
- _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Accepted: false, Error: "client certificate does not match environment"})
- _ = tunnelConn.Close()
- return nil
+ if identityErr := s.requireCertificateIdentity(req.TLS, envID); identityErr != nil {
+ slog.WarnContext(ctx, "Rejected websocket edge tunnel with mismatched client certificate", "environmentId", envID, "error", identityErr)
+ return reject("client certificate does not match environment")
}
- tunnel := NewAgentTunnelWithConn(envID, tunnelConn)
- s.populateSessionMetadata(tunnel, firstMsg, requestSecurityModeInternal(req))
- s.manageConnectedTunnel(ctx, callbackCtx, tunnel)
+ s.manageConnectedTunnel(ctx, envID, tunnelConn, registerMsg, kit.Ternary(hasVerifiedPeerCertificate(req.TLS), "mtls", "token"))
return nil
}
-// HandleMTLSEnroll returns manager-generated edge client certificates for the
-// calling environment when generated edge mTLS is enabled. The response includes
-// private key material, so response-body logging must not be enabled here.
+// HandleMTLSEnroll returns manager-generated edge client certificates for the calling environment.
+// The response contains private keys, so response-body logging must stay disabled here.
func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
req := c.Request()
ctx := req.Context()
c.Response().Header().Set("Cache-Control", "no-store")
c.Response().Header().Set("Pragma", "no-cache")
- if s == nil || s.cfg == nil || !shouldUseGeneratedManagerCAInternal(s.cfg) {
+ if !usesGeneratedManagerCA(s.Config) {
return c.JSON(http.StatusNotFound, map[string]any{"error": "edge mTLS enrollment is not available"})
}
- token := tokenFromHeadersInternal(req)
+ token, _ := agentToken(req.Header.Values)
if token == "" {
return c.JSON(http.StatusUnauthorized, map[string]any{"error": "agent token required"})
}
@@ -206,8 +219,8 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
}
envName := ""
- if s.nameResolver != nil {
- resolvedName, resolveErr := s.nameResolver(ctx, envID)
+ if s.NameResolver != nil {
+ resolvedName, resolveErr := s.NameResolver(ctx, envID)
if resolveErr != nil {
slog.WarnContext(ctx, "Failed to resolve environment name for edge mTLS enrollment", "environmentId", envID, "error", resolveErr)
} else {
@@ -215,14 +228,37 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
}
}
- now := time.Now()
- previouslyEnrolled, enrollmentLimited, err := managerMTLSEnrollmentStateInternal(s.cfg, envID, now)
- if err != nil {
- slog.ErrorContext(ctx, "Failed to read edge mTLS enrollment state", "environmentId", envID, "error", err)
+ // The marker holds the last enrollment time; within the cooldown the existing assets are re-served.
+ failState := func(stateErr error) error {
+ slog.ErrorContext(ctx, "Failed to read edge mTLS enrollment state", "environmentId", envID, "error", stateErr)
return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to read edge mTLS enrollment state"})
}
+ assetsDir, err := edgeMTLSAssetsDir(s.Config, managerMTLSDirName)
+ if err != nil {
+ return failState(err)
+ }
+ safeEnvID := sanitizedEnvID(envID)
+ if safeEnvID == "" {
+ return failState(errors.New("environment ID is required"))
+ }
+ markerPath := filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID, generatedMTLSEnrolledName)
+ marker, readErr := os.ReadFile(markerPath)
+ if readErr != nil && !errors.Is(readErr, fs.ErrNotExist) {
+ return failState(readErr)
+ }
+ now := time.Now()
+ previouslyEnrolled := readErr == nil
+ enrollmentLimited := false
+ if trimmed := strings.TrimSpace(string(marker)); trimmed != "" {
+ enrolledAt, parseErr := time.Parse(time.RFC3339Nano, trimmed)
+ if parseErr != nil {
+ return failState(fmt.Errorf("failed to parse edge mTLS enrollment marker %s: %w", markerPath, parseErr))
+ }
+ enrollmentLimited = now.Sub(enrolledAt) < managerMTLSReenrollCooldown
+ }
+
if enrollmentLimited {
- cachedAssets, cacheErr := GenerateManagerClientMTLSAssetsWithContext(ctx, s.cfg, envID, envName)
+ cachedAssets, cacheErr := GenerateManagerClientMTLSAssetsWithContext(ctx, s.Config, envID, envName)
if cacheErr == nil && cachedAssets != nil {
slog.InfoContext(ctx, "Served cached edge mTLS enrollment during cooldown", "environmentId", envID, "remoteAddr", c.RealIP())
return c.JSON(http.StatusOK, enrollMTLSResponse{Files: cachedAssets.Files})
@@ -234,7 +270,7 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
return c.JSON(http.StatusTooManyRequests, map[string]any{"error": "edge mTLS enrollment was recently completed; retry later"})
}
- assets, err := GenerateManagerClientMTLSAssetsWithContext(ctx, s.cfg, envID, envName)
+ assets, err := GenerateManagerClientMTLSAssetsWithContext(ctx, s.Config, envID, envName)
if err != nil {
slog.ErrorContext(ctx, "Failed to generate edge mTLS enrollment assets", "environmentId", envID, "error", err)
return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to generate edge mTLS assets"})
@@ -243,8 +279,8 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
return c.JSON(http.StatusNotFound, map[string]any{"error": "edge mTLS enrollment assets unavailable"})
}
assets.Reenrolled = previouslyEnrolled
- if recordManagerMTLSEnrollmentErr := recordManagerMTLSEnrollmentInternal(s.cfg, envID, now); recordManagerMTLSEnrollmentErr != nil {
- slog.ErrorContext(ctx, "Failed to record edge mTLS enrollment state", "environmentId", envID, "error", recordManagerMTLSEnrollmentErr)
+ if recordErr := atomic.WriteFile(markerPath, []byte(now.UTC().Format(time.RFC3339Nano)+"\n"), 0o600); recordErr != nil {
+ slog.ErrorContext(ctx, "Failed to record edge mTLS enrollment state", "environmentId", envID, "error", recordErr)
return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to record edge mTLS enrollment state"})
}
if assets.Reenrolled {
@@ -253,8 +289,8 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
slog.InfoContext(ctx, "Edge mTLS certificate assets enrolled", "environmentId", envID, "remoteAddr", c.RealIP(), "certIssued", assets.CertIssued)
}
- if s.enrollmentCallback != nil {
- s.enrollmentCallback(context.WithoutCancel(ctx), envID, c.RealIP(), assets.CertIssued, assets.CAGenerated, assets.Reenrolled)
+ if s.EnrollmentCallback != nil {
+ s.EnrollmentCallback(context.WithoutCancel(ctx), envID, c.RealIP(), assets.CertIssued, assets.CAGenerated, assets.Reenrolled)
}
return c.JSON(http.StatusOK, enrollMTLSResponse{Files: assets.Files})
@@ -263,7 +299,6 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {
// Connect is the gRPC bidi stream handler for edge agent connections.
func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMessage, tunnelpb.ManagerMessage]) error {
ctx := stream.Context()
- callbackCtx := context.WithoutCancel(ctx)
firstMsg, err := stream.Recv()
if err != nil {
@@ -273,51 +308,23 @@ func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMes
return err
}
- register := firstMsg.GetRegister()
- if register == nil {
+ registerMsg, err := agentProtoToTunnelMessage(firstMsg)
+ if err != nil || registerMsg.Type != MessageTypeRegister {
return status.Error(codes.Unauthenticated, "first message must be register")
}
- envID, ok := resolvedEnvironmentIDFromContextInternal(ctx).Get()
- if !ok || envID == "" {
+ // The auth interceptor already verified the token and client certificate for this stream.
+ identity, _ := ctx.Value(streamIdentityKey{}).(streamIdentity)
+ if strings.TrimSpace(identity.environmentID) == "" {
return status.Error(codes.Unauthenticated, "authenticated environment is missing from stream context")
}
- if requireCertificateIdentityFromContextErr := s.requireCertificateIdentityFromContextInternal(ctx, envID); requireCertificateIdentityFromContextErr != nil {
- return status.Error(codes.Unauthenticated, requireCertificateIdentityFromContextErr.Error())
- }
managerConn := NewGRPCManagerTunnelConn(stream)
- managerConn.SetParityEncoding(slices.Contains(register.GetCapabilities(), tunnelCapabilityProtoParity))
- tunnel := NewAgentTunnelWithConn(envID, managerConn)
- s.populateSessionMetadata(tunnel, &TunnelMessage{
- AgentInstance: register.GetAgentInstanceId(),
- Capabilities: append([]string(nil), register.GetCapabilities()...),
- ResumeSession: register.GetResumeSessionId(),
- }, securityModeFromGRPCContextInternal(ctx))
- s.manageConnectedTunnel(ctx, callbackCtx, tunnel)
+ managerConn.parity = slices.Contains(registerMsg.Capabilities, tunnelCapabilityProtoParity)
+ s.manageConnectedTunnel(ctx, identity.environmentID, managerConn, registerMsg, identity.securityMode)
return nil
}
-func (s *TunnelServer) populateSessionMetadata(tunnel *AgentTunnel, registerMsg *TunnelMessage, securityMode string) {
- if tunnel == nil {
- return
- }
-
- tunnel.SessionID = uuid.New().String()
- tunnel.SecurityMode = securityMode
- if tunnel.SecurityMode == "" {
- tunnel.SecurityMode = "token"
- }
- if registerMsg != nil {
- tunnel.AgentInstance = strings.TrimSpace(registerMsg.AgentInstance)
- tunnel.Capabilities = append([]string(nil), registerMsg.Capabilities...)
- }
-
- if tunnel.Conn != nil {
- tunnel.Transport = tunnel.Conn.Transport()
- }
-}
-
func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (string, error) {
if s.resolver == nil {
return "", errors.New("edge resolver is not configured")
@@ -329,257 +336,218 @@ func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (st
return s.resolver(ctx, token)
}
-// agentTokenHeaderPriority is the shared credential fallback order for both
-// header- and metadata-based token extraction, so the transports cannot drift.
-var agentTokenHeaderPriority = []string{HeaderAgentToken, HeaderAPIKey}
-
-func tokenFromMetadataInternal(ctx context.Context) string {
- md, ok := metadata.FromIncomingContext(ctx)
- if !ok {
- return ""
- }
- for _, header := range agentTokenHeaderPriority {
- values := md.Get(strings.ToLower(header))
- for _, value := range values {
- trimmed := strings.TrimSpace(value)
- if trimmed != "" {
- return trimmed
- }
+// agentToken returns the first agent token from X-Arcane-Agent-Token, X-API-Key, then
+// "Authorization: Bearer", for proxies that strip custom headers, plus the header that supplied it.
+func agentToken(values func(key string) []string) (string, string) {
+ for _, header := range []string{HeaderAgentToken, HeaderAPIKey} {
+ if tokens := kit.TrimNonEmpty(values(header)); len(tokens) > 0 {
+ return tokens[0], header
}
}
- for _, value := range md.Get(strings.ToLower(HeaderAuthorization)) {
+ for _, value := range values(HeaderAuthorization) {
if token := remenv.ExtractBearerToken(value); token != "" {
- return token
- }
- }
- return ""
-}
-
-func tokenFromHeadersInternal(req *http.Request) string {
- token, _ := tokenFromHeadersWithSourceInternal(req)
- return token
-}
-
-// tokenFromHeadersWithSourceInternal returns the agent token along with the
-// canonical header name that supplied it. Falls back from X-Arcane-Agent-Token
-// to X-API-Key to "Authorization: Bearer <token>" so deployments behind
-// reverse proxies that strip custom X- headers can still authenticate.
-func tokenFromHeadersWithSourceInternal(req *http.Request) (string, string) {
- if req == nil {
- return "", ""
- }
- for _, header := range agentTokenHeaderPriority {
- if token := strings.TrimSpace(req.Header.Get(header)); token != "" {
- return token, header
+ return token, HeaderAuthorization
}
}
- if token := remenv.ExtractBearerToken(req.Header.Get(HeaderAuthorization)); token != "" {
- return token, HeaderAuthorization
- }
return "", ""
}
-func (s *TunnelServer) manageConnectedTunnel(ctx, callbackCtx context.Context, tunnel *AgentTunnel) {
+// manageConnectedTunnel registers an authenticated agent session and serves its messages until it disconnects.
+func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string, conn TunnelConnection, registerMsg *TunnelMessage, securityMode string) {
+ callbackCtx := context.WithoutCancel(ctx)
+ tunnel := NewAgentTunnelWithConn(envID, conn)
+ tunnel.SessionID = uuid.New().String()
+ tunnel.SecurityMode = securityMode
+ tunnel.AgentInstance = strings.TrimSpace(registerMsg.AgentInstance)
+ tunnel.Capabilities = registerMsg.Capabilities
+ tunnel.Transport = conn.Transport()
+
accepted, drainPrevious, rejectReason, err := s.registry.RegisterSession(callbackCtx, tunnel, TunnelStaleTimeout)
if err != nil {
- slog.ErrorContext(ctx, "Failed to register edge agent session",
- "environmentId", tunnel.EnvironmentID,
- "agentInstanceId", tunnel.AgentInstance,
- "error", err,
- )
+ slog.ErrorContext(ctx, "Failed to register edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "error", err)
_ = tunnel.CloseWithReason("edge agent session registration unavailable")
return
}
if !accepted {
- slog.WarnContext(ctx, "Rejected duplicate edge agent session",
- "environmentId", tunnel.EnvironmentID,
- "agentInstanceId", tunnel.AgentInstance,
- "reason", rejectReason,
- )
- _ = tunnel.Conn.Send(&TunnelMessage{
- Type: MessageTypeRegisterResponse,
- Accepted: false,
- EnvironmentID: tunnel.EnvironmentID,
- Error: rejectReason,
- })
+ slog.WarnContext(ctx, "Rejected duplicate edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "reason", rejectReason)
+ _ = conn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, EnvironmentID: envID, Error: rejectReason})
_ = tunnel.CloseWithReason(rejectReason)
return
}
- slog.InfoContext(ctx, "Edge agent connected",
- "environmentId", tunnel.EnvironmentID,
- "sessionId", tunnel.SessionID,
- "securityMode", tunnel.SecurityMode,
- )
+ defer func() {
+ removed, active := s.registry.UnregisterCurrent(callbackCtx, envID, tunnel)
+ if !removed {
+ return
+ }
+ slog.InfoContext(ctx, "Edge agent disconnected", "environmentId", envID, "sessionId", tunnel.SessionID)
+ if !active {
+ s.updateConnectionStatus(callbackCtx, tunnel, false)
+ }
+ }()
- // Echo the agent's capabilities and append the manager's own so the agent
- // learns what this manager supports (older agents ignore extra strings).
- capabilities := kit.Unique(append(slices.Clone(tunnel.Capabilities), tunnelCapabilityProtoParity, tunnelCapabilityChunkedRequest))
+ slog.InfoContext(ctx, "Edge agent connected", "environmentId", envID, "sessionId", tunnel.SessionID, "securityMode", tunnel.SecurityMode)
- if sendErr := tunnel.Conn.Send(&TunnelMessage{
+ // Echo the agent's capabilities plus the manager's own; older agents ignore extra strings.
+ capabilities := kit.Unique(append(slices.Clone(tunnel.Capabilities), tunnelCapabilityProtoParity, tunnelCapabilityChunkedRequest, tunnelCapabilityCommandCreditGrant))
+ if sendErr := conn.Send(&TunnelMessage{
Type: MessageTypeRegisterResponse,
Accepted: true,
- EnvironmentID: tunnel.EnvironmentID,
+ EnvironmentID: envID,
SessionID: tunnel.SessionID,
SecurityMode: tunnel.SecurityMode,
Capabilities: capabilities,
DrainPrevious: drainPrevious,
}); sendErr != nil {
- slog.WarnContext(ctx, "Failed to send register response", "environmentId", tunnel.EnvironmentID, "error", sendErr)
+ slog.WarnContext(ctx, "Failed to send register response", "environmentId", envID, "error", sendErr)
_ = tunnel.CloseWithReason("")
- removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)
- if removed && !active {
- s.updateConnectionStatusInternal(callbackCtx, tunnel, false)
- }
return
}
+ s.updateConnectionStatus(callbackCtx, tunnel, true)
- s.updateConnectionStatusInternal(callbackCtx, tunnel, true)
-
- defer func() {
- removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)
- if !removed {
+ deliver := func(msg *TunnelMessage, closeStream bool) {
+ value, _ := tunnel.Pending.Load(msg.ID)
+ if pending, ok := value.(*PendingRequest); ok {
+ pending.enqueue(ctx, tunnel, msg, closeStream)
return
}
- slog.InfoContext(ctx, "Edge agent disconnected", "environmentId", tunnel.EnvironmentID, "sessionId", tunnel.SessionID)
- if !active {
- s.updateConnectionStatusInternal(callbackCtx, tunnel, false)
- }
- }()
-
- s.messageLoop(ctx, tunnel)
-}
-
-// messageLoop processes incoming messages from the agent.
-func (s *TunnelServer) messageLoop(ctx context.Context, tunnel *AgentTunnel) {
- // One reusable timer for stream delivery deadlines. time.After per message
- // left a live 5s runtime timer for every frame, so a chatty log tail through
- // a tunnel accumulated thousands of them. This loop is the only user, so the
- // timer is never touched concurrently.
- deliveryTimer := time.NewTimer(streamDeliveryTimeout)
- deliveryTimer.Stop()
- defer deliveryTimer.Stop()
+ slog.DebugContext(ctx, "Received message for unknown request", "id", msg.ID, "type", msg.Type)
+ }
- for {
- select {
- case <-ctx.Done():
- return
- default:
- msg, err := tunnel.Conn.Receive()
- if err != nil {
- if !tunnel.Conn.IsExpectedReceiveError(err) {
- slog.WarnContext(ctx, "Error receiving from edge tunnel", "environmentId", tunnel.EnvironmentID, "error", err)
- }
- return
+ for ctx.Err() == nil {
+ msg, receiveErr := conn.Receive()
+ if receiveErr != nil {
+ if !conn.IsExpectedReceiveError(receiveErr) {
+ slog.WarnContext(ctx, "Error receiving from edge tunnel", "environmentId", envID, "error", receiveErr)
}
-
- s.handleTunnelMessage(ctx, tunnel, msg, deliveryTimer)
+ return
}
- }
-}
-func (s *TunnelServer) handleTunnelMessage(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, deliveryTimer *time.Timer) {
- switch msg.Type {
- case MessageTypeHeartbeat:
- s.handleHeartbeat(ctx, tunnel, msg)
- case MessageTypeResponse:
- s.deliverResponse(ctx, tunnel, msg)
- case MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete, MessageTypeFileChunk:
- s.deliverResponse(ctx, tunnel, msg)
- case MessageTypeEvent:
- s.handleEvent(ctx, tunnel, msg)
- case MessageTypeStreamData, MessageTypeStreamEnd, MessageTypeWebSocketData, MessageTypeWebSocketClose, MessageTypeStreamClose:
- s.deliverStream(ctx, tunnel, msg, deliveryTimer)
- case MessageTypeRequest, MessageTypeHeartbeatAck, MessageTypeWebSocketStart, MessageTypeRegisterResponse, MessageTypeCommandRequest, MessageTypeStreamOpen, MessageTypeCancelRequest:
- slog.DebugContext(ctx, "Ignoring message type from agent", "type", msg.Type, "environmentId", tunnel.EnvironmentID)
- case MessageTypeRegister:
- slog.DebugContext(ctx, "Ignoring duplicate register message from agent", "environmentId", tunnel.EnvironmentID)
- default:
- slog.WarnContext(ctx, "Unknown message type from agent", "type", msg.Type, "environmentId", tunnel.EnvironmentID)
+ switch msg.Type {
+ case MessageTypeHeartbeat:
+ tunnel.UpdateHeartbeat()
+ if ackErr := conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeHeartbeatAck}); ackErr != nil {
+ slog.WarnContext(ctx, "Failed to send heartbeat ack", "error", ackErr)
+ }
+ case MessageTypeResponse, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete, MessageTypeFileChunk:
+ deliver(msg, false)
+ case MessageTypeStreamData, MessageTypeStreamEnd, MessageTypeWebSocketData, MessageTypeWebSocketClose, MessageTypeStreamClose:
+ deliver(msg, true)
+ case MessageTypeEvent:
+ if msg.Event == nil {
+ slog.WarnContext(ctx, "Received event message without payload", "environmentId", envID)
+ continue
+ }
+ if s.EventCallback == nil {
+ continue
+ }
+ event := cloneTunnelEvent(msg.Event)
+ go func() {
+ eventCtx, cancel := context.WithTimeout(callbackCtx, 15*time.Second)
+ defer cancel()
+ if eventErr := s.EventCallback(eventCtx, envID, event); eventErr != nil {
+ slog.WarnContext(eventCtx, "Failed to process edge event", "environmentId", envID, "type", event.Type, "error", eventErr)
+ }
+ }()
+ case MessageTypeRequest, MessageTypeHeartbeatAck, MessageTypeWebSocketStart, MessageTypeRegisterResponse,
+ MessageTypeCommandRequest, MessageTypeStreamOpen, MessageTypeCancelRequest, MessageTypeRegister, MessageTypeCommandCredit:
+ slog.DebugContext(ctx, "Ignoring message type from agent", "type", msg.Type, "environmentId", envID)
+ default:
+ slog.WarnContext(ctx, "Unknown message type from agent", "type", msg.Type, "environmentId", envID)
+ }
}
}
-func (s *TunnelServer) handleHeartbeat(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage) {
- tunnel.UpdateHeartbeat()
- ack := &TunnelMessage{
- ID: msg.ID,
- Type: MessageTypeHeartbeatAck,
- }
- if err := tunnel.Conn.Send(ack); err != nil {
- slog.WarnContext(ctx, "Failed to send heartbeat ack", "error", err)
+// enqueue hands msg to the consumer without blocking the tunnel. Behind a full channel, messages
+// wait in order for up to pendingDeliveryTimeout each. A consumer that stalls or falls past the
+// backlog limits is failed once, and a stream also tells the agent to stop. Queued messages
+// outlive a tunnel close so the collector can still drain a fully received response.
+func (p *PendingRequest) enqueue(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, closeStream bool) {
+ retained := func(m *TunnelMessage) int {
+ size := len(m.Body) + len(m.Error)
+ for k, v := range m.Headers {
+ size += len(k) + len(v)
+ }
+ for k, v := range m.Metadata {
+ size += len(k) + len(v)
+ }
+ return size
}
-}
-func (s *TunnelServer) deliverResponse(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage) {
- if req, ok := tunnel.Pending.Load(msg.ID); ok {
- pending, isPending := req.(*PendingRequest)
- if !isPending {
+ fail := func(stalled error) {
+ // A consumer that already finished has nothing left to fail.
+ if !tunnel.Pending.CompareAndDelete(msg.ID, p) {
return
}
select {
- case pending.ResponseCh <- msg:
+ case p.failureCh <- stalled:
default:
- err := fmt.Errorf("response delivery failed because pending request %s is not consuming messages", msg.ID)
- tunnel.Pending.Delete(msg.ID)
- pending.failureCh <- err
- slog.WarnContext(ctx, "Failed pending request with full response channel", "id", msg.ID)
}
- return
- }
- slog.WarnContext(ctx, "Received response for unknown request", "id", msg.ID)
-}
-
-func (s *TunnelServer) deliverStream(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, deliveryTimer *time.Timer) {
- if req, ok := tunnel.Pending.Load(msg.ID); ok {
- pending, isPending := req.(*PendingRequest)
- if !isPending {
+ slog.WarnContext(ctx, "Failed slow pending request consumer", "id", msg.ID, "type", msg.Type, "error", stalled)
+ if !closeStream {
return
}
+ if closeErr := tunnel.Conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeStreamClose, Error: stalled.Error()}); closeErr != nil {
+ slog.DebugContext(ctx, "Failed to close slow edge stream", "id", msg.ID, "error", closeErr)
+ }
+ }
- // Go 1.23+ timers drop any stale value on Stop/Reset, so no drain is needed.
- deliveryTimer.Stop()
- deliveryTimer.Reset(streamDeliveryTimeout)
- defer deliveryTimer.Stop()
-
+ p.mu.Lock()
+ if p.failed {
+ p.mu.Unlock()
+ return
+ }
+ if !p.draining {
select {
- case pending.ResponseCh <- msg:
- case <-ctx.Done():
+ case p.ResponseCh <- msg:
+ p.mu.Unlock()
return
- case <-deliveryTimer.C:
- err := fmt.Errorf("stream delivery timed out for pending request %s", msg.ID)
- tunnel.Pending.Delete(msg.ID)
- pending.failureCh <- err
- slog.WarnContext(ctx, "Failed slow pending stream consumer",
- "id", msg.ID,
- "type", msg.Type,
- "timeout", streamDeliveryTimeout,
- )
- if sendErr := tunnel.Conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeStreamClose, Error: err.Error()}); sendErr != nil {
- slog.DebugContext(ctx, "Failed to close slow edge stream", "id", msg.ID, "error", sendErr)
- }
+ default:
}
- return
+ p.draining = true
+ go func() {
+ timer := time.NewTimer(pendingDeliveryTimeout)
+ defer timer.Stop()
+ for {
+ p.mu.Lock()
+ if p.failed || len(p.backlog) == 0 {
+ p.draining = false
+ p.mu.Unlock()
+ return
+ }
+ next := p.backlog[0]
+ p.backlog[0] = nil
+ p.backlog = p.backlog[1:]
+ p.backlogBytes -= retained(next)
+ p.mu.Unlock()
+
+ timer.Reset(pendingDeliveryTimeout)
+ select {
+ case p.ResponseCh <- next:
+ case <-timer.C:
+ p.mu.Lock()
+ stalled := !p.failed
+ p.failed, p.backlog = true, nil
+ p.mu.Unlock()
+ if stalled {
+ fail(fmt.Errorf("consumer for pending request %s stalled for %s", next.ID, pendingDeliveryTimeout))
+ }
+ return
+ }
+ }
+ }()
}
- slog.DebugContext(ctx, "Received stream message for unknown request", "id", msg.ID, "type", msg.Type)
-}
-func (s *TunnelServer) handleEvent(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage) {
- if msg.Event == nil {
- slog.WarnContext(ctx, "Received event message without payload", "environmentId", tunnel.EnvironmentID)
- return
+ p.backlog = append(p.backlog, msg)
+ p.backlogBytes += retained(msg)
+ overflow := p.backlogBytes > maxPendingBacklogBytes || len(p.backlog) > maxPendingBacklogMessages
+ if overflow {
+ p.failed, p.backlog = true, nil
}
- if s.eventCallback == nil {
- return
+ p.mu.Unlock()
+ if overflow {
+ fail(fmt.Errorf("consumer for pending request %s fell more than %d messages or %d bytes behind", msg.ID, maxPendingBacklogMessages, maxPendingBacklogBytes))
}
-
- eventCopy := cloneTunnelEvent(msg.Event)
- go func() {
- eventCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
- defer cancel()
- if err := s.eventCallback(eventCtx, tunnel.EnvironmentID, eventCopy); err != nil {
- slog.WarnContext(eventCtx, "Failed to process edge event", "environmentId", tunnel.EnvironmentID, "type", eventCopy.Type, "error", err)
- }
- }()
}
// StartCleanupLoop periodically cleans up stale tunnels.
@@ -595,7 +563,7 @@ func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {
case <-ticker.C:
removed := s.registry.CleanupStale(ctx, TunnelStaleTimeout)
for _, tunnel := range removed {
- s.updateConnectionStatusInternal(context.WithoutCancel(ctx), tunnel, false)
+ s.updateConnectionStatus(context.WithoutCancel(ctx), tunnel, false)
}
if len(removed) > 0 {
slog.InfoContext(ctx, "Cleaned up stale tunnels", "count", len(removed))
@@ -604,8 +572,10 @@ func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {
}
}
-func (s *TunnelServer) updateConnectionStatusInternal(ctx context.Context, tunnel *AgentTunnel, connected bool) {
- if s == nil || s.statusCallback == nil || tunnel == nil {
+// updateConnectionStatus reports a connect or disconnect, skipping a disconnect
+// when a newer live tunnel already serves the environment.
+func (s *TunnelServer) updateConnectionStatus(ctx context.Context, tunnel *AgentTunnel, connected bool) {
+ if s.statusCallback == nil {
return
}
@@ -624,135 +594,40 @@ func (s *TunnelServer) WaitForCleanupDone() {
<-s.cleanupDone
}
-func (s *TunnelServer) authStreamInterceptorInternal() grpc.StreamServerInterceptor {
- // TunnelService currently exposes only Connect. Keep this explicit method
- // gate aligned with tunnel.proto if new RPCs are added.
- return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
- if info == nil || info.FullMethod != tunnelpb.TunnelService_Connect_FullMethodName {
- return handler(srv, ss)
- }
-
- token := tokenFromMetadataInternal(ss.Context())
- envID, err := s.resolveEnvironment(ss.Context(), token)
- if err != nil {
- return status.Error(codes.Unauthenticated, "invalid agent token")
- }
- if requireCertificateIdentityFromContextErr := s.requireCertificateIdentityFromContextInternal(ss.Context(), envID); requireCertificateIdentityFromContextErr != nil {
- return status.Error(codes.Unauthenticated, requireCertificateIdentityFromContextErr.Error())
- }
-
- ctx := context.WithValue(ss.Context(), resolvedEnvironmentIDKey{}, envID)
- return handler(srv, &contextualServerStream{ServerStream: ss, ctx: ctx})
- }
-}
-
-func (s *TunnelServer) loggingStreamInterceptorInternal() grpc.StreamServerInterceptor {
- return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
- start := time.Now()
- err := handler(srv, ss)
- duration := time.Since(start)
-
- if err != nil {
- if isExpectedGRPCReceiveErrorInternal(err) {
- slog.DebugContext(ss.Context(), "gRPC stream closed", "method", info.FullMethod, "duration", duration, "error", err)
- } else {
- slog.WarnContext(ss.Context(), "gRPC stream failed", "method", info.FullMethod, "duration", duration, "error", err)
- }
- return err
- }
-
- slog.DebugContext(ss.Context(), "gRPC stream completed", "method", info.FullMethod, "duration", duration)
- return nil
- }
-}
-
-func (s *TunnelServer) recoveryStreamInterceptorInternal() grpc.StreamServerInterceptor {
- return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {
- defer func() {
- if panicErr := utils.PanicToError(recover()); panicErr != nil {
- slog.ErrorContext(ss.Context(), "panic in gRPC tunnel stream",
- "method", info.FullMethod,
- "error", panicErr,
- )
- err = status.Error(codes.Internal, "internal tunnel error")
- }
- }()
-
- return handler(srv, ss)
- }
-}
-
func (s *contextualServerStream) Context() context.Context {
return s.ctx
}
-func resolvedEnvironmentIDFromContextInternal(ctx context.Context) mo.Option[string] {
- if ctx == nil {
- return mo.None[string]()
- }
-
- envID, ok := ctx.Value(resolvedEnvironmentIDKey{}).(string)
- if !ok || strings.TrimSpace(envID) == "" {
- return mo.None[string]()
+// requireCertificateIdentity checks a direct-TLS client certificate against envID. A nil state
+// means TLS ended before Arcane, such as at a proxy, so only the token applies.
+func (s *TunnelServer) requireCertificateIdentity(state *tls.ConnectionState, envID string) error {
+ mode := EdgeMTLSModeDisabled
+ if s.Config != nil {
+ mode = NormalizeEdgeMTLSMode(s.Config.EdgeMTLSMode)
}
-
- return mo.Some(envID)
-}
-
-func (s *TunnelServer) requireCertificateIdentityInternal(state *tls.ConnectionState, envID string) error {
- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeDisabled {
+ if mode == EdgeMTLSModeDisabled || state == nil {
return nil
}
- return verifiedPeerCertificateEnvironmentIDMatchesInternal(state, envID, certgen.EdgeMTLSTrustDomain(edgeMTLSAppURLInternal(s.cfg)))
-}
-
-func (s *TunnelServer) requireRequestCertificateIdentityInternal(req *http.Request, envID string) error {
- if req == nil || req.TLS == nil {
- return nil
- }
- if !hasVerifiedPeerCertificateInternal(req.TLS) {
- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeRequired {
+ if !hasVerifiedPeerCertificate(state) {
+ if mode == EdgeMTLSModeRequired {
return errors.New("verified edge mTLS client certificate is required")
}
return nil
}
- return s.requireCertificateIdentityInternal(req.TLS, envID)
-}
-func (s *TunnelServer) requireCertificateIdentityFromContextInternal(ctx context.Context, envID string) error {
- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeDisabled {
- return nil
+ safeEnvID := sanitizedEnvID(envID)
+ if safeEnvID == "" {
+ return errors.New("environment ID is required for edge mTLS certificate identity check")
}
- p, ok := peer.FromContext(ctx)
- if !ok {
- return nil
+ appURL := cmp.Or(strings.TrimSpace(s.Config.AppURL), strings.TrimSpace(httpx.ManagerBaseURL(s.Config.ManagerApiUrl)))
+ expected := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)
+ if expected == nil {
+ return errors.New("edge mTLS trust domain is required for certificate identity check")
}
- tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo)
- if !ok {
- return nil
- }
- if !hasVerifiedPeerCertificateInternal(&tlsInfo.State) {
- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeRequired {
- return errors.New("verified edge mTLS client certificate is required")
- }
- return nil
- }
- return verifiedPeerCertificateEnvironmentIDMatchesInternal(&tlsInfo.State, envID, certgen.EdgeMTLSTrustDomain(edgeMTLSAppURLInternal(s.cfg)))
-}
-
-func (s *TunnelServer) edgeMTLSModeInternal() string {
- if s == nil || s.cfg == nil {
- return EdgeMTLSModeDisabled
- }
- return s.cfg.EdgeMTLSMode
-}
-
-func securityModeFromGRPCContextInternal(ctx context.Context) string {
- p, ok := peer.FromContext(ctx)
- if !ok {
- return "token"
+ if !certificateHasURISAN(state.VerifiedChains[0][0], expected) {
+ return fmt.Errorf("verified edge mTLS client certificate does not match environment %s", strings.TrimSpace(envID))
}
- return grpcContextSecurityModeInternal(*p)
+ return nil
}
// IsInternalTunnelRequest reports whether a request is being dispatched by the
diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.go
index 26ce4a991..5d11d88d3 100644
--- a/backend/pkg/libarcane/edge/server_test.go
+++ b/backend/pkg/libarcane/edge/server_test.go
@@ -10,6 +10,7 @@ import (
"net/http"
"net/http/httptest"
"os"
+ "path/filepath"
"strings"
"testing"
"time"
@@ -174,10 +175,10 @@ func TestTunnelServer_HandleConnect_AcceptsTokenAfterProxyTerminatedMTLS(t *test
}
return "", errors.New("invalid token")
}, nil)
- server.SetConfig(&Config{
+ server.Config = &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
AppURL: "https://manager.example.com",
- })
+ }
router := echo.New()
router.GET("/connect", server.HandleConnect)
@@ -334,14 +335,14 @@ func TestTunnelServer_HandleMTLSEnroll(t *testing.T) {
}
return "env-mtls", nil
}, nil)
- server.SetEnvironmentNameResolver(func(ctx context.Context, environmentID string) (string, error) {
+ server.NameResolver = func(ctx context.Context, environmentID string) (string, error) {
require.Equal(t, "env-mtls", environmentID)
return "Lab Server", nil
- })
- server.SetConfig(&Config{
+ }
+ server.Config = &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
EdgeMTLSAssetsDir: t.TempDir(),
- })
+ }
router := echo.New()
router.POST("/enroll", server.HandleMTLSEnroll)
@@ -378,10 +379,10 @@ func TestTunnelServer_HandleMTLSEnroll_ServesCachedAssetsDuringCooldown(t *testi
}
return "env-repeat", nil
}, nil)
- server.SetConfig(&Config{
+ server.Config = &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
EdgeMTLSAssetsDir: t.TempDir(),
- })
+ }
router := echo.New()
router.POST("/enroll", server.HandleMTLSEnroll)
@@ -418,13 +419,13 @@ func TestTunnelServer_HandleMTLSEnroll_AllowsRepeatAfterCooldownAndMarksReenroll
}
return "env-repeat-old", nil
}, nil)
- server.SetConfig(cfg)
+ server.Config = cfg
reenrolled := make(chan bool, 2)
- server.SetEnrollmentCallback(func(ctx context.Context, environmentID, remoteAddr string, certIssued, caGenerated, wasReenrolled bool) {
+ server.EnrollmentCallback = func(ctx context.Context, environmentID, remoteAddr string, certIssued, caGenerated, wasReenrolled bool) {
require.Equal(t, "env-repeat-old", environmentID)
reenrolled <- wasReenrolled
- })
+ }
router := echo.New()
router.POST("/enroll", server.HandleMTLSEnroll)
@@ -436,8 +437,7 @@ func TestTunnelServer_HandleMTLSEnroll_AllowsRepeatAfterCooldownAndMarksReenroll
require.Equal(t, http.StatusOK, w.Code)
require.False(t, <-reenrolled)
- markerPath, err := managerMTLSEnrollmentMarkerPathInternal(cfg, "env-repeat-old")
- require.NoError(t, err)
+ markerPath := filepath.Join(cfg.EdgeMTLSAssetsDir, generatedClientMTLSSubdir, "env-repeat-old", generatedMTLSEnrolledName)
oldEnrollment := time.Now().Add(-(managerMTLSReenrollCooldown + time.Minute)).UTC().Format(time.RFC3339Nano) + "\n"
require.NoError(t, os.WriteFile(markerPath, []byte(oldEnrollment), 0o600))
@@ -494,65 +494,71 @@ func TestTunnelServer_resolveEnvironment_Errors(t *testing.T) {
})
}
-func TestTokenFromMetadata(t *testing.T) {
+func TestAgentToken(t *testing.T) {
t.Run("prefers agent token and trims whitespace", func(t *testing.T) {
- ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs(
+ md := metadata.Pairs(
strings.ToLower(HeaderAgentToken), " agent-token ",
strings.ToLower(HeaderAPIKey), "api-token",
- ))
+ )
- assert.Equal(t, "agent-token", tokenFromMetadataInternal(ctx))
+ token, source := agentToken(md.Get)
+ assert.Equal(t, "agent-token", token)
+ assert.Equal(t, HeaderAgentToken, source)
})
t.Run("falls back to api key", func(t *testing.T) {
- ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs(
- strings.ToLower(HeaderAPIKey), "api-token",
- ))
+ md := metadata.Pairs(strings.ToLower(HeaderAPIKey), "api-token")
- assert.Equal(t, "api-token", tokenFromMetadataInternal(ctx))
+ token, source := agentToken(md.Get)
+ assert.Equal(t, "api-token", token)
+ assert.Equal(t, HeaderAPIKey, source)
})
t.Run("returns empty when metadata missing", func(t *testing.T) {
- assert.Empty(t, tokenFromMetadataInternal(t.Context()))
+ var md metadata.MD
+ token, source := agentToken(md.Get)
+ assert.Empty(t, token)
+ assert.Empty(t, source)
})
}
func TestIsExpectedTunnelReceiveError(t *testing.T) {
t.Run("nil is not expected", func(t *testing.T) {
- assert.False(t, isExpectedGRPCReceiveErrorInternal(nil))
+ assert.False(t, isExpectedReceiveError(nil))
})
t.Run("io eof expected", func(t *testing.T) {
- assert.True(t, isExpectedGRPCReceiveErrorInternal(io.EOF))
+ assert.True(t, isExpectedReceiveError(io.EOF))
})
t.Run("context canceled expected", func(t *testing.T) {
- assert.True(t, isExpectedGRPCReceiveErrorInternal(context.Canceled))
+ assert.True(t, isExpectedReceiveError(context.Canceled))
})
t.Run("context deadline expected", func(t *testing.T) {
- assert.True(t, isExpectedGRPCReceiveErrorInternal(context.DeadlineExceeded))
+ assert.True(t, isExpectedReceiveError(context.DeadlineExceeded))
})
t.Run("grpc canceled expected", func(t *testing.T) {
err := status.Error(codes.Canceled, "context canceled")
- assert.True(t, isExpectedGRPCReceiveErrorInternal(err))
+ assert.True(t, isExpectedReceiveError(err))
})
t.Run("grpc deadline exceeded expected", func(t *testing.T) {
err := status.Error(codes.DeadlineExceeded, "deadline exceeded")
- assert.True(t, isExpectedGRPCReceiveErrorInternal(err))
+ assert.True(t, isExpectedReceiveError(err))
})
t.Run("unexpected error not expected", func(t *testing.T) {
- assert.False(t, isExpectedGRPCReceiveErrorInternal(errors.New("boom")))
+ assert.False(t, isExpectedReceiveError(errors.New("boom")))
})
}
func TestTunnelServer_HandleEventCallback(t *testing.T) {
called := make(chan struct{}, 1)
- server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)
- server.SetEventCallback(func(ctx context.Context, environmentID string, event *TunnelEvent) error {
+ registry := NewTunnelRegistry()
+ server := NewTunnelServerWithRegistry(registry, nil, nil)
+ server.EventCallback = func(ctx context.Context, environmentID string, event *TunnelEvent) error {
assert.Equal(t, "env-edge", environmentID)
require.NotNil(t, event)
assert.Equal(t, "container.start", event.Type)
@@ -562,32 +568,43 @@ func TestTunnelServer_HandleEventCallback(t *testing.T) {
default:
}
return nil
- })
+ }
- tunnel := NewAgentTunnelWithConn("env-edge", &fakeServerTunnelConn{})
- deliveryTimer := time.NewTimer(streamDeliveryTimeout)
- deliveryTimer.Stop()
- defer deliveryTimer.Stop()
- server.handleTunnelMessage(t.Context(), tunnel, &TunnelMessage{
+ conn := &fakeServerTunnelConn{messages: []*TunnelMessage{{
Type: MessageTypeEvent,
Event: &TunnelEvent{
Type: "container.start",
Title: "Container started",
},
- }, deliveryTimer)
+ }}}
+ server.manageConnectedTunnel(t.Context(), "env-edge", conn, &TunnelMessage{Type: MessageTypeRegister}, "token")
select {
case <-called:
case <-time.After(2 * time.Second):
require.FailNow(t, "timeout waiting for event callback")
}
+
+ stopCtx, cancelStop := context.WithTimeout(t.Context(), time.Second)
+ defer cancelStop()
+ require.NoError(t, registry.Stop(stopCtx))
}
-type fakeServerTunnelConn struct{}
+// fakeServerTunnelConn delivers messages in order, then reports EOF.
+type fakeServerTunnelConn struct {
+ messages []*TunnelMessage
+}
func (f *fakeServerTunnelConn) Send(msg *TunnelMessage) error { return nil }
-func (f *fakeServerTunnelConn) Receive() (*TunnelMessage, error) { return nil, io.EOF }
+func (f *fakeServerTunnelConn) Receive() (*TunnelMessage, error) {
+ if len(f.messages) == 0 {
+ return nil, io.EOF
+ }
+ msg := f.messages[0]
+ f.messages = f.messages[1:]
+ return msg, nil
+}
func (f *fakeServerTunnelConn) IsExpectedReceiveError(error) bool { return false }
@@ -638,23 +655,23 @@ func TestTunnelServer_ManageConnectedTunnel_RegistersBeforeSendingGRPCRegisterRe
return nil
}
- tunnel := NewAgentTunnelWithConn(envID, conn)
- server.manageConnectedTunnel(t.Context(), t.Context(), tunnel)
+ server.manageConnectedTunnel(t.Context(), envID, conn, &TunnelMessage{Type: MessageTypeRegister}, "token")
_, ok := server.registry.Get(envID).Get()
assert.False(t, ok)
}
-func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCancellationInternal(t *testing.T) {
+func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCancellation(t *testing.T) {
registry := NewTunnelRegistry()
server := NewTunnelServerWithRegistry(registry, nil, nil)
- tunnel := NewAgentTunnelWithConn("env-cancelled-disconnect", &registerResponseOrderConn{recvErr: context.Canceled})
+ envID := "env-cancelled-disconnect"
+ conn := &registerResponseOrderConn{recvErr: context.Canceled}
connectionCtx, cancelConnection := context.WithCancel(t.Context())
cancelConnection()
- server.manageConnectedTunnel(connectionCtx, context.WithoutCancel(connectionCtx), tunnel)
+ server.manageConnectedTunnel(connectionCtx, envID, conn, &TunnelMessage{Type: MessageTypeRegister}, "token")
- _, ok := registry.Get(tunnel.EnvironmentID).Get()
+ _, ok := registry.Get(envID).Get()
require.False(t, ok)
stopCtx, cancelStop := context.WithTimeout(t.Context(), time.Second)
diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.go
index b7d8871ad..a6f386e43 100644
--- a/backend/pkg/libarcane/edge/tls.go
+++ b/backend/pkg/libarcane/edge/tls.go
@@ -1,7 +1,6 @@
package edge
import (
- "bytes"
"cmp"
"context"
"crypto"
@@ -16,7 +15,6 @@ import (
"errors"
"fmt"
"io"
- "io/fs"
"log/slog"
"net/http"
"net/url"
@@ -24,6 +22,7 @@ import (
"path"
"path/filepath"
"regexp"
+ "slices"
"strconv"
"strings"
"syscall"
@@ -34,18 +33,15 @@ import (
"go.getarcane.app/acfs/atomic"
kit "go.getarcane.app/kit/pkg"
libcrypto "go.getarcane.app/sys/crypto"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/peer"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
)
const (
- defaultGeneratedMTLSDir = "data/edge-mtls"
- defaultAgentMTLSDir = "data/edge-mtls-agent"
- generatedMTLSContainerDir = "/app/data/edge-mtls-agent"
- generatedMTLSCertValidity = 5 * 365 * 24 * time.Hour
+ managerMTLSDirName = "edge-mtls"
+ agentMTLSDirName = "edge-mtls-agent"
+ generatedMTLSContainerDir = "/app/data/" + agentMTLSDirName
generatedClientMTLSSubdir = "clients"
generatedMTLSCACertFileName = "ca.crt"
generatedMTLSCAKeyFileName = "ca.key"
@@ -58,15 +54,16 @@ const (
managerCALockTimeout = 2 * time.Minute
managerCALockPollInterval = 100 * time.Millisecond
- // caKeyEncryptedPrefix marks files written with libcrypto envelope encryption.
- // The payload after the prefix is the base64 ciphertext returned by
- // libcrypto.Encrypt of the plain PEM-encoded CA private key.
+ // caKeyEncryptedPrefix marks a CA key file holding the libcrypto.Encrypt output of the PEM key.
caKeyEncryptedPrefix = "ARCANE-ENC-V1:"
)
-var generatedAssetNameSanitizer = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
-
-var managerCALocks utils.KeyedMutex
+var (
+ generatedAssetNameSanitizer = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
+ managerCALocks utils.KeyedMutex
+ // caKeyEncrypt is a variable so tests can force encryption failures.
+ caKeyEncrypt = libcrypto.Encrypt
+)
// BuildManagerServerTLSConfig returns the manager TLS configuration needed to
// support optional edge mTLS on the shared Arcane listener.
@@ -75,19 +72,13 @@ func BuildManagerServerTLSConfig(cfg *Config) (*tls.Config, error) {
return nil, nil
}
- caPool, err := loadCertPoolInternal(strings.TrimSpace(cfg.EdgeMTLSCAFile))
- if err != nil {
+ caPool := x509.NewCertPool()
+ if err := appendCAFile(caPool, cfg.EdgeMTLSCAFile); err != nil {
return nil, fmt.Errorf("failed to load edge mTLS CA file: %w", err)
}
- // ClientAuth is intentionally VerifyClientCertIfGiven even when
- // EdgeMTLSMode is "required". Enforcement of certificate identity is done
- // per-request at the application layer so that the mTLS enrollment endpoint,
- // which agents must reach before they own a client certificate, remains
- // accessible. In proxy-terminated deployments the TLS state is not visible
- // here; identity is enforced by the upstream proxy.
- // If the handshake were set to RequireAndVerifyClientCert, bootstrap would fail.
- // TODO: reload ClientCAs when CA rotation support is added.
+ // Even "required" mode only verifies certificates when given: agents enroll before owning one,
+ // so identity is enforced per request. TODO: reload ClientCAs when CA rotation support is added.
return &tls.Config{
MinVersion: tls.VersionTLS12,
ClientAuth: tls.VerifyClientCertIfGiven,
@@ -103,7 +94,7 @@ func NewManagerHTTPClient(cfg *Config, timeout time.Duration) (*http.Client, err
return nil, errors.New("http.DefaultTransport is not *http.Transport")
}
transport := baseTransport.Clone()
- tlsConfig, err := buildManagerClientTLSConfigInternal(cfg)
+ tlsConfig, err := buildManagerClientTLSConfig(cfg)
if err != nil {
return nil, err
}
@@ -118,40 +109,25 @@ func NewManagerHTTPClient(cfg *Config, timeout time.Duration) (*http.Client, err
return client, nil
}
-// PrepareManagerMTLSAssetsWithContext ensures Arcane-managed edge mTLS assets exist when
-// edge mTLS is enabled and no explicit manager CA file is configured.
+// PrepareManagerMTLSAssetsWithContext generates the manager CA when edge mTLS is
+// enabled and no explicit manager CA file is configured.
func PrepareManagerMTLSAssetsWithContext(ctx context.Context, cfg *Config) error {
- if !shouldAutoGenerateManagerCAInternal(cfg) {
+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled || strings.TrimSpace(cfg.EdgeMTLSCAFile) != "" {
return nil
}
- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)
+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)
if err != nil {
return err
}
-
- if _, _, _, ensureManagerCAErr := ensureManagerCAInternal(ctx, assetsDir); ensureManagerCAErr != nil {
- return ensureManagerCAErr
+ if _, _, caErr := ensureManagerCA(ctx, assetsDir); caErr != nil {
+ return caErr
}
cfg.EdgeMTLSCAFile = filepath.Join(assetsDir, generatedMTLSCACertFileName)
return nil
}
-func generatedManagerMTLSCAPathInternal(cfg *Config) (string, error) {
- if cfg == nil {
- return "", errors.New("edge config is required")
- }
- if configured := strings.TrimSpace(cfg.EdgeMTLSCAFile); configured != "" {
- return configured, nil
- }
- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)
- if err != nil {
- return "", err
- }
- return filepath.Join(assetsDir, generatedMTLSCACertFileName), nil
-}
-
// AvailableManagerMTLSCAPath resolves an existing manager CA certificate.
func AvailableManagerMTLSCAPath(cfg *Config) (string, error) {
if cfg == nil {
@@ -161,50 +137,123 @@ func AvailableManagerMTLSCAPath(cfg *Config) (string, error) {
return "", errors.New("edge mTLS is disabled")
}
- caPath, err := generatedManagerMTLSCAPathInternal(cfg)
- if err != nil {
- return "", fmt.Errorf("resolve edge mTLS CA path: %w", err)
+ caPath := strings.TrimSpace(cfg.EdgeMTLSCAFile)
+ if caPath == "" {
+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)
+ if err != nil {
+ return "", fmt.Errorf("resolve edge mTLS CA path: %w", err)
+ }
+ caPath = filepath.Join(assetsDir, generatedMTLSCACertFileName)
}
- // os.* rather than acfs: the assets dir may be user-configured to anywhere on
- // the host, so no confinement root handle is in scope for this probe.
+ // os rather than acfs: the assets dir may be configured anywhere on the host.
if _, statErr := os.Stat(caPath); statErr != nil {
return "", fmt.Errorf("stat edge mTLS CA: %w", statErr)
}
return caPath, nil
}
-// GenerateManagerClientMTLSAssetsWithContext creates or loads the generated CA and per-environment client certificate bundle.
+// GenerateManagerClientMTLSAssetsWithContext ensures the generated CA and the environment's client
+// certificate, reusing a client certificate that is still valid for the environment's identity.
func GenerateManagerClientMTLSAssetsWithContext(ctx context.Context, cfg *Config, envID, envName string) (*GeneratedMTLSAssets, error) {
- if !shouldUseGeneratedManagerCAInternal(cfg) {
+ if !usesGeneratedManagerCA(cfg) {
return nil, nil
}
- if strings.TrimSpace(envID) == "" {
+ safeEnvID := sanitizedEnvID(envID)
+ if safeEnvID == "" {
return nil, errors.New("environment ID is required")
}
- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)
+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)
if err != nil {
return nil, err
}
- caCertPath, _, caGenerated, err := ensureManagerCAInternal(ctx, assetsDir)
+ ca, caGenerated, err := ensureManagerCA(ctx, assetsDir)
if err != nil {
return nil, err
}
- appURL := edgeMTLSAppURLInternal(cfg)
- clientCertPath, clientKeyPath, certIssued, err := ensureClientCertificateInternal(ctx, assetsDir, envID, envName, appURL)
+
+ clientLogicalDir := path.Join("/", generatedClientMTLSSubdir, safeEnvID)
+ clientDir := filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID)
+ if mkdirErr := acfs.MkdirAll(ctx, assetsDir, clientLogicalDir, utils.DirPerm); mkdirErr != nil {
+ return nil, fmt.Errorf("failed to create client cert dir: %w", mkdirErr)
+ }
+ unlock, err := lockEdgeMTLSPath(ctx, clientDir, ".client.lock")
if err != nil {
return nil, err
}
+ defer unlock()
+
+ certPath := filepath.Join(clientDir, generatedMTLSClientCertName)
+ keyPath := filepath.Join(clientDir, generatedMTLSClientKeyName)
+ appURL := cmp.Or(strings.TrimSpace(cfg.AppURL), strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)))
+ uriSAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)
+
+ // The URI SAN is the stable identity, so renaming an environment keeps its certificate.
+ existing, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ certIssued := reason != "" || !supportedGeneratedKey(existing.Leaf.PublicKey) ||
+ existing.Leaf.CheckSignatureFrom(ca.Leaf) != nil || (uriSAN != nil && !certificateHasURISAN(existing.Leaf, uriSAN))
+ if certIssued {
+ // Client keys follow the CA: P-384 under a legacy ECDSA CA, otherwise ML-DSA-87.
+ var clientKey crypto.Signer
+ if _, ecdsaCA := ca.PrivateKey.(*ecdsa.PrivateKey); ecdsaCA {
+ clientKey, err = certgen.GenerateP384PrivateKey()
+ } else {
+ clientKey, err = certgen.GenerateMLDSA87PrivateKey()
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to generate client private key: %w", err)
+ }
- caPEM, err := readGeneratedAssetInternal(ctx, assetsDir, caCertPath)
+ // The common name is display metadata: "<name>-<id>", capped at 64 characters.
+ safeEnvName := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envName), "-")
+ commonName := safeEnvID
+ if name := strings.Trim(safeEnvName, "-_"); name != "" && len(safeEnvID) < 63 {
+ if name = strings.Trim(name[:min(len(name), 63-len(safeEnvID))], "-_"); name != "" {
+ commonName = name + "-" + safeEnvID
+ }
+ }
+ dnsSANs := []string{"arcane-agent"}
+ if name := strings.Trim(safeEnvName, "-_."); name != "" && uriSAN != nil {
+ dnsSANs = append(dnsSANs, name+".agent."+uriSAN.Host)
+ }
+
+ template, templateErr := certgen.NewEdgeMTLSClientTemplate(commonName, uriSAN, dnsSANs)
+ if templateErr != nil {
+ return nil, templateErr
+ }
+ certDER, createErr := x509.CreateCertificate(rand.Reader, template, ca.Leaf, clientKey.Public(), ca.PrivateKey)
+ if createErr != nil {
+ return nil, fmt.Errorf("failed to create client certificate: %w", createErr)
+ }
+ if writeErr := atomic.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), utils.FilePerm); writeErr != nil {
+ return nil, writeErr
+ }
+
+ // P-384 keys keep SEC1 framing so agents from before the ML-DSA migration can parse them.
+ keyPEM := &pem.Block{Type: "PRIVATE KEY"}
+ if ecKey, ok := clientKey.(*ecdsa.PrivateKey); ok {
+ keyPEM.Type = "EC PRIVATE KEY"
+ keyPEM.Bytes, err = x509.MarshalECPrivateKey(ecKey)
+ } else {
+ keyPEM.Bytes, err = x509.MarshalPKCS8PrivateKey(clientKey)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal client private key: %w", err)
+ }
+ if writeErr := atomic.WriteFile(keyPath, pem.EncodeToMemory(keyPEM), 0o600); writeErr != nil {
+ return nil, writeErr
+ }
+ }
+
+ caPEM, err := acfs.ReadFile(ctx, assetsDir, "/"+generatedMTLSCACertFileName)
if err != nil {
return nil, fmt.Errorf("failed to read generated CA certificate: %w", err)
}
- clientCertPEM, err := readGeneratedAssetInternal(ctx, assetsDir, clientCertPath)
+ clientCertPEM, err := acfs.ReadFile(ctx, assetsDir, path.Join(clientLogicalDir, generatedMTLSClientCertName))
if err != nil {
return nil, fmt.Errorf("failed to read generated client certificate: %w", err)
}
- clientKeyPEM, err := readGeneratedAssetInternal(ctx, assetsDir, clientKeyPath)
+ clientKeyPEM, err := acfs.ReadFile(ctx, assetsDir, path.Join(clientLogicalDir, generatedMTLSClientKeyName))
if err != nil {
return nil, fmt.Errorf("failed to read generated client key: %w", err)
}
@@ -214,185 +263,113 @@ func GenerateManagerClientMTLSAssetsWithContext(ctx context.Context, cfg *Config
CertIssued: certIssued,
CAGenerated: caGenerated,
Files: []GeneratedMTLSFile{
- {Name: generatedMTLSCACertFileName, Content: string(caPEM), ContainerPath: filepath.ToSlash(filepath.Join(generatedMTLSContainerDir, generatedMTLSCACertFileName)), Permissions: "0644"},
- {Name: generatedMTLSClientCertName, Content: string(clientCertPEM), ContainerPath: filepath.ToSlash(filepath.Join(generatedMTLSContainerDir, generatedMTLSClientCertName)), Permissions: "0644"},
- {Name: generatedMTLSClientKeyName, Content: string(clientKeyPEM), ContainerPath: filepath.ToSlash(filepath.Join(generatedMTLSContainerDir, generatedMTLSClientKeyName)), Permissions: "0600"},
+ {Name: generatedMTLSCACertFileName, Content: string(caPEM), ContainerPath: path.Join(generatedMTLSContainerDir, generatedMTLSCACertFileName), Permissions: "0644"},
+ {Name: generatedMTLSClientCertName, Content: string(clientCertPEM), ContainerPath: path.Join(generatedMTLSContainerDir, generatedMTLSClientCertName), Permissions: "0644"},
+ {Name: generatedMTLSClientKeyName, Content: string(clientKeyPEM), ContainerPath: path.Join(generatedMTLSContainerDir, generatedMTLSClientKeyName), Permissions: "0600"},
},
}, nil
}
-// readGeneratedAssetInternal reads a file Arcane itself wrote under the
-// generated-assets directory, confined to that root.
-func readGeneratedAssetInternal(ctx context.Context, assetsDir, absPath string) ([]byte, error) {
- logicalPath, err := acfs.LogicalPath(assetsDir, absPath)
- if err != nil {
- return nil, err
- }
- return acfs.ReadFile(ctx, assetsDir, logicalPath)
-}
-
-func managerMTLSEnrollmentStateInternal(cfg *Config, envID string, now time.Time) (bool, bool, error) {
- markerPath, err := managerMTLSEnrollmentMarkerPathInternal(cfg, envID)
- if err != nil {
- return false, false, err
- }
- enrolledAt, err := readMTLSEnrollmentMarkerInternal(markerPath)
- if err != nil {
- return false, false, kit.Ternary(errors.Is(err, fs.ErrNotExist), nil, err)
- }
- if now.IsZero() {
- now = time.Now()
- }
- if enrolledAt.IsZero() {
- return true, false, nil
- }
- return true, now.Sub(enrolledAt) < managerMTLSReenrollCooldown, nil
-}
-
-func recordManagerMTLSEnrollmentInternal(cfg *Config, envID string, now time.Time) error {
- markerPath, err := managerMTLSEnrollmentMarkerPathInternal(cfg, envID)
- if err != nil {
- return err
- }
- if now.IsZero() {
- now = time.Now()
- }
- return atomic.WriteFile(markerPath, []byte(now.UTC().Format(time.RFC3339Nano)+"\n"), 0o600)
-}
-
-func managerMTLSEnrollmentMarkerPathInternal(cfg *Config, envID string) (string, error) {
- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)
- if err != nil {
- return "", err
- }
- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")
- if safeEnvID == "" {
- return "", errors.New("environment ID is required")
- }
- return filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID, generatedMTLSEnrolledName), nil
-}
-
-func readMTLSEnrollmentMarkerInternal(localPath string) (time.Time, error) {
- // Path-only helper; callers derive the path from the assets root, kept on
- // os.* with the other path-only readers.
- content, err := os.ReadFile(localPath)
- if err != nil {
- return time.Time{}, err
- }
- trimmed := strings.TrimSpace(string(content))
- if trimmed == "" {
- return time.Time{}, nil
- }
- enrolledAt, err := time.Parse(time.RFC3339Nano, trimmed)
- if err != nil {
- return time.Time{}, fmt.Errorf("failed to parse edge mTLS enrollment marker %s: %w", localPath, err)
- }
- return enrolledAt, nil
-}
-
// GeneratedManagerClientMTLSCertPath returns the manager-side generated client certificate path for an environment.
func GeneratedManagerClientMTLSCertPath(cfg *Config, envID string) (string, error) {
- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)
+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)
if err != nil {
return "", err
}
-
- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")
+ safeEnvID := sanitizedEnvID(envID)
if safeEnvID == "" {
return "", errors.New("environment ID is required")
}
-
- return filepath.Join(assetsDir, "clients", safeEnvID, generatedMTLSClientCertName), nil
+ return filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID, generatedMTLSClientCertName), nil
}
-// EnsureAgentMTLSAssets downloads manager-generated client certificates when
-// edge mTLS is enabled and explicit client cert/key files are not configured.
+// EnsureAgentMTLSAssets downloads manager-generated client certificates when edge mTLS is
+// enabled without configured client cert/key files, and renews them before they expire.
func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {
- if !shouldAutoEnrollAgentMTLSInternal(cfg) {
- return nil
- }
- if hasClientCertificateInternal(cfg) {
+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled ||
+ (strings.TrimSpace(cfg.EdgeMTLSCertFile) != "" && strings.TrimSpace(cfg.EdgeMTLSKeyFile) != "") {
return nil
}
- assetsDir, err := edgeAgentMTLSAssetsDirInternal(cfg)
+ assetsDir, err := edgeMTLSAssetsDir(cfg, agentMTLSDirName)
if err != nil {
return err
}
certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)
keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)
- if fileExistsInternal(certPath) && fileExistsInternal(keyPath) {
- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath, time.Now())
- if !needsEnrollment {
- if !fileExistsInternal(filepath.Join(assetsDir, generatedMTLSEnrolledName)) {
- if writeErr := acfs.Write(ctx, assetsDir, "/"+generatedMTLSEnrolledName, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), acfs.WriteOptions{Mode: 0o600}); writeErr != nil {
- return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr)
- }
+
+ _, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ if reason != "" {
+ if fileExists(certPath) {
+ slog.WarnContext(ctx, "Existing edge mTLS assets need renewal; enrolling new assets", "reason", reason, "certPath", certPath)
+ }
+ if enrollErr := enrollAgentMTLSAssets(ctx, cfg, assetsDir, certPath, keyPath); enrollErr != nil {
+ if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired {
+ slog.WarnContext(ctx, "Edge mTLS enrollment failed; proceeding without client certificate", "error", enrollErr)
+ return nil
}
- setAgentMTLSAssetPathsInternal(cfg, assetsDir)
- return nil
+ return enrollErr
}
- slog.WarnContext(ctx, "Existing edge mTLS assets need renewal; enrolling new assets", "reason", reason, "certPath", certPath)
}
- if enrollAgentMTLSAssetsErr := enrollAgentMTLSAssetsInternal(ctx, cfg, assetsDir, certPath, keyPath); enrollAgentMTLSAssetsErr != nil {
- if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired {
- slog.WarnContext(ctx, "Edge mTLS enrollment failed; proceeding without client certificate", "error", enrollAgentMTLSAssetsErr)
- return nil
+ if reason != "" || !fileExists(filepath.Join(assetsDir, generatedMTLSEnrolledName)) {
+ marker := []byte(time.Now().UTC().Format(time.RFC3339) + "\n")
+ if writeErr := acfs.Write(ctx, assetsDir, "/"+generatedMTLSEnrolledName, marker, acfs.WriteOptions{Mode: 0o600}); writeErr != nil {
+ return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr)
}
- return enrollAgentMTLSAssetsErr
+ }
+ cfg.EdgeMTLSCertFile = certPath
+ cfg.EdgeMTLSKeyFile = keyPath
+ if caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName); fileExists(caPath) && strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {
+ cfg.EdgeMTLSCAFile = caPath
}
return nil
}
-func enrollAgentMTLSAssetsInternal(ctx context.Context, cfg *Config, assetsDir, certPath, keyPath string) error {
- if cfg == nil {
- return errors.New("MANAGER_API_URL is required to enroll edge mTLS assets")
- }
+// enrollAgentMTLSAssets downloads and writes the manager-issued client assets. It stays separate
+// from EnsureAgentMTLSAssets to keep that function under the cognitive-complexity limit.
+func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPath, keyPath string) error {
managerBaseURL := strings.TrimRight(strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)), "/")
if managerBaseURL == "" {
return errors.New("MANAGER_API_URL is required to enroll edge mTLS assets")
}
- if !managerUsesTLSInternal(cfg) {
+ if !managerUsesTLS(cfg) {
return errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https for certificate enrollment")
}
- httpClient, err := NewManagerHTTPClient(cfg, 30*time.Second)
- if err != nil {
- return fmt.Errorf("failed to configure edge mTLS enrollment client: %w", err)
+ httpClient, clientErr := NewManagerHTTPClient(cfg, 30*time.Second)
+ if clientErr != nil {
+ return fmt.Errorf("failed to configure edge mTLS enrollment client: %w", clientErr)
}
-
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, managerBaseURL+"/api/tunnel/mtls/enroll", http.NoBody)
- if err != nil {
- return fmt.Errorf("failed to create edge mTLS enrollment request: %w", err)
+ req, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, managerBaseURL+"/api/tunnel/mtls/enroll", http.NoBody)
+ if reqErr != nil {
+ return fmt.Errorf("failed to create edge mTLS enrollment request: %w", reqErr)
+ }
+ for header, value := range agentAuthCredentialsInternal(cfg.AgentToken) {
+ req.Header.Set(header, value)
}
- req.Header.Set(HeaderAgentToken, cfg.AgentToken)
- req.Header.Set(HeaderAPIKey, cfg.AgentToken)
- req.Header.Set(HeaderAuthorization, "Bearer "+cfg.AgentToken)
- resp, err := httpClient.Do(req)
- if err != nil {
- return fmt.Errorf("edge mTLS enrollment request failed: %w", err)
+ resp, doErr := httpClient.Do(req)
+ if doErr != nil {
+ return fmt.Errorf("edge mTLS enrollment request failed: %w", doErr)
}
defer func() { _ = resp.Body.Close() }()
-
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, maxEnrollResponseBytes))
return fmt.Errorf("edge mTLS enrollment failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var enrollResp enrollMTLSResponse
- if unmarshalReadErr := json.UnmarshalRead(io.LimitReader(resp.Body, maxEnrollResponseBytes), &enrollResp); unmarshalReadErr != nil {
- return fmt.Errorf("failed to decode edge mTLS enrollment response: %w", unmarshalReadErr)
+ if decodeErr := json.UnmarshalRead(io.LimitReader(resp.Body, maxEnrollResponseBytes), &enrollResp); decodeErr != nil {
+ return fmt.Errorf("failed to decode edge mTLS enrollment response: %w", decodeErr)
}
if len(enrollResp.Files) == 0 {
return errors.New("edge mTLS enrollment response did not include any files")
}
- // The assets dir is the confinement root for the writes below, so it is
- // created through os before acfs opens it.
- if mkdirAllErr := os.MkdirAll(assetsDir, utils.DirPerm); mkdirAllErr != nil {
- return fmt.Errorf("failed to create edge mTLS asset dir: %w", mkdirAllErr)
+ // The assets dir is the acfs confinement root for the writes below, so os creates it.
+ if mkdirErr := os.MkdirAll(assetsDir, utils.DirPerm); mkdirErr != nil {
+ return fmt.Errorf("failed to create edge mTLS asset dir: %w", mkdirErr)
}
for _, file := range enrollResp.Files {
perm := kit.Ternary(strings.TrimSpace(file.Permissions) == "0600", 0o600, utils.FilePerm)
@@ -400,309 +377,142 @@ func enrollAgentMTLSAssetsInternal(ctx context.Context, cfg *Config, assetsDir,
return fmt.Errorf("failed to write edge mTLS asset %s: %w", file.Name, writeErr)
}
}
- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath, time.Now())
- if needsEnrollment {
- return fmt.Errorf("edge mTLS enrollment wrote unusable assets: %s", reason)
+ if _, writtenReason := loadClientCertificate(certPath, keyPath, time.Now()); writtenReason != "" {
+ return fmt.Errorf("edge mTLS enrollment wrote unusable assets: %s", writtenReason)
}
- if writeErr2 := acfs.Write(ctx, assetsDir, "/"+generatedMTLSEnrolledName, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), acfs.WriteOptions{Mode: 0o600}); writeErr2 != nil {
- return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr2)
- }
-
- setAgentMTLSAssetPathsInternal(cfg, assetsDir)
return nil
}
-func buildManagerClientTLSConfigInternal(cfg *Config) (*tls.Config, error) {
- ctx := context.Background() //nolint:forbidigo // TLS configuration is built at startup without a request context.
- if cfg == nil || !managerUsesTLSInternal(cfg) {
+// buildManagerClientTLSConfig returns the agent's TLS config for an https manager URL, adding the
+// configured CA to the system roots and the configured client certificate.
+func buildManagerClientTLSConfig(cfg *Config) (*tls.Config, error) {
+ if cfg == nil || !managerUsesTLS(cfg) {
return nil, nil
}
+ ctx := context.Background() //nolint:forbidigo // TLS configuration is built at startup without a request context.
+ tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: strings.TrimSpace(cfg.EdgeMTLSServerName)}
- tlsConfig := &tls.Config{
- MinVersion: tls.VersionTLS12,
- }
-
- if serverName := strings.TrimSpace(cfg.EdgeMTLSServerName); serverName != "" {
- tlsConfig.ServerName = serverName
- }
-
- if caFile := strings.TrimSpace(cfg.EdgeMTLSCAFile); caFile != "" {
- pool, err := loadSystemOrCustomCertPoolInternal(caFile)
- if err != nil {
- return nil, fmt.Errorf("failed to load edge mTLS CA file: %w", err)
+ if strings.TrimSpace(cfg.EdgeMTLSCAFile) != "" {
+ pool, err := x509.SystemCertPool()
+ if err != nil || pool == nil {
+ slog.WarnContext(ctx, "Failed to load system certificate pool; falling back to configured edge mTLS CA only", "error", err)
+ pool = x509.NewCertPool()
+ }
+ if appendErr := appendCAFile(pool, cfg.EdgeMTLSCAFile); appendErr != nil {
+ return nil, fmt.Errorf("failed to load edge mTLS CA file: %w", appendErr)
}
tlsConfig.RootCAs = pool
}
- if hasClientCertificateInternal(cfg) {
- mode := NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode)
- certPath := strings.TrimSpace(cfg.EdgeMTLSCertFile)
- keyPath := strings.TrimSpace(cfg.EdgeMTLSKeyFile)
- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath, time.Now())
- if needsEnrollment {
- err := fmt.Errorf("edge mTLS client certificate is unusable: %s", reason)
- if mode == EdgeMTLSModeOptional {
- slog.WarnContext(ctx, "Ignoring unusable optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "error", err.Error())
- return tlsConfig, nil
- }
- return nil, err
- }
- cert, err := tls.LoadX509KeyPair(certPath, keyPath)
- if err != nil {
- if mode == EdgeMTLSModeOptional {
- slog.WarnContext(ctx, "Failed to load optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "error", err.Error())
- return tlsConfig, nil
- }
- return nil, fmt.Errorf("failed to load edge mTLS client certificate: %w", err)
- }
- if cert.Leaf != nil {
- if _, ok := cert.Leaf.PublicKey.(*mldsa.PublicKey); ok {
- tlsConfig.MinVersion = tls.VersionTLS13
- }
+ certPath, keyPath := strings.TrimSpace(cfg.EdgeMTLSCertFile), strings.TrimSpace(cfg.EdgeMTLSKeyFile)
+ if certPath == "" || keyPath == "" {
+ return tlsConfig, nil
+ }
+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ if reason != "" {
+ if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeOptional {
+ slog.WarnContext(ctx, "Ignoring unusable optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "reason", reason)
+ return tlsConfig, nil
}
- tlsConfig.Certificates = []tls.Certificate{cert}
+ return nil, fmt.Errorf("edge mTLS client certificate is unusable: %s", reason)
}
-
+ // ML-DSA client certificates require TLS 1.3.
+ if _, ok := cert.Leaf.PublicKey.(*mldsa.PublicKey); ok {
+ tlsConfig.MinVersion = tls.VersionTLS13
+ }
+ tlsConfig.Certificates = []tls.Certificate{cert}
return tlsConfig, nil
}
// ValidateAgentMTLSConfig validates the edge agent TLS configuration before the
// reverse tunnel client starts.
func ValidateAgentMTLSConfig(cfg *Config) error {
- if cfg == nil {
- return nil
- }
-
- mode := NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode)
- if mode == EdgeMTLSModeDisabled {
+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {
return nil
}
-
- if !managerUsesTLSInternal(cfg) {
+ if !managerUsesTLS(cfg) {
return errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https")
}
-
- _, err := buildManagerClientTLSConfigInternal(cfg)
+ _, err := buildManagerClientTLSConfig(cfg)
return err
}
-// ValidateManagerMTLSConfig validates the manager-side mTLS configuration used
-// by edge tunnel endpoints.
+// ValidateManagerMTLSConfig validates a configured manager CA file. An empty CA file is
+// allowed because PrepareManagerMTLSAssetsWithContext generates one.
func ValidateManagerMTLSConfig(cfg *Config) error {
- if cfg == nil {
- return nil
- }
-
- mode := NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode)
- if mode == EdgeMTLSModeDisabled {
- return nil
- }
-
- if strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {
+ if cfg == nil || strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {
return nil
}
-
- _, err := loadCertPoolInternal(cfg.EdgeMTLSCAFile)
- if err != nil {
- return fmt.Errorf("failed to load edge mTLS CA file: %w", err)
- }
- return nil
-}
-
-func loadCertPoolInternal(caFile string) (*x509.CertPool, error) {
- caFile = strings.TrimSpace(caFile)
- if caFile == "" {
- return nil, errors.New("CA file is required")
- }
-
- // os.* rather than acfs: the CA path may be user-configured to anywhere on
- // the host, so no confinement root exists for it.
- pemBytes, err := os.ReadFile(caFile)
- if err != nil {
- return nil, err
- }
-
- pool := x509.NewCertPool()
- if !pool.AppendCertsFromPEM(pemBytes) {
- return nil, errors.New("failed to parse PEM certificates")
- }
-
- return pool, nil
+ _, err := BuildManagerServerTLSConfig(cfg)
+ return err
}
-func loadSystemOrCustomCertPoolInternal(caFile string) (*x509.CertPool, error) {
- pool, err := x509.SystemCertPool()
- if err != nil || pool == nil {
- ctx := context.Background() //nolint:forbidigo // TLS configuration is built at startup without a request context.
- slog.WarnContext(ctx, "Failed to load system certificate pool; falling back to configured edge mTLS CA only",
- "error", err)
- pool = x509.NewCertPool()
- }
-
+// appendCAFile adds the PEM certificates in caFile to pool. The path may be configured
+// anywhere on the host, so it is read with os rather than acfs.
+func appendCAFile(pool *x509.CertPool, caFile string) error {
caFile = strings.TrimSpace(caFile)
if caFile == "" {
- return pool, nil
+ return errors.New("CA file is required")
}
-
- // os.* rather than acfs: the CA path may be user-configured to anywhere on
- // the host, so no confinement root exists for it.
pemBytes, err := os.ReadFile(caFile)
if err != nil {
- return nil, err
+ return err
}
if !pool.AppendCertsFromPEM(pemBytes) {
- return nil, errors.New("failed to parse PEM certificates")
+ return errors.New("failed to parse PEM certificates")
}
- return pool, nil
+ return nil
}
-func hasClientCertificateInternal(cfg *Config) bool {
- if cfg == nil {
- return false
- }
-
- return strings.TrimSpace(cfg.EdgeMTLSCertFile) != "" && strings.TrimSpace(cfg.EdgeMTLSKeyFile) != ""
+func managerUsesTLS(cfg *Config) bool {
+ return strings.HasPrefix(strings.ToLower(strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))), "https://")
}
-func managerUsesTLSInternal(cfg *Config) bool {
- if cfg == nil {
- return false
- }
-
- baseURL := strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))
- return strings.HasPrefix(strings.ToLower(baseURL), "https://")
-}
-
-func shouldAutoGenerateManagerCAInternal(cfg *Config) bool {
- if cfg == nil {
- return false
- }
-
- return NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeDisabled &&
- strings.TrimSpace(cfg.EdgeMTLSCAFile) == ""
-}
-
-func shouldUseGeneratedManagerCAInternal(cfg *Config) bool {
- if cfg == nil {
- return false
- }
- if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {
+// usesGeneratedManagerCA reports whether edge mTLS runs on the Arcane-generated CA. The configured
+// CA also counts once PrepareManagerMTLSAssetsWithContext has pointed it at the generated file.
+func usesGeneratedManagerCA(cfg *Config) bool {
+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {
return false
}
configuredCA := strings.TrimSpace(cfg.EdgeMTLSCAFile)
if configuredCA == "" {
return true
}
- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)
- if err != nil {
- return false
- }
- generatedCA := filepath.Join(assetsDir, generatedMTLSCACertFileName)
- return filepath.Clean(configuredCA) == filepath.Clean(generatedCA)
-}
-
-func shouldAutoEnrollAgentMTLSInternal(cfg *Config) bool {
- if cfg == nil {
- return false
- }
-
- return NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeDisabled &&
- !hasClientCertificateInternal(cfg)
-}
-
-func setAgentMTLSAssetPathsInternal(cfg *Config, assetsDir string) {
- if cfg == nil {
- return
- }
-
- cfg.EdgeMTLSCertFile = filepath.Join(assetsDir, generatedMTLSClientCertName)
- cfg.EdgeMTLSKeyFile = filepath.Join(assetsDir, generatedMTLSClientKeyName)
-
- caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)
- if fileExistsInternal(caPath) && strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {
- cfg.EdgeMTLSCAFile = caPath
- }
-}
-
-func requestSecurityModeInternal(req *http.Request) string {
- return kit.Ternary(hasVerifiedEdgeMTLSRequestInternal(req), "mtls", "token")
-}
-
-func hasVerifiedEdgeMTLSRequestInternal(req *http.Request) bool {
- if req == nil {
- return false
- }
- return req.TLS != nil && hasVerifiedPeerCertificateInternal(req.TLS)
+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)
+ return err == nil && filepath.Clean(configuredCA) == filepath.Join(assetsDir, generatedMTLSCACertFileName)
}
-func grpcContextSecurityModeInternal(pctx peer.Peer) string {
- if tlsInfo, ok := pctx.AuthInfo.(credentials.TLSInfo); ok && hasVerifiedPeerCertificateInternal(&tlsInfo.State) {
- return "mtls"
- }
- return "token"
+func hasVerifiedPeerCertificate(state *tls.ConnectionState) bool {
+ return state != nil && len(state.PeerCertificates) > 0 && len(state.VerifiedChains) > 0
}
-func hasVerifiedPeerCertificateInternal(state *tls.ConnectionState) bool {
- if state == nil {
- return false
- }
- return len(state.PeerCertificates) > 0 && len(state.VerifiedChains) > 0
+// certificateHasURISAN reports whether cert carries expected as a URI SAN, ignoring case and a trailing dot in the host.
+func certificateHasURISAN(cert *x509.Certificate, expected *url.URL) bool {
+ return slices.ContainsFunc(cert.URIs, func(uri *url.URL) bool {
+ return uri != nil && strings.EqualFold(uri.Scheme, expected.Scheme) && uri.Path == expected.Path &&
+ strings.EqualFold(strings.TrimSuffix(uri.Host, "."), strings.TrimSuffix(expected.Host, "."))
+ })
}
-func verifiedPeerCertificateEnvironmentIDMatchesInternal(state *tls.ConnectionState, envID, trustDomain string) error {
- if !hasVerifiedPeerCertificateInternal(state) {
- return nil
- }
- expectedPath := expectedEdgeMTLSURIPathInternal(envID)
- if expectedPath == "" {
- return errors.New("environment ID is required for edge mTLS certificate identity check")
- }
- trustDomain = strings.TrimSpace(strings.ToLower(strings.TrimSuffix(trustDomain, ".")))
- if trustDomain == "" {
- return errors.New("edge mTLS trust domain is required for certificate identity check")
- }
- leaf := state.VerifiedChains[0][0]
- for _, uri := range leaf.URIs {
- if uri == nil {
- continue
- }
- if uri.Scheme == "spiffe" && strings.EqualFold(strings.TrimSuffix(uri.Host, "."), trustDomain) && uri.Path == expectedPath {
- return nil
- }
- }
- return fmt.Errorf("verified edge mTLS client certificate does not match environment %s", strings.TrimSpace(envID))
+// sanitizedEnvID makes an environment ID safe for asset paths and certificate identities.
+func sanitizedEnvID(envID string) string {
+ return generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")
}
-func edgeMTLSAppURLInternal(cfg *Config) string {
- if cfg == nil {
- return ""
- }
- if appURL := strings.TrimSpace(cfg.AppURL); appURL != "" {
- return appURL
- }
- return strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))
-}
-
-func expectedEdgeMTLSURIPathInternal(envID string) string {
- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")
- return kit.Ternary(safeEnvID == "", "", "/edge/"+safeEnvID)
-}
-
-func edgeMTLSAssetsDirInternal(cfg *Config) (string, error) {
+// edgeMTLSAssetsDir returns the configured assets dir, else data/<name> under /app when it exists.
+func edgeMTLSAssetsDir(cfg *Config, name string) (string, error) {
if cfg == nil {
return "", errors.New("edge config is required")
}
-
if configured := strings.TrimSpace(cfg.EdgeMTLSAssetsDir); configured != "" {
return configured, nil
}
- baseDir := defaultGeneratedMTLSDir
+ baseDir := filepath.Join("data", name)
// Probes the fixed system path /app/data, which is not under any acfs root.
if _, err := os.Stat("/app/data"); err == nil {
- baseDir = "/app/data/edge-mtls"
+ baseDir = "/app/data/" + name
}
-
resolved, err := filepath.Abs(baseDir)
if err != nil {
return "", fmt.Errorf("failed to resolve edge mTLS assets dir: %w", err)
@@ -710,598 +520,179 @@ func edgeMTLSAssetsDirInternal(cfg *Config) (string, error) {
return resolved, nil
}
-func edgeAgentMTLSAssetsDirInternal(cfg *Config) (string, error) {
- if cfg == nil {
- return "", errors.New("edge config is required")
- }
- if configured := strings.TrimSpace(cfg.EdgeMTLSAssetsDir); configured != "" {
- return configured, nil
- }
-
- baseDir := defaultAgentMTLSDir
- // Probes the fixed system path /app/data, which is not under any acfs root.
- if _, err := os.Stat("/app/data"); err == nil {
- baseDir = "/app/data/edge-mtls-agent"
- }
-
- resolved, err := filepath.Abs(baseDir)
- if err != nil {
- return "", fmt.Errorf("failed to resolve edge agent mTLS assets dir: %w", err)
- }
- return resolved, nil
-}
-
-func ensureManagerCAInternal(ctx context.Context, assetsDir string) (string, string, bool, error) {
- // The assets dir is the confinement root for everything below, so it is
- // created through os before acfs opens it.
+// ensureManagerCA returns the generated CA, creating it unless the existing one decrypts,
+// matches its key, and uses a supported algorithm.
+func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, bool, error) {
+ // The assets dir is the acfs confinement root for callers, so os creates it.
if err := os.MkdirAll(assetsDir, utils.DirPerm); err != nil {
- return "", "", false, fmt.Errorf("failed to create edge mTLS assets dir: %w", err)
+ return tls.Certificate{}, false, fmt.Errorf("failed to create edge mTLS assets dir: %w", err)
}
-
- unlock, err := lockEdgeMTLSPathInternal(ctx, assetsDir, ".ca.lock")
+ unlock, err := lockEdgeMTLSPath(ctx, assetsDir, ".ca.lock")
if err != nil {
- return "", "", false, err
+ return tls.Certificate{}, false, err
}
defer unlock()
- caCertPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)
- caKeyPath := filepath.Join(assetsDir, generatedMTLSCAKeyFileName)
- if generatedCAReadyInternal(caCertPath, caKeyPath) {
- return caCertPath, caKeyPath, false, nil
+ certPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)
+ keyPath := filepath.Join(assetsDir, generatedMTLSCAKeyFileName)
+ // os rather than acfs: the assets dir may be configured anywhere on the host.
+ certPEM, certErr := os.ReadFile(certPath)
+ rawKey, keyErr := os.ReadFile(keyPath)
+ keyPEM := ""
+ if ciphertext, encrypted := strings.CutPrefix(strings.TrimSpace(string(rawKey)), caKeyEncryptedPrefix); certErr == nil && keyErr == nil && encrypted {
+ keyPEM, _ = libcrypto.Decrypt(ciphertext)
+ }
+ if ca, pairErr := tls.X509KeyPair(certPEM, []byte(keyPEM)); pairErr == nil && ca.Leaf.IsCA && supportedGeneratedKey(ca.Leaf.PublicKey) {
+ return ca, false, nil
}
- _ = acfs.Remove(ctx, assetsDir, "/"+generatedMTLSCACertFileName)
- _ = acfs.Remove(ctx, assetsDir, "/"+generatedMTLSCAKeyFileName)
privateKey, err := certgen.GenerateMLDSA87PrivateKey()
if err != nil {
- return "", "", false, fmt.Errorf("failed to generate CA private key: %w", err)
+ return tls.Certificate{}, false, fmt.Errorf("failed to generate CA private key: %w", err)
}
-
template, err := certgen.NewEdgeMTLSCATemplate()
if err != nil {
- return "", "", false, err
+ return tls.Certificate{}, false, err
}
-
certDER, err := x509.CreateCertificate(rand.Reader, template, template, privateKey.PublicKey(), privateKey)
if err != nil {
- return "", "", false, fmt.Errorf("failed to create CA certificate: %w", err)
+ return tls.Certificate{}, false, fmt.Errorf("failed to create CA certificate: %w", err)
}
-
- if writePEMFileErr := writePEMFileInternal(caCertPath, "CERTIFICATE", certDER, utils.FilePerm); writePEMFileErr != nil {
- return "", "", false, writePEMFileErr
- }
- caKeyDER, err := x509.MarshalPKCS8PrivateKey(privateKey)
+ leaf, err := x509.ParseCertificate(certDER)
if err != nil {
- return "", "", false, fmt.Errorf("failed to marshal CA private key: %w", err)
- }
- if writeCAKeyFileErr := writeCAKeyFileInternal(caKeyPath, caKeyDER); writeCAKeyFileErr != nil {
- return "", "", false, writeCAKeyFileErr
+ return tls.Certificate{}, false, fmt.Errorf("failed to parse CA certificate: %w", err)
}
-
- slog.InfoContext(ctx, "generated edge mTLS CA", "certPath", caCertPath)
- return caCertPath, caKeyPath, true, nil
-}
-
-func generatedCAReadyInternal(caCertPath, caKeyPath string) bool {
- if !fileExistsInternal(caCertPath) || !fileExistsInternal(caKeyPath) {
- return false
+ if writeErr := atomic.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), utils.FilePerm); writeErr != nil {
+ return tls.Certificate{}, false, writeErr
}
- if err := validateGeneratedCAInternal(caCertPath, caKeyPath); err != nil {
- return false
- }
- return true
-}
-func ensureClientCertificateInternal(ctx context.Context, assetsDir, envID, envName, appURL string) (string, string, bool, error) {
- caCertPath, caKeyPath, _, err := ensureManagerCAInternal(ctx, assetsDir)
+ keyDER, err := x509.MarshalPKCS8PrivateKey(privateKey)
if err != nil {
- return "", "", false, err
+ return tls.Certificate{}, false, fmt.Errorf("failed to marshal CA private key: %w", err)
}
-
- caCertPEM, err := readGeneratedAssetInternal(ctx, assetsDir, caCertPath)
- if err != nil {
- return "", "", false, fmt.Errorf("failed to read CA certificate: %w", err)
- }
- caKeyPEM, err := readCAKeyPEMInternal(caKeyPath)
- if err != nil {
- return "", "", false, err
- }
-
- caCertBlock, _ := pem.Decode(caCertPEM)
- if caCertBlock == nil {
- return "", "", false, errors.New("failed to parse CA certificate PEM")
- }
- caCert, err := x509.ParseCertificate(caCertBlock.Bytes)
- if err != nil {
- return "", "", false, fmt.Errorf("failed to parse CA certificate: %w", err)
- }
-
- caKey, err := parsePrivateKeyPEMInternal(caKeyPEM, "CA")
- if err != nil {
- return "", "", false, err
- }
-
- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")
- clientDir := filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID)
- if mkdirAllErr := acfs.MkdirAll(ctx, assetsDir, path.Join("/", generatedClientMTLSSubdir, safeEnvID), utils.DirPerm); mkdirAllErr != nil {
- return "", "", false, fmt.Errorf("failed to create client cert dir: %w", mkdirAllErr)
- }
- unlock, err := lockEdgeMTLSPathInternal(ctx, clientDir, ".client.lock")
- if err != nil {
- return "", "", false, err
- }
- defer unlock()
-
- clientCertPath := filepath.Join(clientDir, generatedMTLSClientCertName)
- clientKeyPath := filepath.Join(clientDir, generatedMTLSClientKeyName)
- expectedCommonName := buildGeneratedClientCommonNameInternal(envName, safeEnvID)
- expectedURISAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)
- if fileExistsInternal(clientCertPath) && fileExistsInternal(clientKeyPath) {
- // The URI SAN is the stable edge identity. Common Name is display metadata
- // for newly issued certs only, so environment renames must not rotate keys.
- if validateGeneratedClientCertificateErr := validateGeneratedClientCertificateInternal(clientCertPath, clientKeyPath, "", expectedURISAN); validateGeneratedClientCertificateErr == nil {
- return clientCertPath, clientKeyPath, false, nil
- }
- _ = acfs.Remove(ctx, clientDir, "/"+generatedMTLSClientCertName)
- _ = acfs.Remove(ctx, clientDir, "/"+generatedMTLSClientKeyName)
- }
-
- privateKey, err := generateKeyLikeInternal(caKey)
- if err != nil {
- return "", "", false, fmt.Errorf("failed to generate client private key: %w", err)
- }
- uriSAN, dnsSANs := buildGeneratedClientSANsInternal(envName, safeEnvID, appURL)
- template, err := certgen.NewEdgeMTLSClientTemplate(expectedCommonName, uriSAN, dnsSANs)
- if err != nil {
- return "", "", false, err
- }
-
- certDER, err := x509.CreateCertificate(rand.Reader, template, caCert, privateKey.Public(), caKey)
+ ciphertext, err := caKeyEncrypt(string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})))
if err != nil {
- return "", "", false, fmt.Errorf("failed to create client certificate: %w", err)
- }
-
- if writePEMFileErr := writePEMFileInternal(clientCertPath, "CERTIFICATE", certDER, utils.FilePerm); writePEMFileErr != nil {
- return "", "", false, writePEMFileErr
- }
- // P-384 leaves keep the SEC1 framing so agents from before the ML-DSA
- // migration can still parse keys issued during a rolling upgrade.
- var clientKeyDER []byte
- keyPEMType := "PRIVATE KEY"
- if ecKey, ok := privateKey.(*ecdsa.PrivateKey); ok {
- keyPEMType = "EC PRIVATE KEY"
- clientKeyDER, err = x509.MarshalECPrivateKey(ecKey)
- } else {
- clientKeyDER, err = x509.MarshalPKCS8PrivateKey(privateKey)
- }
- if err != nil {
- return "", "", false, fmt.Errorf("failed to marshal client private key: %w", err)
- }
- if writePEMFileErr2 := writePEMFileInternal(clientKeyPath, keyPEMType, clientKeyDER, 0o600); writePEMFileErr2 != nil {
- return "", "", false, writePEMFileErr2
- }
-
- return clientCertPath, clientKeyPath, true, nil
-}
-
-func buildGeneratedClientCommonNameInternal(envName, safeEnvID string) string {
- safeEnvID = strings.TrimSpace(safeEnvID)
- if safeEnvID == "" {
- return ""
- }
-
- safeEnvName := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envName), "-")
- safeEnvName = strings.Trim(safeEnvName, "-_")
- if safeEnvName == "" {
- return safeEnvID
- }
-
- const maxCommonNameLength = 64
-
- maxEnvNameLength := maxCommonNameLength - len(safeEnvID) - 1
- if maxEnvNameLength <= 0 {
- return safeEnvID
- }
- if len(safeEnvName) > maxEnvNameLength {
- safeEnvName = strings.Trim(safeEnvName[:maxEnvNameLength], "-_")
- if safeEnvName == "" {
- return safeEnvID
- }
- }
-
- return fmt.Sprintf("%s-%s", safeEnvName, safeEnvID)
-}
-
-// buildGeneratedClientSANsInternal returns the URI and DNS Subject Alternative
-// Names to embed in a generated edge agent client certificate. The URI SAN
-// provides a stable machine-readable identity; DNS SANs improve interop with
-// stricter verifiers. Returns a nil URI if safeEnvID is empty.
-func buildGeneratedClientSANsInternal(envName, safeEnvID, appURL string) (*url.URL, []string) {
- safeEnvID = strings.TrimSpace(safeEnvID)
- if safeEnvID == "" {
- return nil, nil
- }
-
- uriSAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)
- trustDomain := certgen.EdgeMTLSTrustDomain(appURL)
-
- dnsSANs := []string{"arcane-agent"}
- safeEnvName := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envName), "-")
- safeEnvName = strings.Trim(safeEnvName, "-_.")
- if safeEnvName != "" && trustDomain != "" {
- dnsSANs = append(dnsSANs, safeEnvName+".agent."+trustDomain)
- }
-
- return uriSAN, dnsSANs
-}
-
-func validateGeneratedCAInternal(certPath, keyPath string) error {
- cert, err := readCertificateInternal(certPath)
- if err != nil {
- return err
- }
- if !cert.IsCA {
- return errors.New("generated CA certificate is not a CA")
- }
- if validateGeneratedKeyTypeErr := validateGeneratedKeyTypeInternal(cert.PublicKey, "generated CA certificate"); validateGeneratedKeyTypeErr != nil {
- return validateGeneratedKeyTypeErr
- }
- keyPEM, err := readCAKeyPEMInternal(keyPath)
- if err != nil {
- return err
- }
- privateKey, err := parsePrivateKeyPEMInternal(keyPEM, "generated CA")
- if err != nil {
- return err
- }
- if validateCertificateKeyPairErr := validateCertificateKeyPairInternal(cert, privateKey, "generated CA"); validateCertificateKeyPairErr != nil {
- return validateCertificateKeyPairErr
- }
- return nil
-}
-
-func validateGeneratedClientCertificateInternal(certPath, keyPath, expectedCommonName string, expectedURISAN *url.URL) error {
- cert, err := readCertificateInternal(certPath)
- if err != nil {
- return err
- }
- now := time.Now()
- if now.Before(cert.NotBefore) || !now.Before(cert.NotAfter) {
- return errors.New("generated client certificate is not currently valid")
- }
- if strings.TrimSpace(expectedCommonName) != "" && cert.Subject.CommonName != expectedCommonName {
- return fmt.Errorf("generated client certificate common name %q does not match expected %q", cert.Subject.CommonName, expectedCommonName)
- }
- if expectedURISAN != nil && !certificateHasURISANInternal(cert, expectedURISAN) {
- return fmt.Errorf("generated client certificate URI SAN does not match expected %s", expectedURISAN.String())
- }
- if validateGeneratedKeyTypeErr := validateGeneratedKeyTypeInternal(cert.PublicKey, "generated client certificate"); validateGeneratedKeyTypeErr != nil {
- return validateGeneratedKeyTypeErr
- }
- privateKey, err := readPrivateKeyInternal(keyPath)
- if err != nil {
- return err
+ return tls.Certificate{}, false, fmt.Errorf("failed to encrypt edge mTLS CA private key: %w", err)
}
- if validateCertificateKeyPairErr := validateCertificateKeyPairInternal(cert, privateKey, "generated client"); validateCertificateKeyPairErr != nil {
- return validateCertificateKeyPairErr
- }
- return nil
-}
-
-func certificateHasURISANInternal(cert *x509.Certificate, expected *url.URL) bool {
- if cert == nil || expected == nil {
- return false
+ if ciphertext == "" {
+ return tls.Certificate{}, false, errors.New("failed to encrypt edge mTLS CA private key: encrypted payload is empty")
}
- for _, uri := range cert.URIs {
- if uri == nil {
- continue
- }
- if strings.EqualFold(uri.Scheme, expected.Scheme) &&
- strings.EqualFold(strings.TrimSuffix(uri.Host, "."), strings.TrimSuffix(expected.Host, ".")) &&
- uri.Path == expected.Path {
- return true
- }
+ if writeErr := atomic.WriteFile(keyPath, []byte(caKeyEncryptedPrefix+ciphertext), 0o600); writeErr != nil {
+ return tls.Certificate{}, false, writeErr
}
- return false
-}
-func agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath string, now time.Time) (bool, string) {
- if !fileExistsInternal(certPath) || !fileExistsInternal(keyPath) {
- return true, "certificate or key is missing"
- }
- cert, err := readCertificateInternal(certPath)
- if err != nil {
- return true, err.Error()
- }
- privateKey, err := readPrivateKeyInternal(keyPath)
- if err != nil {
- return true, err.Error()
- }
- if validateCertificateKeyPairErr := validateCertificateKeyPairInternal(cert, privateKey, "edge mTLS client"); validateCertificateKeyPairErr != nil {
- return true, validateCertificateKeyPairErr.Error()
- }
- if now.IsZero() {
- now = time.Now()
- }
- if now.Before(cert.NotBefore) {
- return true, "certificate is not valid before " + cert.NotBefore.UTC().Format(time.RFC3339)
- }
- if !now.Before(cert.NotAfter) {
- return true, "certificate expired at " + cert.NotAfter.UTC().Format(time.RFC3339)
- }
- if now.Add(agentMTLSRenewBefore).After(cert.NotAfter) {
- return true, "certificate expires soon at " + cert.NotAfter.UTC().Format(time.RFC3339)
- }
- return false, ""
+ slog.InfoContext(ctx, "generated edge mTLS CA", "certPath", certPath)
+ return tls.Certificate{Certificate: [][]byte{certDER}, PrivateKey: privateKey, Leaf: leaf}, true, nil
}
-func validateCertificateKeyPairInternal(cert *x509.Certificate, privateKey crypto.Signer, label string) error {
- if cert == nil {
- return fmt.Errorf("%s certificate is required", label)
- }
- if privateKey == nil {
- return fmt.Errorf("%s private key is required", label)
- }
-
- certPublicKeyDER, err := x509.MarshalPKIXPublicKey(cert.PublicKey)
- if err != nil {
- return fmt.Errorf("failed to marshal %s certificate public key: %w", label, err)
+// loadClientCertificate loads a client key pair, or returns why it must be (re)issued: unreadable,
+// mismatched, outside its validity window, or within agentMTLSRenewBefore of expiry.
+func loadClientCertificate(certPath, keyPath string, now time.Time) (tls.Certificate, string) {
+ cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+ switch {
+ case err != nil:
+ return tls.Certificate{}, err.Error()
+ case now.Before(cert.Leaf.NotBefore):
+ return tls.Certificate{}, "certificate is not valid before " + cert.Leaf.NotBefore.UTC().Format(time.RFC3339)
+ case !now.Before(cert.Leaf.NotAfter):
+ return tls.Certificate{}, "certificate expired at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)
+ case now.Add(agentMTLSRenewBefore).After(cert.Leaf.NotAfter):
+ return tls.Certificate{}, "certificate expires soon at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)
}
- privatePublicKeyDER, err := x509.MarshalPKIXPublicKey(privateKey.Public())
- if err != nil {
- return fmt.Errorf("failed to marshal %s private key public key: %w", label, err)
- }
- if !bytes.Equal(certPublicKeyDER, privatePublicKeyDER) {
- return fmt.Errorf("%s certificate public key does not match private key", label)
- }
-
- return nil
+ return cert, ""
}
-func validateGeneratedKeyTypeInternal(publicKey any, label string) error {
+// supportedGeneratedKey reports whether a generated certificate uses ECDSA P-384 or ML-DSA-87.
+func supportedGeneratedKey(publicKey crypto.PublicKey) bool {
switch key := publicKey.(type) {
case *ecdsa.PublicKey:
- if key.Curve == elliptic.P384() {
- return nil
- }
+ return key.Curve == elliptic.P384()
case *mldsa.PublicKey:
- if key.Parameters() == mldsa.MLDSA87() {
- return nil
- }
- }
- return fmt.Errorf("%s is not ECDSA P-384 or ML-DSA-87", label)
-}
-
-func generateKeyLikeInternal(caKey crypto.Signer) (crypto.Signer, error) {
- if _, ok := caKey.(*ecdsa.PrivateKey); ok {
- return certgen.GenerateP384PrivateKey()
- }
- return certgen.GenerateMLDSA87PrivateKey()
-}
-
-func parsePrivateKeyPEMInternal(pemBytes []byte, label string) (crypto.Signer, error) {
- block, _ := pem.Decode(pemBytes)
- if block == nil {
- return nil, fmt.Errorf("failed to parse %s private key PEM", label)
- }
- if key, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil {
- signer, ok := key.(crypto.Signer)
- if !ok {
- return nil, fmt.Errorf("%s private key is not a signer", label)
- }
- return signer, nil
+ return key.Parameters() == mldsa.MLDSA87()
}
- key, err := x509.ParseECPrivateKey(block.Bytes)
- if err != nil {
- return nil, fmt.Errorf("failed to parse %s private key: %w", label, err)
- }
- return key, nil
-}
-
-func readCertificateInternal(localPath string) (*x509.Certificate, error) {
- // os.* rather than acfs: callers pass both generated-asset paths and
- // user-configured absolute cert paths, so no single confinement root exists.
- pemBytes, err := os.ReadFile(localPath)
- if err != nil {
- return nil, fmt.Errorf("failed to read certificate %s: %w", localPath, err)
- }
- block, _ := pem.Decode(pemBytes)
- if block == nil {
- return nil, fmt.Errorf("failed to parse certificate PEM %s", localPath)
- }
- cert, err := x509.ParseCertificate(block.Bytes)
- if err != nil {
- return nil, fmt.Errorf("failed to parse certificate %s: %w", localPath, err)
- }
- return cert, nil
-}
-
-func readPrivateKeyInternal(localPath string) (crypto.Signer, error) {
- // os.* rather than acfs: callers pass both generated-asset paths and
- // user-configured absolute key paths, so no single confinement root exists.
- pemBytes, err := os.ReadFile(localPath)
- if err != nil {
- return nil, fmt.Errorf("failed to read private key %s: %w", localPath, err)
- }
- return parsePrivateKeyPEMInternal(pemBytes, localPath)
+ return false
}
-func lockEdgeMTLSPathInternal(ctx context.Context, dir, lockName string) (func(), error) {
+// lockEdgeMTLSPath serializes asset generation in this process and, through an O_EXCL lock
+// file (which acfs cannot create), across processes sharing the directory.
+func lockEdgeMTLSPath(ctx context.Context, dir, lockName string) (func(), error) {
absDir, err := filepath.Abs(dir)
if err != nil {
return nil, fmt.Errorf("failed to resolve edge mTLS lock dir: %w", err)
}
- lockName = cmp.Or(strings.TrimSpace(lockName), ".lock")
-
lockPath := filepath.Join(absDir, lockName)
deadline := time.Now().Add(managerCALockTimeout)
for {
- unlock, held := managerCALocks.TryLock(lockPath)
- if !held {
- if waitForEdgeMTLSLockPollErr := waitForEdgeMTLSLockPollInternal(ctx); waitForEdgeMTLSLockPollErr != nil {
- return nil, waitForEdgeMTLSLockPollErr
+ if unlock, held := managerCALocks.TryLock(lockPath); held {
+ file, openErr := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
+ if openErr == nil {
+ _, _ = fmt.Fprintf(file, "%d %s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano))
+ _ = file.Close()
+ return func() {
+ _ = os.Remove(lockPath)
+ unlock()
+ }, nil
}
- continue
- }
- // os.* rather than acfs, here and in the stale-lock helpers below: acfs
- // has no exclusive-create (O_EXCL) lockfile API.
- file, openFileErr := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
- if openFileErr == nil {
- _, _ = fmt.Fprintf(file, "%d %s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano))
- _ = file.Close()
- return func() {
- _ = os.Remove(lockPath)
- unlock()
- }, nil
- }
- if !os.IsExist(openFileErr) {
+ expired := time.Now().After(deadline)
+ removedStale := os.IsExist(openErr) && expired && removeStaleLock(lockPath)
unlock()
- return nil, fmt.Errorf("failed to acquire edge mTLS CA lock: %w", openFileErr)
- }
- if time.Now().After(deadline) {
- if removeStaleEdgeMTLSLockInternal(lockPath) {
- // Retake the lock from the top; continuing while still holding it
- // makes the next TryLock fail against ourselves and spin until the
- // context is cancelled.
- unlock()
+ switch {
+ case !os.IsExist(openErr):
+ return nil, fmt.Errorf("failed to acquire edge mTLS CA lock: %w", openErr)
+ case removedStale:
+ // Retry at once with a fresh deadline now that the stale lock is gone.
deadline = time.Now().Add(managerCALockTimeout)
continue
+ case expired:
+ return nil, fmt.Errorf("timed out waiting for edge mTLS CA lock %s", lockPath)
}
- unlock()
- return nil, fmt.Errorf("timed out waiting for edge mTLS CA lock %s", lockPath)
}
- unlock()
- if waitForEdgeMTLSLockPollErr2 := waitForEdgeMTLSLockPollInternal(ctx); waitForEdgeMTLSLockPollErr2 != nil {
- return nil, waitForEdgeMTLSLockPollErr2
- }
- }
-}
-
-func waitForEdgeMTLSLockPollInternal(ctx context.Context) error {
- timer := time.NewTimer(managerCALockPollInterval)
- defer timer.Stop()
-
- select {
- case <-ctx.Done():
- return fmt.Errorf("cancelled waiting for edge mTLS CA lock: %w", ctx.Err())
- case <-timer.C:
- return nil
- }
-}
-func removeStaleEdgeMTLSLockInternal(lockPath string) bool {
- info, err := readEdgeMTLSLockInfoInternal(lockPath)
- if err != nil {
- return false
- }
- if !info.createdAt.IsZero() && time.Since(info.createdAt) > 2*managerCALockTimeout {
- return removeEdgeMTLSLockFileInternal(lockPath)
- }
- if edgeMTLSLockPIDAliveInternal(info.pid) {
- return false
+ select {
+ case <-ctx.Done():
+ return nil, fmt.Errorf("cancelled waiting for edge mTLS CA lock: %w", ctx.Err())
+ case <-time.After(managerCALockPollInterval):
+ }
}
- return removeEdgeMTLSLockFileInternal(lockPath)
}
-func readEdgeMTLSLockInfoInternal(lockPath string) (*edgeMTLSLockInfo, error) {
+// removeStaleLock removes a lock file whose owning process is gone or that is far older than any holder.
+func removeStaleLock(lockPath string) bool {
content, err := os.ReadFile(lockPath)
if err != nil {
- return nil, err
+ return false
}
fields := strings.Fields(string(content))
if len(fields) == 0 {
- return nil, errors.New("edge mTLS lock does not contain a PID")
- }
- pid, err := strconv.Atoi(fields[0])
- if err != nil {
- return nil, fmt.Errorf("parse edge mTLS lock PID: %w", err)
- }
- if pid <= 0 {
- return nil, errors.New("edge mTLS lock PID must be positive")
- }
- info := &edgeMTLSLockInfo{pid: pid}
- if len(fields) > 1 {
- createdAt, parseErr := time.Parse(time.RFC3339Nano, fields[1])
- if parseErr == nil {
- info.createdAt = createdAt
- }
- }
- return info, nil
-}
-
-func removeEdgeMTLSLockFileInternal(lockPath string) bool {
- if err := os.Remove(lockPath); err != nil && !errors.Is(err, os.ErrNotExist) {
return false
}
- return true
-}
-
-func edgeMTLSLockPIDAliveInternal(pid int) bool {
- process, err := os.FindProcess(pid)
- if err != nil {
+ pid, err := strconv.Atoi(fields[0])
+ if err != nil || pid <= 0 {
return false
}
- err = process.Signal(syscall.Signal(0))
- return err == nil || errors.Is(err, syscall.EPERM)
-}
-
-func writePEMFileInternal(localPath, blockType string, derBytes []byte, perm os.FileMode) error {
- pemBytes := pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: derBytes})
- if pemBytes == nil {
- return fmt.Errorf("failed to encode PEM file %s", localPath)
- }
- return atomic.WriteFile(localPath, pemBytes, perm)
-}
-var caKeyEncryptInternal = libcrypto.Encrypt
-
-// writeCAKeyFileInternal writes the edge CA private key to disk using envelope
-// encryption via libcrypto.
-func writeCAKeyFileInternal(localPath string, derBytes []byte) error {
- pemBytes := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: derBytes})
- if pemBytes == nil {
- return errors.New("failed to encode CA private key to PEM")
- }
-
- ciphertext, err := caKeyEncryptInternal(string(pemBytes))
- if err != nil {
- return fmt.Errorf("failed to encrypt edge mTLS CA private key: %w", err)
+ expired := false
+ if len(fields) > 1 {
+ if createdAt, parseErr := time.Parse(time.RFC3339Nano, fields[1]); parseErr == nil {
+ expired = time.Since(createdAt) > 2*managerCALockTimeout
+ }
}
- if ciphertext == "" {
- return errors.New("failed to encrypt edge mTLS CA private key: encrypted payload is empty")
+ if !expired {
+ // Signal 0 only probes the process; EPERM means it exists under another user.
+ if process, findErr := os.FindProcess(pid); findErr == nil {
+ if signalErr := process.Signal(syscall.Signal(0)); signalErr == nil || errors.Is(signalErr, syscall.EPERM) {
+ return false
+ }
+ }
}
- return atomic.WriteFile(localPath, []byte(caKeyEncryptedPrefix+ciphertext), 0o600)
+ removeErr := os.Remove(lockPath)
+ return removeErr == nil || errors.Is(removeErr, os.ErrNotExist)
}
-// readCAKeyPEMInternal returns the plain PEM bytes of the edge CA private key,
-// reading a libcrypto-envelope-encrypted file written by writeCAKeyFileInternal.
-// os.* rather than acfs: callers pass paths derived from a possibly
-// user-configured assets dir, so no confinement root handle is in scope.
-func readCAKeyPEMInternal(localPath string) ([]byte, error) {
- raw, err := os.ReadFile(localPath)
- if err != nil {
- return nil, fmt.Errorf("failed to read CA private key %s: %w", localPath, err)
- }
- trimmed := strings.TrimSpace(string(raw))
- if !strings.HasPrefix(trimmed, caKeyEncryptedPrefix) {
- return nil, fmt.Errorf("CA private key %s is not in the expected encrypted envelope format", localPath)
- }
- ciphertext := strings.TrimPrefix(trimmed, caKeyEncryptedPrefix)
- plaintext, err := libcrypto.Decrypt(ciphertext)
- if err != nil {
- return nil, fmt.Errorf("failed to decrypt CA private key %s: %w", localPath, err)
- }
- return []byte(plaintext), nil
-}
-
-func fileExistsInternal(localPath string) bool {
- if strings.TrimSpace(localPath) == "" {
- return false
- }
- // os.* rather than acfs: callers probe both generated-asset paths and
- // user-configured absolute paths, so no single confinement root exists.
+// fileExists reports whether localPath is a regular file. Paths may be configured anywhere on the host.
+func fileExists(localPath string) bool {
info, err := os.Stat(localPath)
return err == nil && !info.IsDir()
}
diff --git a/backend/pkg/libarcane/edge/tls_test.go b/backend/pkg/libarcane/edge/tls_test.go
index b2de181f4..e7e6f8861 100644
--- a/backend/pkg/libarcane/edge/tls_test.go
+++ b/backend/pkg/libarcane/edge/tls_test.go
@@ -25,16 +25,10 @@ import (
"github.com/stretchr/testify/require"
kit "go.getarcane.app/kit/pkg"
libcrypto "go.getarcane.app/sys/crypto"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/peer"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
-type testAuthInfo struct{}
-
-func (testAuthInfo) AuthType() string { return "test" }
-
func TestMain(m *testing.M) {
libcrypto.InitEncryption(&libcrypto.Config{
EncryptionKey: "test-encryption-key-for-edge-mtls-32bytes-min",
@@ -61,6 +55,14 @@ func TestPrepareManagerMTLSAssetsWithContext(t *testing.T) {
require.NoError(t, PrepareManagerMTLSAssetsWithContext(t.Context(), cfg))
require.NotEmpty(t, cfg.EdgeMTLSCAFile)
require.FileExists(t, cfg.EdgeMTLSCAFile)
+
+ configured := &Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSAssetsDir: t.TempDir(),
+ EdgeMTLSCAFile: filepath.Join(t.TempDir(), "ca.crt"),
+ }
+ require.NoError(t, PrepareManagerMTLSAssetsWithContext(t.Context(), configured))
+ require.NoFileExists(t, filepath.Join(configured.EdgeMTLSAssetsDir, generatedMTLSCACertFileName))
}
func TestGenerateManagerClientMTLSAssetsWithContext(t *testing.T) {
@@ -136,10 +138,10 @@ func TestGeneratedClientCertificate_IncludesSANs(t *testing.T) {
uri := cert.URIs[0]
require.Equal(t, "spiffe", uri.Scheme)
require.Equal(t, "manager.example.com", uri.Host)
- require.Contains(t, uri.Path, "env-abc")
+ require.Equal(t, "/edge/env-abc", uri.Path)
- require.NotEmpty(t, cert.DNSNames, "agent cert must include at least one DNS SAN")
require.Contains(t, cert.DNSNames, "arcane-agent")
+ require.Contains(t, cert.DNSNames, "Lab-Server.agent.manager.example.com")
require.True(t, cert.NotBefore.Before(cert.NotAfter))
skew := cert.NotAfter.Sub(cert.NotBefore)
@@ -157,6 +159,10 @@ func TestEnsureAgentMTLSAssets_RejectsPlainHTTPEnrollment(t *testing.T) {
err := EnsureAgentMTLSAssets(t.Context(), cfg)
require.Error(t, err)
require.Contains(t, err.Error(), "MANAGER_API_URL to use https for certificate enrollment")
+
+ cfg.EdgeMTLSCertFile = filepath.Join(t.TempDir(), "agent.crt")
+ cfg.EdgeMTLSKeyFile = filepath.Join(t.TempDir(), "agent.key")
+ require.NoError(t, EnsureAgentMTLSAssets(t.Context(), cfg), "configured client files skip enrollment")
}
func TestEnsureAgentMTLSAssets_LimitsEnrollmentErrorBody(t *testing.T) {
@@ -217,31 +223,31 @@ func TestEnsureAgentMTLSAssets_UsesDownloadedCAPathWhenPresent(t *testing.T) {
require.Equal(t, caPath, cfg.EdgeMTLSCAFile)
}
-func TestRemoveStaleEdgeMTLSLockInternal_PreservesLivePID(t *testing.T) {
+func TestRemoveStaleLock_PreservesLivePID(t *testing.T) {
lockPath := filepath.Join(t.TempDir(), ".ca.lock")
require.NoError(t, os.WriteFile(lockPath, fmt.Appendf(nil, "%d %s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano)), 0o600))
- require.False(t, removeStaleEdgeMTLSLockInternal(lockPath))
+ require.False(t, removeStaleLock(lockPath))
require.FileExists(t, lockPath)
}
-func TestRemoveStaleEdgeMTLSLockInternal_RemovesOldLockDespiteLivePID(t *testing.T) {
+func TestRemoveStaleLock_RemovesOldLockDespiteLivePID(t *testing.T) {
lockPath := filepath.Join(t.TempDir(), ".ca.lock")
oldTimestamp := time.Now().Add(-(2*managerCALockTimeout + time.Second)).UTC().Format(time.RFC3339Nano)
require.NoError(t, os.WriteFile(lockPath, fmt.Appendf(nil, "%d %s\n", os.Getpid(), oldTimestamp), 0o600))
- require.True(t, removeStaleEdgeMTLSLockInternal(lockPath))
+ require.True(t, removeStaleLock(lockPath))
require.NoFileExists(t, lockPath)
}
-func TestBuildManagerClientTLSConfigInternal_OptionalIgnoresBrokenClientCertificate(t *testing.T) {
+func TestBuildManagerClientTLSConfig_OptionalIgnoresBrokenClientCertificate(t *testing.T) {
assetsDir := t.TempDir()
certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)
keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)
require.NoError(t, os.WriteFile(certPath, []byte("not a cert"), 0o644))
require.NoError(t, os.WriteFile(keyPath, []byte("not a key"), 0o600))
- tlsConfig, err := buildManagerClientTLSConfigInternal(&Config{
+ tlsConfig, err := buildManagerClientTLSConfig(&Config{
ManagerApiUrl: "https://manager.example.com",
EdgeMTLSMode: EdgeMTLSModeOptional,
EdgeMTLSCertFile: certPath,
@@ -252,21 +258,7 @@ func TestBuildManagerClientTLSConfigInternal_OptionalIgnoresBrokenClientCertific
require.Empty(t, tlsConfig.Certificates)
}
-func TestVerifiedPeerCertificateEnvironmentIDMatchesInternal(t *testing.T) {
- uriSAN, err := url.Parse("spiffe://manager.example.com/edge/env-123")
- require.NoError(t, err)
- cert := &x509.Certificate{URIs: []*url.URL{uriSAN}}
- state := &tls.ConnectionState{
- PeerCertificates: []*x509.Certificate{cert},
- VerifiedChains: [][]*x509.Certificate{{cert}},
- }
-
- require.NoError(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(state, "env-123", "manager.example.com"))
- require.Error(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(state, "env-456", "manager.example.com"))
- require.Error(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(state, "env-123", "other.example.com"))
-}
-
-func TestTunnelServerRequireCertificateIdentityInternal_RejectsWrongEnvironmentURI(t *testing.T) {
+func TestTunnelServerRequireCertificateIdentity_RejectsWrongEnvironmentURI(t *testing.T) {
uriSAN, err := url.Parse("spiffe://manager.example.com/edge/env-a")
require.NoError(t, err)
cert := &x509.Certificate{URIs: []*url.URL{uriSAN}}
@@ -275,13 +267,16 @@ func TestTunnelServerRequireCertificateIdentityInternal_RejectsWrongEnvironmentU
VerifiedChains: [][]*x509.Certificate{{cert}},
}
server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)
- server.SetConfig(&Config{
+ server.Config = &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
AppURL: "https://manager.example.com",
- })
+ }
+
+ require.NoError(t, server.requireCertificateIdentity(state, "env-a"))
+ require.Error(t, server.requireCertificateIdentity(state, "env-b"))
- require.NoError(t, server.requireCertificateIdentityInternal(state, "env-a"))
- require.Error(t, server.requireCertificateIdentityInternal(state, "env-b"))
+ server.Config.AppURL = "https://other.example.com"
+ require.Error(t, server.requireCertificateIdentity(state, "env-a"), "a different trust domain must not match")
}
func TestValidateManagerMTLSConfig_DoesNotRequireArcaneTLSTermination(t *testing.T) {
@@ -290,8 +285,9 @@ func TestValidateManagerMTLSConfig_DoesNotRequireArcaneTLSTermination(t *testing
}))
assetsDir := t.TempDir()
- caPath, _, _, err := ensureManagerCAInternal(t.Context(), assetsDir)
+ _, _, err := ensureManagerCA(t.Context(), assetsDir)
require.NoError(t, err)
+ caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)
err = ValidateManagerMTLSConfig(&Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
@@ -325,33 +321,33 @@ func TestValidateManagerMTLSConfig_RejectsMissingOrMalformedCAFile(t *testing.T)
func TestTunnelServerRequiredMTLS_AllowsHTTPRequestsWithoutVisibleTLSState(t *testing.T) {
server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)
- server.SetConfig(&Config{
+ server.Config = &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
AppURL: "https://manager.example.com",
- })
+ }
req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)
- require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))
+ require.NoError(t, server.requireCertificateIdentity(req.TLS, "env-a"))
}
func TestTunnelServerRequiredMTLS_RejectsDirectTLSWithoutVerifiedClientCertificate(t *testing.T) {
server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)
- server.SetConfig(&Config{EdgeMTLSMode: EdgeMTLSModeRequired})
+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired}
req := httptest.NewRequest(http.MethodGet, "https://manager.example.com/api/tunnel/connect", http.NoBody)
req.TLS = &tls.ConnectionState{}
- require.ErrorContains(t, server.requireRequestCertificateIdentityInternal(req, "env-a"), "verified edge mTLS client certificate is required")
+ require.ErrorContains(t, server.requireCertificateIdentity(req.TLS, "env-a"), "verified edge mTLS client certificate is required")
- server.SetConfig(&Config{EdgeMTLSMode: EdgeMTLSModeOptional})
- require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))
+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeOptional}
+ require.NoError(t, server.requireCertificateIdentity(req.TLS, "env-a"))
}
func TestTunnelServerRequiredMTLS_DetectsOnlyDirectTLSForRequestSecurityMode(t *testing.T) {
server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)
- server.SetConfig(&Config{
+ server.Config = &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
AppURL: "https://manager.example.com",
- })
+ }
cert := &x509.Certificate{}
req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)
@@ -359,73 +355,15 @@ func TestTunnelServerRequiredMTLS_DetectsOnlyDirectTLSForRequestSecurityMode(t *
PeerCertificates: []*x509.Certificate{cert},
VerifiedChains: [][]*x509.Certificate{{cert}},
}
- require.Equal(t, "mtls", requestSecurityModeInternal(req))
+ require.True(t, hasVerifiedPeerCertificate(req.TLS))
req = httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)
req.Header.Set("X-SSL-Client-Verify", "SUCCESS")
- require.Equal(t, "token", requestSecurityModeInternal(req))
- require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))
-}
-
-func TestTunnelServerRequiredMTLS_IgnoresProxyVerificationHeaders(t *testing.T) {
- tests := []struct {
- name string
- value string
- }{
- {name: "empty", value: ""},
- {name: "success", value: "SUCCESS"},
- {name: "true", value: "true"},
- {name: "none", value: "NONE"},
- {name: "failed", value: "FAILED"},
- {name: "unknown", value: "probably"},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)
- req.Header.Set("X-SSL-Client-Verify", tt.value)
-
- require.Equal(t, "token", requestSecurityModeInternal(req))
- })
- }
+ require.False(t, hasVerifiedPeerCertificate(req.TLS))
+ require.NoError(t, server.requireCertificateIdentity(req.TLS, "env-a"))
}
-func TestTunnelServerRequiredMTLS_AllowsGRPCContextsWithoutVisibleTLSState(t *testing.T) {
- server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)
- server.SetConfig(&Config{EdgeMTLSMode: EdgeMTLSModeRequired})
-
- t.Run("missing peer", func(t *testing.T) {
- require.NoError(t, server.requireCertificateIdentityFromContextInternal(t.Context(), "env-a"))
- })
-
- t.Run("non tls peer", func(t *testing.T) {
- ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: testAuthInfo{}})
- require.NoError(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"))
- })
-
- t.Run("direct tls without verified client certificate", func(t *testing.T) {
- ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{}})
- require.ErrorContains(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"), "verified edge mTLS client certificate is required")
- })
-
- t.Run("verified tls peer checks identity", func(t *testing.T) {
- uriSAN, err := url.Parse("spiffe://manager.example.com/edge/env-a")
- require.NoError(t, err)
- cert := &x509.Certificate{URIs: []*url.URL{uriSAN}}
- ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{
- PeerCertificates: []*x509.Certificate{cert},
- VerifiedChains: [][]*x509.Certificate{{cert}},
- }}})
- server.SetConfig(&Config{
- EdgeMTLSMode: EdgeMTLSModeRequired,
- AppURL: "https://manager.example.com",
- })
- require.NoError(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"))
- require.Error(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-b"))
- })
-}
-
-func TestAgentMTLSAssetsNeedEnrollmentInternal_RenewsExpiredCertificate(t *testing.T) {
+func TestLoadClientCertificate_RenewsExpiredCertificate(t *testing.T) {
assetsDir := t.TempDir()
assets, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
@@ -440,58 +378,54 @@ func TestAgentMTLSAssetsNeedEnrollmentInternal_RenewsExpiredCertificate(t *testi
require.NoError(t, os.WriteFile(targetPath, []byte(file.Content), perm))
}
- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(
- filepath.Join(assetsDir, generatedMTLSClientCertName),
- filepath.Join(assetsDir, generatedMTLSClientKeyName),
- time.Now().Add(generatedMTLSCertValidity+24*time.Hour),
- )
- require.True(t, needsEnrollment)
+ certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)
+ keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)
+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ require.Empty(t, reason)
+
+ _, reason = loadClientCertificate(certPath, keyPath, cert.Leaf.NotAfter.Add(24*time.Hour))
require.Contains(t, reason, "expired")
}
-func TestValidateGeneratedClientCertificateInternal_RejectsMismatchedKeyPair(t *testing.T) {
- assetsDir := t.TempDir()
-
- clientCertPath, _, _, err := ensureClientCertificateInternal(t.Context(), assetsDir, "env-123", "Lab Server", "https://manager.example.com")
+func TestGenerateManagerClientMTLSAssets_ReissuesMismatchedKeyPair(t *testing.T) {
+ cfg := &Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSAssetsDir: t.TempDir(),
+ AppURL: "https://manager.example.com",
+ }
+ _, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "Lab Server")
require.NoError(t, err)
replacementKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
require.NoError(t, err)
-
replacementKeyDER, err := x509.MarshalECPrivateKey(replacementKey)
require.NoError(t, err)
+ clientKeyPath := filepath.Join(cfg.EdgeMTLSAssetsDir, generatedClientMTLSSubdir, "env-123", generatedMTLSClientKeyName)
+ require.NoError(t, os.WriteFile(clientKeyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: replacementKeyDER}), 0o600))
- clientKeyPath := filepath.Join(assetsDir, generatedClientMTLSSubdir, "env-123", generatedMTLSClientKeyName)
- err = writePEMFileInternal(clientKeyPath, "EC PRIVATE KEY", replacementKeyDER, 0o600)
+ assets, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "Lab Server")
require.NoError(t, err)
-
- expectedURI, err := url.Parse("spiffe://manager.example.com/edge/env-123")
+ require.True(t, assets.CertIssued, "a key that no longer matches the certificate must be reissued")
+ _, err = tls.X509KeyPair([]byte(assets.Files[1].Content), []byte(assets.Files[2].Content))
require.NoError(t, err)
-
- err = validateGeneratedClientCertificateInternal(clientCertPath, clientKeyPath, "Lab-Server-env-123", expectedURI)
- require.Error(t, err)
- require.Contains(t, err.Error(), "does not match private key")
}
-func TestEnsureClientCertificateInternal_PreservesCertificateWhenEnvironmentNameChanges(t *testing.T) {
- assetsDir := t.TempDir()
-
- clientCertPath, _, _, err := ensureClientCertificateInternal(t.Context(), assetsDir, "env-123", "", "https://manager.example.com")
- require.NoError(t, err)
-
- originalPEM, err := os.ReadFile(clientCertPath)
- require.NoError(t, err)
-
- clientCertPath, _, _, err = ensureClientCertificateInternal(t.Context(), assetsDir, "env-123", "Lab Server", "https://manager.example.com")
+func TestGenerateManagerClientMTLSAssets_PreservesCertificateWhenEnvironmentNameChanges(t *testing.T) {
+ cfg := &Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSAssetsDir: t.TempDir(),
+ AppURL: "https://manager.example.com",
+ }
+ original, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "")
require.NoError(t, err)
- updatedPEM, err := os.ReadFile(clientCertPath)
+ renamed, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "Lab Server")
require.NoError(t, err)
- require.Equal(t, string(originalPEM), string(updatedPEM))
+ require.False(t, renamed.CertIssued)
+ require.Equal(t, original.Files[1].Content, renamed.Files[1].Content)
- certBlock, _ := pem.Decode(updatedPEM)
+ certBlock, _ := pem.Decode([]byte(renamed.Files[1].Content))
require.NotNil(t, certBlock)
-
cert, err := x509.ParseCertificate(certBlock.Bytes)
require.NoError(t, err)
require.Equal(t, "env-123", cert.Subject.CommonName)
@@ -501,114 +435,73 @@ func TestCAKey_EncryptedOnDiskWhenCryptoInitialized(t *testing.T) {
initEdgeTestCrypto(t)
assetsDir := t.TempDir()
- caCertPath, caKeyPath, _, err := ensureManagerCAInternal(t.Context(), assetsDir)
+ ca, generated, err := ensureManagerCA(t.Context(), assetsDir)
require.NoError(t, err)
- require.FileExists(t, caCertPath)
- require.FileExists(t, caKeyPath)
+ require.True(t, generated)
- raw, err := os.ReadFile(caKeyPath)
+ raw, err := os.ReadFile(filepath.Join(assetsDir, generatedMTLSCAKeyFileName))
require.NoError(t, err)
- require.NotContains(t, string(raw), "BEGIN PRIVATE KEY",
- "encrypted CA key file must not contain plain PEM markers")
+ require.True(t, strings.HasPrefix(string(raw), caKeyEncryptedPrefix))
+ require.NotContains(t, string(raw), "BEGIN PRIVATE KEY", "encrypted CA key file must not contain plain PEM markers")
- pemBytes, err := readCAKeyPEMInternal(caKeyPath)
- require.NoError(t, err)
- block, _ := pem.Decode(pemBytes)
- require.NotNil(t, block)
- _, err = x509.ParsePKCS8PrivateKey(block.Bytes)
+ reused, generated, err := ensureManagerCA(t.Context(), assetsDir)
require.NoError(t, err)
+ require.False(t, generated, "the encrypted CA key must decrypt and be reused")
+ require.Equal(t, ca.Certificate, reused.Certificate)
- clientCertPath, clientKeyPath, _, err := ensureClientCertificateInternal(t.Context(), assetsDir, "env-round", "Round Trip", "https://manager.example.com")
+ _, err = GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSAssetsDir: assetsDir,
+ AppURL: "https://manager.example.com",
+ }, "env-round", "Round Trip")
require.NoError(t, err)
- require.FileExists(t, clientCertPath)
- require.FileExists(t, clientKeyPath)
}
func TestCAKey_EncryptionFailureReturnsError(t *testing.T) {
- originalEncrypt := caKeyEncryptInternal
+ originalEncrypt := caKeyEncrypt
t.Cleanup(func() {
- caKeyEncryptInternal = originalEncrypt
+ caKeyEncrypt = originalEncrypt
})
-
- caKeyPath := filepath.Join(t.TempDir(), "ca.key")
- caKeyEncryptInternal = func(string) (string, error) {
+ caKeyEncrypt = func(string) (string, error) {
return "", errors.New("encrypt failed")
}
- err := writeCAKeyFileInternal(caKeyPath, []byte{1, 2, 3})
+ assetsDir := t.TempDir()
+ _, _, err := ensureManagerCA(t.Context(), assetsDir)
require.Error(t, err)
require.Contains(t, err.Error(), "failed to encrypt edge mTLS CA private key")
- require.NoFileExists(t, caKeyPath)
+ require.NoFileExists(t, filepath.Join(assetsDir, generatedMTLSCAKeyFileName))
}
-func TestLockEdgeMTLSPathInternal_ReturnsOnContextCancellation(t *testing.T) {
+func TestLockEdgeMTLSPath_ReturnsOnContextCancellation(t *testing.T) {
assetsDir := t.TempDir()
- unlock, err := lockEdgeMTLSPathInternal(t.Context(), assetsDir, ".ca.lock")
+ unlock, err := lockEdgeMTLSPath(t.Context(), assetsDir, ".ca.lock")
require.NoError(t, err)
defer unlock()
ctx, cancel := context.WithCancel(t.Context())
cancel()
- _, err = lockEdgeMTLSPathInternal(ctx, assetsDir, ".ca.lock")
+ _, err = lockEdgeMTLSPath(ctx, assetsDir, ".ca.lock")
require.Error(t, err)
require.Contains(t, err.Error(), "cancelled waiting for edge mTLS CA lock")
}
func TestCAKey_EmptyEncryptionPayloadReturnsError(t *testing.T) {
- originalEncrypt := caKeyEncryptInternal
+ originalEncrypt := caKeyEncrypt
t.Cleanup(func() {
- caKeyEncryptInternal = originalEncrypt
+ caKeyEncrypt = originalEncrypt
})
-
- caKeyPath := filepath.Join(t.TempDir(), "ca.key")
- caKeyEncryptInternal = func(string) (string, error) {
+ caKeyEncrypt = func(string) (string, error) {
return "", nil
}
- err := writeCAKeyFileInternal(caKeyPath, []byte{1, 2, 3})
+ assetsDir := t.TempDir()
+ _, _, err := ensureManagerCA(t.Context(), assetsDir)
require.Error(t, err)
require.Contains(t, err.Error(), "encrypted payload is empty")
- require.NoFileExists(t, caKeyPath)
-}
-
-func TestBuildGeneratedClientSANsInternal(t *testing.T) {
- uri, dns := buildGeneratedClientSANsInternal("Lab Server", "env-123", "https://manager.example.com")
- require.NotNil(t, uri)
- require.Equal(t, "spiffe", uri.Scheme)
- require.Equal(t, "manager.example.com", uri.Host)
- require.Equal(t, "/edge/env-123", uri.Path)
- require.Contains(t, dns, "arcane-agent")
- require.Contains(t, dns, "Lab-Server.agent.manager.example.com")
-
- uriEmpty, dnsEmpty := buildGeneratedClientSANsInternal("", "", "https://manager.example.com")
- require.Nil(t, uriEmpty)
- require.Nil(t, dnsEmpty)
-}
-
-func TestShouldAutoGenerateManagerCAInternal(t *testing.T) {
- cfg := &Config{
- EdgeMTLSMode: EdgeMTLSModeRequired,
- EdgeMTLSAssetsDir: t.TempDir(),
- }
-
- require.True(t, shouldAutoGenerateManagerCAInternal(cfg))
-
- cfg.EdgeMTLSCAFile = filepath.Join(t.TempDir(), "ca.crt")
- require.False(t, shouldAutoGenerateManagerCAInternal(cfg))
-}
-
-func TestShouldAutoEnrollAgentMTLSInternal(t *testing.T) {
- cfg := &Config{
- EdgeMTLSMode: EdgeMTLSModeRequired,
- }
-
- require.True(t, shouldAutoEnrollAgentMTLSInternal(cfg))
-
- cfg.EdgeMTLSCertFile = filepath.Join(t.TempDir(), "agent.crt")
- cfg.EdgeMTLSKeyFile = filepath.Join(t.TempDir(), "agent.key")
- require.False(t, shouldAutoEnrollAgentMTLSInternal(cfg))
+ require.NoFileExists(t, filepath.Join(assetsDir, generatedMTLSCAKeyFileName))
}
func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.T) {
@@ -622,22 +515,15 @@ func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.
require.NoError(t, err)
caCertPath := filepath.Join(assetsDir, "ca.crt")
- caCertPEM, err := os.ReadFile(caCertPath)
- require.NoError(t, err)
- caBlock, _ := pem.Decode(caCertPEM)
- require.NotNil(t, caBlock)
- caCert, err := x509.ParseCertificate(caBlock.Bytes)
- require.NoError(t, err)
- caKeyPEM, err := readCAKeyPEMInternal(filepath.Join(assetsDir, "ca.key"))
- require.NoError(t, err)
- caKey, err := parsePrivateKeyPEMInternal(caKeyPEM, "CA")
+ ca, caGenerated, err := ensureManagerCA(t.Context(), assetsDir)
require.NoError(t, err)
+ require.False(t, caGenerated)
serverKey, err := certgen.GenerateMLDSA87PrivateKey()
require.NoError(t, err)
serverTemplate, err := certgen.NewServerTLSTemplate("localhost", []string{"localhost", "127.0.0.1"})
require.NoError(t, err)
- serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caCert, serverKey.PublicKey(), caKey)
+ serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, ca.Leaf, serverKey.PublicKey(), ca.PrivateKey)
require.NoError(t, err)
serverTLS, err := BuildManagerServerTLSConfig(&Config{
@@ -650,7 +536,7 @@ func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.
clientCertPath, err := GeneratedManagerClientMTLSCertPath(cfg, "env-hs")
require.NoError(t, err)
- clientTLS, err := buildManagerClientTLSConfigInternal(&Config{
+ clientTLS, err := buildManagerClientTLSConfig(&Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
EdgeMTLSCAFile: caCertPath,
EdgeMTLSCertFile: clientCertPath,
@@ -694,6 +580,8 @@ func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.
result := <-resultCh
require.NoError(t, result.err)
require.Equal(t, uint16(tls.VersionTLS13), result.state.Version)
- require.True(t, hasVerifiedPeerCertificateInternal(&result.state))
- require.NoError(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(&result.state, "env-hs", "manager.example.com"))
+ require.True(t, hasVerifiedPeerCertificate(&result.state))
+ server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)
+ server.Config = cfg
+ require.NoError(t, server.requireCertificateIdentity(&result.state, "env-hs"))
}
diff --git a/backend/pkg/libarcane/edge/tunnel.go b/backend/pkg/libarcane/edge/tunnel.go
index 2a61d89d1..232b5abb2 100644
--- a/backend/pkg/libarcane/edge/tunnel.go
+++ b/backend/pkg/libarcane/edge/tunnel.go
@@ -7,6 +7,7 @@ import (
"io"
"log/slog"
"net"
+ "sync/atomic"
"github.com/coder/websocket"
"google.golang.org/grpc/codes"
@@ -26,9 +27,6 @@ const (
// gRPC limit, since JSON base64-encodes binary bodies a gRPC peer would accept.
maxWebSocketTunnelMessageSize = 2 * maxGRPCTunnelMessageSize
- // ErrTunnelConnectionClosed is returned by Send and Receive on every transport
- // once the tunnel connection is closed.
-
// MessageTypeRequest is sent from manager to agent to initiate a request.
MessageTypeRequest TunnelMessageType = "request"
// MessageTypeResponse is sent from agent to manager with the response.
@@ -69,15 +67,17 @@ const (
MessageTypeStreamClose TunnelMessageType = "stream_close"
// MessageTypeCancelRequest requests cancellation of an in-flight command.
MessageTypeCancelRequest TunnelMessageType = "cancel_request"
+ // MessageTypeCommandCredit returns flow-control credit for consumed command output.
+ MessageTypeCommandCredit TunnelMessageType = "command_credit"
)
-var ErrTunnelConnectionClosed = errors.New("edge tunnel connection is closed")
-
-// tunnelReadWait bounds how long a websocket tunnel read may sit idle. Both
-// peers see traffic at least every DefaultHeartbeatInterval (agent heartbeat,
-// manager heartbeat_ack), so 3x tolerates transient stalls while still
-// detecting a silently dead peer. Variable so tests can shorten it.
-var tunnelReadWait = 3 * DefaultHeartbeatInterval
+var (
+ // ErrTunnelConnectionClosed is returned by Send and Receive on every transport once the connection is closed.
+ ErrTunnelConnectionClosed = errors.New("edge tunnel connection is closed")
+ // tunnelReadWait bounds an idle websocket read; heartbeats arrive every DefaultHeartbeatInterval.
+ // It is a variable so tests can shorten it.
+ tunnelReadWait = 3 * DefaultHeartbeatInterval
+)
// TunnelConnection is the transport contract shared by WebSocket and gRPC wrappers.
type TunnelConnection interface {
@@ -143,18 +143,7 @@ func (t *TunnelConn) Receive() (*TunnelMessage, error) {
// IsExpectedReceiveError returns true for normal WebSocket close/teardown errors.
func (t *TunnelConn) IsExpectedReceiveError(err error) bool {
- if err == nil {
- return false
- }
-
- if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
- return true
- }
- if errors.Is(err, net.ErrClosed) || errors.Is(err, ErrTunnelConnectionClosed) {
- return true
- }
-
- return wshub.IsExpectedClose(err)
+ return isExpectedReceiveError(err)
}
// Close closes the WebSocket tunnel connection, sending a close frame on the
@@ -208,6 +197,7 @@ func NewGRPCManagerTunnelConn(stream grpcManagerStream) *GRPCManagerTunnelConn {
stream: &cancelableGRPCManagerStream{
stream: stream,
ctx: recvCtx,
+ recvCh: make(chan grpcRecvResult),
},
cancel: cancel,
}
@@ -222,61 +212,34 @@ func (t *GRPCManagerTunnelConn) Send(msg *TunnelMessage) error {
return ErrTunnelConnectionClosed
}
- protoMsg, err := tunnelMessageToManagerProto(msg, t.parityEncoding.Load())
+ protoMsg, err := tunnelMessageToManagerProto(msg, t.parity)
if err != nil {
return err
}
if sendErr := t.stream.Send(protoMsg); sendErr != nil {
- t.markClosed()
+ t.closed.Store(true)
return sendErr
}
return nil
}
-// SetParityEncoding enables native proto encodings for message types that are
-// otherwise re-encoded for legacy agents. Set when the agent advertises the
-// proto-parity-v1 capability during registration.
-func (t *GRPCManagerTunnelConn) SetParityEncoding(enabled bool) {
- t.parityEncoding.Store(enabled)
-}
-
// Receive receives an agent->manager tunnel message from gRPC.
func (t *GRPCManagerTunnelConn) Receive() (*TunnelMessage, error) {
if t.stream == nil {
return nil, ErrTunnelConnectionClosed
}
-
- for {
- protoMsg, err := t.stream.Recv()
- if err != nil {
- // A gRPC stream is dead after any Recv error, not just io.EOF.
- t.markClosed()
- return nil, err
- }
-
- msg, err := agentProtoToTunnelMessage(protoMsg)
- if err != nil {
- if errors.Is(err, errUnknownTunnelPayload) {
- // A newer peer sent a payload this build cannot decode yet;
- // skip it like unknown websocket message types are skipped.
- slog.DebugContext(t.stream.Context(), "Ignoring unknown edge tunnel payload", "error", err)
- continue
- }
- return nil, err
- }
- return msg, nil
- }
+ return receiveGRPC(t.stream.Context(), t.stream.Recv, agentProtoToTunnelMessage, &t.closed)
}
// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.
func (t *GRPCManagerTunnelConn) IsExpectedReceiveError(err error) bool {
- return isExpectedGRPCReceiveErrorInternal(err)
+ return isExpectedReceiveError(err)
}
// Close marks the stream closed on manager side.
func (t *GRPCManagerTunnelConn) Close() error {
- t.markClosed()
+ t.closed.Store(true)
if t.cancel != nil {
t.cancel()
}
@@ -288,10 +251,6 @@ func (t *GRPCManagerTunnelConn) IsClosed() bool {
return t.closed.Load()
}
-func (t *GRPCManagerTunnelConn) markClosed() {
- t.closed.Store(true)
-}
-
// Transport identifies the underlying tunnel transport.
func (t *GRPCManagerTunnelConn) Transport() string {
return EdgeTransportGRPC
@@ -322,7 +281,7 @@ func (t *GRPCAgentTunnelConn) Send(msg *TunnelMessage) error {
}
if sendErr := t.stream.Send(protoMsg); sendErr != nil {
- t.markClosed()
+ t.closed.Store(true)
return sendErr
}
return nil
@@ -333,37 +292,16 @@ func (t *GRPCAgentTunnelConn) Receive() (*TunnelMessage, error) {
if t.stream == nil {
return nil, ErrTunnelConnectionClosed
}
-
- for {
- protoMsg, err := t.stream.Recv()
- if err != nil {
- // A gRPC stream is dead after any Recv error, not just io.EOF.
- t.markClosed()
- return nil, err
- }
-
- msg, err := managerProtoToTunnelMessage(protoMsg)
- if err != nil {
- if errors.Is(err, errUnknownTunnelPayload) {
- // A newer peer sent a payload this build cannot decode yet;
- // skip it like unknown websocket message types are skipped.
- slog.DebugContext(t.stream.Context(), "Ignoring unknown edge tunnel payload", "error", err)
- continue
- }
- return nil, err
- }
- return msg, nil
- }
+ return receiveGRPC(t.stream.Context(), t.stream.Recv, managerProtoToTunnelMessage, &t.closed)
}
// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.
func (t *GRPCAgentTunnelConn) IsExpectedReceiveError(err error) bool {
- return isExpectedGRPCReceiveErrorInternal(err)
+ return isExpectedReceiveError(err)
}
-// Close closes the client send stream. The half-close happens before cancel so
-// the manager observes a clean io.EOF (the gRPC analog of a websocket close
-// frame) instead of codes.Canceled.
+// Close half-closes the send stream before cancelling, so the manager sees a
+// clean io.EOF instead of codes.Canceled.
func (t *GRPCAgentTunnelConn) Close() error {
if t.closed.Swap(true) {
return nil
@@ -394,10 +332,6 @@ func (t *GRPCAgentTunnelConn) IsClosed() bool {
return t.closed.Load()
}
-func (t *GRPCAgentTunnelConn) markClosed() {
- t.closed.Store(true)
-}
-
// Transport identifies the underlying tunnel transport.
func (t *GRPCAgentTunnelConn) Transport() string {
return EdgeTransportGRPC
@@ -407,22 +341,29 @@ func (s *cancelableGRPCManagerStream) Send(msg *tunnelpb.ManagerMessage) error {
return s.stream.Send(msg)
}
+// Recv returns the next message, or ctx's error once ctx is done. gRPC Recv ignores ctx,
+// so a single reader goroutine waits on the stream instead.
func (s *cancelableGRPCManagerStream) Recv() (*tunnelpb.AgentMessage, error) {
- type recvResult struct {
- msg *tunnelpb.AgentMessage
- err error
- }
-
- recvCh := make(chan recvResult, 1)
- go func() {
- msg, err := s.stream.Recv()
- recvCh <- recvResult{msg: msg, err: err}
- }()
+ s.recvOnce.Do(func() {
+ go func() {
+ for {
+ msg, err := s.stream.Recv()
+ select {
+ case s.recvCh <- grpcRecvResult{msg: msg, err: err}:
+ case <-s.ctx.Done():
+ return
+ }
+ if err != nil {
+ return
+ }
+ }
+ }()
+ })
select {
case <-s.ctx.Done():
return nil, s.ctx.Err()
- case result := <-recvCh:
+ case result := <-s.recvCh:
return result.msg, result.err
}
}
@@ -431,18 +372,35 @@ func (s *cancelableGRPCManagerStream) Context() context.Context {
return s.ctx
}
-func isExpectedGRPCReceiveErrorInternal(err error) bool {
- if err == nil {
- return false
+// receiveGRPC decodes the next stream message, skipping payloads from newer peers
+// that this build cannot decode, as unknown websocket message types are skipped.
+func receiveGRPC[P any](ctx context.Context, recv func() (P, error), decode func(P) (*TunnelMessage, error), closed *atomic.Bool) (*TunnelMessage, error) {
+ for {
+ protoMsg, err := recv()
+ if err != nil {
+ // A gRPC stream is dead after any Recv error, not just io.EOF.
+ closed.Store(true)
+ return nil, err
+ }
+ msg, err := decode(protoMsg)
+ if errors.Is(err, errUnknownTunnelPayload) {
+ slog.DebugContext(ctx, "Ignoring unknown edge tunnel payload", "error", err)
+ continue
+ }
+ return msg, err
}
+}
- if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
- return true
+// isExpectedReceiveError reports normal teardown on either transport: EOF, cancellation,
+// a closed connection, a clean websocket close, or a cancelled gRPC stream.
+func isExpectedReceiveError(err error) bool {
+ if err == nil {
+ return false
}
- if errors.Is(err, ErrTunnelConnectionClosed) {
+ if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) ||
+ errors.Is(err, net.ErrClosed) || errors.Is(err, ErrTunnelConnectionClosed) || wshub.IsExpectedClose(err) {
return true
}
-
code := status.Code(err)
return code == codes.Canceled || code == codes.DeadlineExceeded
}
diff --git a/backend/pkg/libarcane/edge/tunnel_proto_adapter.go b/backend/pkg/libarcane/edge/tunnel_proto_adapter.go
index e2ba97d1a..ea989394a 100644
--- a/backend/pkg/libarcane/edge/tunnel_proto_adapter.go
+++ b/backend/pkg/libarcane/edge/tunnel_proto_adapter.go
@@ -1,27 +1,30 @@
package edge
import (
+ "bytes"
"errors"
"fmt"
"maps"
"math"
+ "slices"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)
-// errUnknownTunnelPayload marks a proto payload this peer cannot decode, e.g.
-// a oneof case added by a newer peer. Receivers skip such messages instead of
-// killing the stream, mirroring how unknown JSON message types are ignored on
-// the websocket transport.
+// errUnknownTunnelPayload marks a proto payload this peer cannot decode, such as a oneof case
+// added by a newer peer. Receivers skip it, as unknown websocket message types are skipped.
var errUnknownTunnelPayload = errors.New("unknown tunnel payload type")
-// tunnelMessageToManagerProto encodes a manager->agent message. parity is true
-// once the agent advertised tunnelCapabilityProtoParity; without it, legacy
-// agents receive stream_data re-encoded as ws_data and cannot receive stream_end.
+// tunnelMessageToManagerProto encodes a manager->agent message. Without parity, legacy agents
+// receive stream_data re-encoded as ws_data and cannot receive stream_end.
func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManagerMessage, error) {
if msg == nil {
return nil, errors.New("message is nil")
}
+ messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")
+ if err != nil {
+ return nil, err
+ }
switch msg.Type {
case MessageTypeRequest:
@@ -30,7 +33,7 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Man
Method: msg.Method,
Path: msg.Path,
Query: msg.Query,
- Headers: cloneHeaderMap(msg.Headers),
+ Headers: maps.Clone(msg.Headers),
Body: msg.Body,
}}}, nil
case MessageTypeHeartbeatAck:
@@ -40,24 +43,10 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Man
StreamId: msg.ID,
Path: msg.Path,
Query: msg.Query,
- Headers: cloneHeaderMap(msg.Headers),
+ Headers: maps.Clone(msg.Headers),
}}}, nil
- case MessageTypeWebSocketData:
- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")
- if err != nil {
- return nil, err
- }
- return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_WsData{WsData: &tunnelpb.WebSocketData{
- StreamId: msg.ID,
- Data: msg.Body,
- MessageType: messageType,
- }}}, nil
- case MessageTypeStreamData:
- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")
- if err != nil {
- return nil, err
- }
- if parity {
+ case MessageTypeWebSocketData, MessageTypeStreamData:
+ if msg.Type == MessageTypeStreamData && parity {
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamData{StreamData: &tunnelpb.StreamData{
RequestId: msg.ID,
Data: msg.Body,
@@ -73,7 +62,6 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Man
if parity {
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamEnd{StreamEnd: &tunnelpb.StreamEnd{RequestId: msg.ID}}}, nil
}
- return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)
case MessageTypeWebSocketClose:
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_WsClose{WsClose: &tunnelpb.WebSocketClose{StreamId: msg.ID}}}, nil
case MessageTypeRegisterResponse:
@@ -84,7 +72,7 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Man
Error: msg.Error,
SessionId: msg.SessionID,
SecurityMode: msg.SecurityMode,
- Capabilities: append([]string(nil), msg.Capabilities...),
+ Capabilities: slices.Clone(msg.Capabilities),
DrainPrevious: msg.DrainPrevious,
}}}, nil
case MessageTypeCommandRequest:
@@ -94,12 +82,12 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Man
Method: msg.Method,
Path: msg.Path,
Query: msg.Query,
- Headers: cloneHeaderMap(msg.Headers),
+ Headers: maps.Clone(msg.Headers),
Body: msg.Body,
TimeoutMillis: msg.TimeoutMillis,
SessionId: msg.SessionID,
AgentInstanceId: msg.AgentInstance,
- Metadata: cloneHeaderMap(msg.Metadata),
+ Metadata: maps.Clone(msg.Metadata),
}}}, nil
case MessageTypeStreamOpen:
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamOpen{StreamOpen: &tunnelpb.StreamOpen{
@@ -107,7 +95,7 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Man
CommandName: msg.Command,
Path: msg.Path,
Query: msg.Query,
- Headers: cloneHeaderMap(msg.Headers),
+ Headers: maps.Clone(msg.Headers),
SessionId: msg.SessionID,
}}}, nil
case MessageTypeStreamClose:
@@ -119,25 +107,21 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Man
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_CancelRequest{CancelRequest: &tunnelpb.CancelRequest{
CommandId: msg.ID,
}}}, nil
+ case MessageTypeCommandCredit:
+ return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_CommandCredit{CommandCredit: &tunnelpb.CommandCredit{
+ CommandId: msg.ID,
+ Bytes: msg.Credit,
+ }}}, nil
case MessageTypeFileChunk:
return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_FileChunk{FileChunk: &tunnelpb.FileChunk{
TransferId: msg.ID,
Data: msg.Body,
Sequence: msg.Sequence,
Eof: msg.EOF,
- Metadata: cloneHeaderMap(msg.Metadata),
+ Metadata: maps.Clone(msg.Metadata),
}}}, nil
- case MessageTypeResponse,
- MessageTypeHeartbeat,
- MessageTypeRegister,
- MessageTypeEvent,
- MessageTypeCommandAck,
- MessageTypeCommandOutput,
- MessageTypeCommandComplete:
- return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)
- default:
- return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)
}
+ return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)
}
func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage, error) {
@@ -153,7 +137,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Method: payload.HttpRequest.GetMethod(),
Path: payload.HttpRequest.GetPath(),
Query: payload.HttpRequest.GetQuery(),
- Headers: cloneHeaderMap(payload.HttpRequest.GetHeaders()),
+ Headers: maps.Clone(payload.HttpRequest.GetHeaders()),
Body: payload.HttpRequest.GetBody(),
}, nil
case *tunnelpb.ManagerMessage_HeartbeatPong:
@@ -164,15 +148,14 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Type: MessageTypeWebSocketStart,
Path: payload.WsStart.GetPath(),
Query: payload.WsStart.GetQuery(),
- Headers: cloneHeaderMap(payload.WsStart.GetHeaders()),
+ Headers: maps.Clone(payload.WsStart.GetHeaders()),
}, nil
case *tunnelpb.ManagerMessage_WsData:
return &TunnelMessage{
ID: payload.WsData.GetStreamId(),
Type: MessageTypeWebSocketData,
Body: payload.WsData.GetData(),
- // WSMessageType carries RFC 6455 opcodes (text=1, binary=2) on the
- // wire; a mixed-version fleet depends on these exact values.
+ // RFC 6455 opcodes (text=1, binary=2) on the wire; mixed-version fleets depend on them.
WSMessageType: int(payload.WsData.GetMessageType()),
}, nil
case *tunnelpb.ManagerMessage_WsClose:
@@ -186,7 +169,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Error: payload.RegisterResponse.GetError(),
SessionID: payload.RegisterResponse.GetSessionId(),
SecurityMode: payload.RegisterResponse.GetSecurityMode(),
- Capabilities: append([]string(nil), payload.RegisterResponse.GetCapabilities()...),
+ Capabilities: slices.Clone(payload.RegisterResponse.GetCapabilities()),
DrainPrevious: payload.RegisterResponse.GetDrainPrevious(),
}, nil
case *tunnelpb.ManagerMessage_CommandRequest:
@@ -197,12 +180,12 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Method: payload.CommandRequest.GetMethod(),
Path: payload.CommandRequest.GetPath(),
Query: payload.CommandRequest.GetQuery(),
- Headers: cloneHeaderMap(payload.CommandRequest.GetHeaders()),
+ Headers: maps.Clone(payload.CommandRequest.GetHeaders()),
Body: payload.CommandRequest.GetBody(),
TimeoutMillis: payload.CommandRequest.GetTimeoutMillis(),
SessionID: payload.CommandRequest.GetSessionId(),
AgentInstance: payload.CommandRequest.GetAgentInstanceId(),
- Metadata: cloneHeaderMap(payload.CommandRequest.GetMetadata()),
+ Metadata: maps.Clone(payload.CommandRequest.GetMetadata()),
}, nil
case *tunnelpb.ManagerMessage_StreamOpen:
return &TunnelMessage{
@@ -211,7 +194,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Command: payload.StreamOpen.GetCommandName(),
Path: payload.StreamOpen.GetPath(),
Query: payload.StreamOpen.GetQuery(),
- Headers: cloneHeaderMap(payload.StreamOpen.GetHeaders()),
+ Headers: maps.Clone(payload.StreamOpen.GetHeaders()),
SessionID: payload.StreamOpen.GetSessionId(),
}, nil
case *tunnelpb.ManagerMessage_StreamClose:
@@ -232,7 +215,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
Body: payload.FileChunk.GetData(),
Sequence: payload.FileChunk.GetSequence(),
EOF: payload.FileChunk.GetEof(),
- Metadata: cloneHeaderMap(payload.FileChunk.GetMetadata()),
+ Metadata: maps.Clone(payload.FileChunk.GetMetadata()),
}, nil
case *tunnelpb.ManagerMessage_StreamData:
return &TunnelMessage{
@@ -243,6 +226,8 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,
}, nil
case *tunnelpb.ManagerMessage_StreamEnd:
return &TunnelMessage{ID: payload.StreamEnd.GetRequestId(), Type: MessageTypeStreamEnd}, nil
+ case *tunnelpb.ManagerMessage_CommandCredit:
+ return &TunnelMessage{ID: payload.CommandCredit.GetCommandId(), Type: MessageTypeCommandCredit, Credit: payload.CommandCredit.GetBytes()}, nil
default:
return nil, fmt.Errorf("manager payload %T: %w", payload, errUnknownTunnelPayload)
}
@@ -252,26 +237,26 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
if msg == nil {
return nil, errors.New("message is nil")
}
+ status, err := intToInt32(msg.Status, "status")
+ if err != nil {
+ return nil, err
+ }
+ messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")
+ if err != nil {
+ return nil, err
+ }
switch msg.Type {
case MessageTypeResponse:
- status, err := intToInt32(msg.Status, "status")
- if err != nil {
- return nil, err
- }
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_HttpResponse{HttpResponse: &tunnelpb.HttpResponse{
RequestId: msg.ID,
Status: status,
- Headers: cloneHeaderMap(msg.Headers),
+ Headers: maps.Clone(msg.Headers),
Body: msg.Body,
}}}, nil
case MessageTypeHeartbeat:
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_HeartbeatPing{HeartbeatPing: &tunnelpb.HeartbeatPing{Id: msg.ID}}}, nil
case MessageTypeWebSocketData:
- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")
- if err != nil {
- return nil, err
- }
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_WsData{WsData: &tunnelpb.WebSocketData{
StreamId: msg.ID,
Data: msg.Body,
@@ -280,10 +265,6 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
case MessageTypeWebSocketClose:
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_WsClose{WsClose: &tunnelpb.WebSocketClose{StreamId: msg.ID}}}, nil
case MessageTypeStreamData:
- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")
- if err != nil {
- return nil, err
- }
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_StreamData{StreamData: &tunnelpb.StreamData{
RequestId: msg.ID,
Data: msg.Body,
@@ -295,7 +276,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_Register{Register: &tunnelpb.RegisterRequest{
AgentToken: msg.AgentToken,
AgentInstanceId: msg.AgentInstance,
- Capabilities: append([]string(nil), msg.Capabilities...),
+ Capabilities: slices.Clone(msg.Capabilities),
ResumeSessionId: msg.ResumeSession,
}}}, nil
case MessageTypeEvent:
@@ -313,7 +294,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
ResourceName: msg.Event.ResourceName,
UserId: msg.Event.UserID,
Username: msg.Event.Username,
- MetadataJson: append([]byte(nil), msg.Event.MetadataJSON...),
+ MetadataJson: bytes.Clone(msg.Event.MetadataJSON),
}}}, nil
case MessageTypeCommandAck:
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_CommandAck{CommandAck: &tunnelpb.CommandAck{
@@ -324,16 +305,14 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
CommandId: msg.ID,
Data: msg.Body,
Sequence: msg.Sequence,
+ Status: status,
+ Headers: maps.Clone(msg.Headers),
}}}, nil
case MessageTypeCommandComplete:
- status, err := intToInt32(msg.Status, "status")
- if err != nil {
- return nil, err
- }
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_CommandComplete{CommandComplete: &tunnelpb.CommandComplete{
CommandId: msg.ID,
Status: status,
- Headers: cloneHeaderMap(msg.Headers),
+ Headers: maps.Clone(msg.Headers),
Body: msg.Body,
Error: msg.Error,
Streaming: msg.Streaming,
@@ -344,7 +323,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
Data: msg.Body,
Sequence: msg.Sequence,
Eof: msg.EOF,
- Metadata: cloneHeaderMap(msg.Metadata),
+ Metadata: maps.Clone(msg.Metadata),
}}}, nil
case MessageTypeStreamClose:
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_StreamClose{StreamClose: &tunnelpb.StreamClose{
@@ -357,16 +336,8 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erro
return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_CancelRequest{CancelRequest: &tunnelpb.CancelRequest{
CommandId: msg.ID,
}}}, nil
- case MessageTypeRequest,
- MessageTypeHeartbeatAck,
- MessageTypeWebSocketStart,
- MessageTypeRegisterResponse,
- MessageTypeCommandRequest,
- MessageTypeStreamOpen:
- return nil, fmt.Errorf("unsupported agent message type: %s", msg.Type)
- default:
- return nil, fmt.Errorf("unsupported agent message type: %s", msg.Type)
}
+ return nil, fmt.Errorf("unsupported agent message type: %s", msg.Type)
}
func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, error) {
@@ -380,7 +351,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
ID: payload.HttpResponse.GetRequestId(),
Type: MessageTypeResponse,
Status: int(payload.HttpResponse.GetStatus()),
- Headers: cloneHeaderMap(payload.HttpResponse.GetHeaders()),
+ Headers: maps.Clone(payload.HttpResponse.GetHeaders()),
Body: payload.HttpResponse.GetBody(),
}, nil
case *tunnelpb.AgentMessage_HeartbeatPing:
@@ -408,7 +379,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
Type: MessageTypeRegister,
AgentToken: payload.Register.GetAgentToken(),
AgentInstance: payload.Register.GetAgentInstanceId(),
- Capabilities: append([]string(nil), payload.Register.GetCapabilities()...),
+ Capabilities: slices.Clone(payload.Register.GetCapabilities()),
ResumeSession: payload.Register.GetResumeSessionId(),
}, nil
case *tunnelpb.AgentMessage_Event:
@@ -425,7 +396,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
ResourceName: payload.Event.GetResourceName(),
UserID: payload.Event.GetUserId(),
Username: payload.Event.GetUsername(),
- MetadataJSON: append([]byte(nil), payload.Event.GetMetadataJson()...),
+ MetadataJSON: bytes.Clone(payload.Event.GetMetadataJson()),
},
}, nil
case *tunnelpb.AgentMessage_CommandAck:
@@ -436,13 +407,15 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
Type: MessageTypeCommandOutput,
Body: payload.CommandOutput.GetData(),
Sequence: payload.CommandOutput.GetSequence(),
+ Status: int(payload.CommandOutput.GetStatus()),
+ Headers: maps.Clone(payload.CommandOutput.GetHeaders()),
}, nil
case *tunnelpb.AgentMessage_CommandComplete:
return &TunnelMessage{
ID: payload.CommandComplete.GetCommandId(),
Type: MessageTypeCommandComplete,
Status: int(payload.CommandComplete.GetStatus()),
- Headers: cloneHeaderMap(payload.CommandComplete.GetHeaders()),
+ Headers: maps.Clone(payload.CommandComplete.GetHeaders()),
Body: payload.CommandComplete.GetBody(),
Error: payload.CommandComplete.GetError(),
Streaming: payload.CommandComplete.GetStreaming(),
@@ -454,7 +427,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
Body: payload.FileChunk.GetData(),
Sequence: payload.FileChunk.GetSequence(),
EOF: payload.FileChunk.GetEof(),
- Metadata: cloneHeaderMap(payload.FileChunk.GetMetadata()),
+ Metadata: maps.Clone(payload.FileChunk.GetMetadata()),
}, nil
case *tunnelpb.AgentMessage_StreamClose:
return &TunnelMessage{
@@ -472,15 +445,6 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro
}
}
-func cloneHeaderMap(in map[string]string) map[string]string {
- if len(in) == 0 {
- return nil
- }
- out := make(map[string]string, len(in))
- maps.Copy(out, in)
- return out
-}
-
func intToInt32(value int, field string) (int32, error) {
if value < math.MinInt32 || value > math.MaxInt32 {
return 0, fmt.Errorf("%s value %d is out of int32 range", field, value)
diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.go
index cccbb2298..8a177c970 100644
--- a/backend/pkg/libarcane/edge/types.go
+++ b/backend/pkg/libarcane/edge/types.go
@@ -12,6 +12,7 @@ import (
"google.golang.org/grpc"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
)
// activeWSStream tracks an active WebSocket stream on the agent side.
@@ -54,6 +55,7 @@ type TunnelClient struct {
requestTimeout time.Duration
activeStreams sync.Map // map[string]*activeWSStream
requestTransfers sync.Map // map[string]*commandRequestTransfer
+ commandRecorders sync.Map // map[string]*commandResponseRecorder awaiting credit
transportPreferenceMu sync.RWMutex
preferWebSocketUntil time.Time
grpcFailureStreak int
@@ -63,6 +65,8 @@ type TunnelClient struct {
type clientRegistrationInternal struct {
sessionID string
+ // commandCredit is set when the manager grants command output flow-control credit.
+ commandCredit bool
}
// connBox wraps the active TunnelConnection so it can be swapped atomically on
@@ -93,6 +97,13 @@ type commandResponseRecorder struct {
buffer bytes.Buffer
statusCode int
sequence int64
+ // flushErr keeps the first send failure from Flush, which cannot return it.
+ flushErr error
+ // With manager flow control, sends wait while inFlight would exceed window; credits signal room.
+ ctx context.Context
+ window int64
+ inFlight atomic.Int64
+ credited chan struct{}
mu sync.Mutex
wroteHeader bool
streaming bool
@@ -124,6 +135,8 @@ type CommandRequest struct {
Headers map[string]string
Body []byte
TimeoutMillis int64
+ // Output, when set, receives the response as it streams in; CommandResult.Body is then nil.
+ Output http.ResponseWriter
}
type CommandResult struct {
@@ -191,10 +204,14 @@ type PollRuntimeRegistry struct {
}
type grpcResponseState struct {
+ out http.ResponseWriter
+ outErr error
+ credit func(consumed int)
status int
respHeaders map[string]string
respBody bytes.Buffer
gotResponse bool
+ committed bool
}
// AgentTunnel represents an active tunnel connection from an edge agent
@@ -214,6 +231,12 @@ type AgentTunnel struct {
mu sync.RWMutex
done chan struct{}
closeOnce sync.Once
+
+ // Command credits queue here for a per-tunnel sender, so readers never block on the connection.
+ creditOnce sync.Once
+ creditMu sync.Mutex
+ credits map[string]int64
+ creditSignal chan struct{}
}
// TunnelRegistry manages active edge agent tunnel connections
@@ -226,19 +249,30 @@ type internalTunnelRequestContextKey struct{}
// TunnelServer handles incoming edge agent connections on the manager side.
type TunnelServer struct {
- registry *TunnelRegistry
- resolver EnvironmentResolver
- nameResolver EnvironmentNameResolver
- statusCallback StatusUpdateCallback
- eventCallback EventCallback
- enrollmentCallback EnrollmentCallback
- cleanupDone chan struct{}
- cleanupDoneOnce sync.Once
- cfg *Config
- statusMu sync.Mutex
+ // Config holds the edge runtime settings; set it before serving.
+ Config *Config
+ // NameResolver, when set, names environments in generated mTLS assets.
+ NameResolver EnvironmentNameResolver
+ // EventCallback, when set, receives agent events.
+ EventCallback EventCallback
+ // EnrollmentCallback, when set, runs after a successful edge mTLS enrollment.
+ EnrollmentCallback EnrollmentCallback
+
+ registry *TunnelRegistry
+ resolver EnvironmentResolver
+ statusCallback StatusUpdateCallback
+ cleanupDone chan struct{}
+ cleanupDoneOnce sync.Once
+ statusMu sync.Mutex
}
-type resolvedEnvironmentIDKey struct{}
+type streamIdentityKey struct{}
+
+// streamIdentity is what the gRPC auth interceptor verified for a tunnel stream.
+type streamIdentity struct {
+ environmentID string
+ securityMode string
+}
type contextualServerStream struct {
grpc.ServerStream
@@ -267,11 +301,6 @@ type enrollMTLSResponse struct {
Files []GeneratedMTLSFile `json:"files"`
}
-type edgeMTLSLockInfo struct {
- pid int
- createdAt time.Time
-}
-
// Config contains the public edge-tunnel runtime settings needed by pkg/libarcane/edge.
type Config struct {
EdgeAgent bool
@@ -326,6 +355,7 @@ type TunnelMessage struct {
Status int `json:"status,omitempty"` // HTTP status for responses
TimeoutMillis int64 `json:"timeout_millis,omitempty"` // Command timeout
Sequence int64 `json:"sequence,omitempty"` // Chunk sequence number
+ Credit int64 `json:"credit,omitempty"` // Command output bytes the manager consumed
Accepted bool `json:"accepted,omitempty"` // Registration accepted
DrainPrevious bool `json:"drain_previous,omitempty"` // Replace previous session
Streaming bool `json:"streaming,omitempty"` // Response used chunked output
@@ -350,6 +380,13 @@ type TunnelEvent struct {
type PendingRequest struct {
ResponseCh chan *TunnelMessage
failureCh chan error
+
+ // mu guards the ordered backlog that absorbs bursts while ResponseCh is full.
+ mu sync.Mutex
+ backlog []*TunnelMessage
+ backlogBytes int
+ draining bool
+ failed bool
}
// TunnelConn wraps a WebSocket connection with send/receive helpers.
@@ -365,10 +402,9 @@ type GRPCManagerTunnelConn struct {
cancel context.CancelFunc
mu sync.Mutex
closed atomic.Bool
- // parityEncoding is set once the agent advertises the proto-parity-v1
- // capability, enabling native stream_data/stream_end encodings instead of
- // the legacy ws_data re-encode.
- parityEncoding atomic.Bool
+ // parity is set at registration when the agent advertises proto-parity-v1,
+ // enabling native stream_data/stream_end instead of the legacy ws_data re-encode.
+ parity bool
}
// GRPCAgentTunnelConn wraps the agent-side gRPC tunnel stream.
@@ -380,6 +416,13 @@ type GRPCAgentTunnelConn struct {
}
type cancelableGRPCManagerStream struct {
- stream grpcManagerStream
- ctx context.Context
+ stream grpcManagerStream
+ ctx context.Context
+ recvCh chan grpcRecvResult
+ recvOnce sync.Once
+}
+
+type grpcRecvResult struct {
+ msg *tunnelpb.AgentMessage
+ err error
}
diff --git a/backend/pkg/libarcane/startup/startup.go b/backend/pkg/libarcane/startup/startup.go
index 566485b33..4acd8634f 100644
--- a/backend/pkg/libarcane/startup/startup.go
+++ b/backend/pkg/libarcane/startup/startup.go
@@ -3,6 +3,10 @@ package startup
import (
"context"
"log/slog"
+ "os"
+ "runtime/debug"
+
+ "go.getarcane.app/sys/cgroup"
"github.com/getarcaneapp/arcane/backend/v2/buildables"
)
@@ -16,6 +20,22 @@ type RuntimeConfig struct {
AdminStaticAPIKey string
}
+// ApplyMemoryLimit sets the Go soft memory limit to 90% of the cgroup memory
+// limit so the GC tightens before the container is OOM-killed. GOMEMLIMIT wins.
+func ApplyMemoryLimit(ctx context.Context) {
+ if os.Getenv("GOMEMLIMIT") != "" {
+ return
+ }
+ limits, err := cgroup.DetectLimits()
+ // cgroup v1 reports "no limit" as a page-aligned value near MaxInt64.
+ if err != nil || limits.MemoryLimit <= 0 || limits.MemoryLimit >= 1<<62 {
+ return
+ }
+ limit := limits.MemoryLimit / 10 * 9
+ debug.SetMemoryLimit(limit)
+ slog.InfoContext(ctx, "Set Go memory limit from cgroup", "cgroupLimitBytes", limits.MemoryLimit, "goMemoryLimitBytes", limit)
+}
+
func LoadAgentToken(ctx context.Context, cfg *RuntimeConfig, getSettingFunc func(context.Context, string, string) string) {
if cfg.AgentMode && cfg.AgentToken == "" {
if tok := getSettingFunc(ctx, "agentToken", ""); tok != "" {
diff --git a/backend/pkg/libarcane/ws/proxy.go b/backend/pkg/libarcane/ws/proxy.go
index 04a84260a..bc14da766 100644
--- a/backend/pkg/libarcane/ws/proxy.go
+++ b/backend/pkg/libarcane/ws/proxy.go
@@ -9,8 +9,14 @@ import (
"time"
"github.com/coder/websocket"
+ httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
)
+// dialClient is shared by every upstream dial instead of building a transport per connection.
+var dialClient = httpx.NewHTTPClient(httpxtypes.ClientOptions{TLSHandshakeTimeout: 10 * time.Second})
+
// ProxyHTTP upgrades the incoming client connection and bridges it to remoteWS.
//
// checkOrigin must be the same Origin validator the local WebSocket endpoints
@@ -35,7 +41,7 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header h
dialCtx, dialCancel := context.WithTimeout(r.Context(), 45*time.Second)
remoteConn, resp, err := websocket.Dial(dialCtx, remoteWS, &websocket.DialOptions{
HTTPHeader: header,
- HTTPClient: &http.Client{Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}},
+ HTTPClient: dialClient,
})
dialCancel()
// Ensure the response body is drained & closed to avoid leaking resources.
@@ -71,7 +77,7 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header h
for {
mt, msg, readErr := clientConn.Read(pumpCtx)
if readErr != nil {
- relayCloseInternal(r.Context(), remoteConn, readErr)
+ closeRelay(r.Context(), remoteConn, readErr)
return
}
if writeErr := remoteConn.Write(pumpCtx, mt, msg); writeErr != nil {
@@ -86,7 +92,7 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header h
for {
mt, msg, readErr := remoteConn.Read(pumpCtx)
if readErr != nil {
- relayCloseInternal(r.Context(), clientConn, readErr)
+ closeRelay(r.Context(), clientConn, readErr)
return
}
if writeErr := clientConn.Write(pumpCtx, mt, msg); writeErr != nil {
@@ -99,9 +105,9 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header h
return nil
}
-// relayCloseInternal forwards a peer's close status to the other side of the
+// closeRelay forwards a peer's close status to the other side of the
// bridge so the terminating reason survives the proxy hop.
-func relayCloseInternal(ctx context.Context, conn *websocket.Conn, err error) {
+func closeRelay(ctx context.Context, conn *websocket.Conn, err error) {
var ce websocket.CloseError
if !errors.As(err, &ce) {
return
diff --git a/backend/pkg/projects/cache.go b/backend/pkg/projects/cache.go
index 3e417a201..812a7cf0f 100644
--- a/backend/pkg/projects/cache.go
+++ b/backend/pkg/projects/cache.go
@@ -150,18 +150,35 @@ func LoadCachedComposeProject(ctx context.Context,
if cache == nil {
return model, nil
}
+ composeFiles, envFiles, ok := ComposeCacheDependencies(ctx, model, dependencies, composePath, projectsDirectory, autoInject)
+ if !ok {
+ // Metadata discovery failure disables caching; the executable model is valid.
+ return model, nil
+ }
+ if setErr := cache.Set(projectID, fingerprint, projectPath, projectsDirectory, composePath, composeFiles, envFiles, model); setErr != nil {
+ return nil, setErr
+ }
+ return model, nil
+}
+
+// ComposeCacheDependencies returns the compose and env files a model loaded
+// from composePath depends on. ok is false when they cannot be discovered.
+func ComposeCacheDependencies(
+ ctx context.Context,
+ model *composetypes.Project,
+ dependencies projecttypes.ComposeDependencies,
+ composePath, projectsDirectory string,
+ autoInject bool,
+) ([]string, []string, bool) {
+ envFiles := dependencies.EnvFiles
if envOpts, parseErr := ParseComposeEnvOptions(model.WorkingDir, EnvMap(model.Environment)); parseErr == nil {
- dependencies.EnvFiles = append(dependencies.EnvFiles, envOpts.EnvFiles...)
+ envFiles = append(envFiles, envOpts.EnvFiles...)
}
// Metadata discovery already follows recursive includes and their interpolation env files.
meta, err := ParseArcaneComposeMetadata(ctx, composePath, projectsDirectory, autoInject)
if err != nil {
- return model, nil //nolint:nilerr // Metadata discovery failure disables caching; the executable model is valid.
+ return nil, nil, false
}
composeFiles := append(append([]string{}, model.ComposeFiles...), meta.ComposeFiles...)
- dependencies.EnvFiles = append(dependencies.EnvFiles, meta.EnvFiles...)
- if setErr := cache.Set(projectID, fingerprint, projectPath, projectsDirectory, composePath, composeFiles, dependencies.EnvFiles, model); setErr != nil {
- return nil, setErr
- }
- return model, nil
+ return composeFiles, append(envFiles, meta.EnvFiles...), true
}
diff --git a/backend/pkg/utils/httpx/request.go b/backend/pkg/utils/httpx/request.go
index 480686cd2..e0bceab76 100644
--- a/backend/pkg/utils/httpx/request.go
+++ b/backend/pkg/utils/httpx/request.go
@@ -147,6 +147,7 @@ func NewHTTPClient(options httpxtypes.ClientOptions) *http.Client {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
MaxIdleConns: 100,
+ MaxIdleConnsPerHost: 32,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: options.TLSHandshakeTimeout,
ExpectContinueTimeout: 1 * time.Second,
diff --git a/backend/proto/tunnel/v1/tunnel.pb.go b/backend/proto/tunnel/v1/tunnel.pb.go
index 0e2a2d958..19c1b5a7e 100644
--- a/backend/proto/tunnel/v1/tunnel.pb.go
+++ b/backend/proto/tunnel/v1/tunnel.pb.go
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
-// protoc-gen-go v1.36.11
+// protoc-gen-go v1.36.12
// protoc (unknown)
// source: tunnel/v1/tunnel.proto
@@ -317,6 +317,7 @@ type ManagerMessage struct {
// *ManagerMessage_FileChunk
// *ManagerMessage_StreamData
// *ManagerMessage_StreamEnd
+ // *ManagerMessage_CommandCredit
Payload isManagerMessage_Payload `protobuf_oneof:"payload"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -476,6 +477,15 @@ func (x *ManagerMessage) GetStreamEnd() *StreamEnd {
return nil
}
+func (x *ManagerMessage) GetCommandCredit() *CommandCredit {
+ if x != nil {
+ if x, ok := x.Payload.(*ManagerMessage_CommandCredit); ok {
+ return x.CommandCredit
+ }
+ }
+ return nil
+}
+
type isManagerMessage_Payload interface {
isManagerMessage_Payload()
}
@@ -534,6 +544,11 @@ type ManagerMessage_StreamEnd struct {
StreamEnd *StreamEnd `protobuf:"bytes,13,opt,name=stream_end,json=streamEnd,proto3,oneof"`
}
+type ManagerMessage_CommandCredit struct {
+ // Only sent to agents that advertised command-credit-v1.
+ CommandCredit *CommandCredit `protobuf:"bytes,14,opt,name=command_credit,json=commandCredit,proto3,oneof"`
+}
+
func (*ManagerMessage_HttpRequest) isManagerMessage_Payload() {}
func (*ManagerMessage_HeartbeatPong) isManagerMessage_Payload() {}
@@ -560,6 +575,8 @@ func (*ManagerMessage_StreamData) isManagerMessage_Payload() {}
func (*ManagerMessage_StreamEnd) isManagerMessage_Payload() {}
+func (*ManagerMessage_CommandCredit) isManagerMessage_Payload() {}
+
type RegisterRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
AgentToken string `protobuf:"bytes,1,opt,name=agent_token,json=agentToken,proto3" json:"agent_token,omitempty"`
@@ -897,10 +914,14 @@ func (x *CommandAck) GetCommandId() string {
}
type CommandOutput struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
- Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
- Sequence int64 `protobuf:"varint,3,opt,name=sequence,proto3" json:"sequence,omitempty"`
+ state protoimpl.MessageState `protogen:"open.v1"`
+ CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
+ Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
+ Sequence int64 `protobuf:"varint,3,opt,name=sequence,proto3" json:"sequence,omitempty"`
+ // Set on the first chunk so the manager can stream the response before
+ // command_complete arrives. Older managers ignore these fields.
+ Status int32 `protobuf:"varint,4,opt,name=status,proto3" json:"status,omitempty"`
+ Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -956,6 +977,20 @@ func (x *CommandOutput) GetSequence() int64 {
return 0
}
+func (x *CommandOutput) GetStatus() int32 {
+ if x != nil {
+ return x.Status
+ }
+ return 0
+}
+
+func (x *CommandOutput) GetHeaders() map[string]string {
+ if x != nil {
+ return x.Headers
+ }
+ return nil
+}
+
type CommandComplete struct {
state protoimpl.MessageState `protogen:"open.v1"`
CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
@@ -1296,6 +1331,59 @@ func (x *CancelRequest) GetCommandId() string {
return ""
}
+// CommandCredit returns flow-control credit for command output the manager has consumed.
+type CommandCredit struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`
+ Bytes int64 `protobuf:"varint,2,opt,name=bytes,proto3" json:"bytes,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *CommandCredit) Reset() {
+ *x = CommandCredit{}
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *CommandCredit) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CommandCredit) ProtoMessage() {}
+
+func (x *CommandCredit) ProtoReflect() protoreflect.Message {
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use CommandCredit.ProtoReflect.Descriptor instead.
+func (*CommandCredit) Descriptor() ([]byte, []int) {
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{12}
+}
+
+func (x *CommandCredit) GetCommandId() string {
+ if x != nil {
+ return x.CommandId
+ }
+ return ""
+}
+
+func (x *CommandCredit) GetBytes() int64 {
+ if x != nil {
+ return x.Bytes
+ }
+ return 0
+}
+
type HttpRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
@@ -1310,7 +1398,7 @@ type HttpRequest struct {
func (x *HttpRequest) Reset() {
*x = HttpRequest{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1322,7 +1410,7 @@ func (x *HttpRequest) String() string {
func (*HttpRequest) ProtoMessage() {}
func (x *HttpRequest) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1335,7 +1423,7 @@ func (x *HttpRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use HttpRequest.ProtoReflect.Descriptor instead.
func (*HttpRequest) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{12}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{13}
}
func (x *HttpRequest) GetRequestId() string {
@@ -1392,7 +1480,7 @@ type HttpResponse struct {
func (x *HttpResponse) Reset() {
*x = HttpResponse{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1404,7 +1492,7 @@ func (x *HttpResponse) String() string {
func (*HttpResponse) ProtoMessage() {}
func (x *HttpResponse) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1417,7 +1505,7 @@ func (x *HttpResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use HttpResponse.ProtoReflect.Descriptor instead.
func (*HttpResponse) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{13}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{14}
}
func (x *HttpResponse) GetRequestId() string {
@@ -1460,7 +1548,7 @@ type StreamData struct {
func (x *StreamData) Reset() {
*x = StreamData{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1472,7 +1560,7 @@ func (x *StreamData) String() string {
func (*StreamData) ProtoMessage() {}
func (x *StreamData) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1485,7 +1573,7 @@ func (x *StreamData) ProtoReflect() protoreflect.Message {
// Deprecated: Use StreamData.ProtoReflect.Descriptor instead.
func (*StreamData) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{14}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{15}
}
func (x *StreamData) GetRequestId() string {
@@ -1518,7 +1606,7 @@ type StreamEnd struct {
func (x *StreamEnd) Reset() {
*x = StreamEnd{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1530,7 +1618,7 @@ func (x *StreamEnd) String() string {
func (*StreamEnd) ProtoMessage() {}
func (x *StreamEnd) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1543,7 +1631,7 @@ func (x *StreamEnd) ProtoReflect() protoreflect.Message {
// Deprecated: Use StreamEnd.ProtoReflect.Descriptor instead.
func (*StreamEnd) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{15}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{16}
}
func (x *StreamEnd) GetRequestId() string {
@@ -1562,7 +1650,7 @@ type HeartbeatPing struct {
func (x *HeartbeatPing) Reset() {
*x = HeartbeatPing{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1574,7 +1662,7 @@ func (x *HeartbeatPing) String() string {
func (*HeartbeatPing) ProtoMessage() {}
func (x *HeartbeatPing) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1587,7 +1675,7 @@ func (x *HeartbeatPing) ProtoReflect() protoreflect.Message {
// Deprecated: Use HeartbeatPing.ProtoReflect.Descriptor instead.
func (*HeartbeatPing) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{16}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{17}
}
func (x *HeartbeatPing) GetId() string {
@@ -1606,7 +1694,7 @@ type HeartbeatPong struct {
func (x *HeartbeatPong) Reset() {
*x = HeartbeatPong{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1618,7 +1706,7 @@ func (x *HeartbeatPong) String() string {
func (*HeartbeatPong) ProtoMessage() {}
func (x *HeartbeatPong) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1631,7 +1719,7 @@ func (x *HeartbeatPong) ProtoReflect() protoreflect.Message {
// Deprecated: Use HeartbeatPong.ProtoReflect.Descriptor instead.
func (*HeartbeatPong) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{17}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{18}
}
func (x *HeartbeatPong) GetId() string {
@@ -1653,7 +1741,7 @@ type WebSocketStart struct {
func (x *WebSocketStart) Reset() {
*x = WebSocketStart{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1665,7 +1753,7 @@ func (x *WebSocketStart) String() string {
func (*WebSocketStart) ProtoMessage() {}
func (x *WebSocketStart) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1678,7 +1766,7 @@ func (x *WebSocketStart) ProtoReflect() protoreflect.Message {
// Deprecated: Use WebSocketStart.ProtoReflect.Descriptor instead.
func (*WebSocketStart) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{18}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{19}
}
func (x *WebSocketStart) GetStreamId() string {
@@ -1720,7 +1808,7 @@ type WebSocketData struct {
func (x *WebSocketData) Reset() {
*x = WebSocketData{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1732,7 +1820,7 @@ func (x *WebSocketData) String() string {
func (*WebSocketData) ProtoMessage() {}
func (x *WebSocketData) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1745,7 +1833,7 @@ func (x *WebSocketData) ProtoReflect() protoreflect.Message {
// Deprecated: Use WebSocketData.ProtoReflect.Descriptor instead.
func (*WebSocketData) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{19}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{20}
}
func (x *WebSocketData) GetStreamId() string {
@@ -1778,7 +1866,7 @@ type WebSocketClose struct {
func (x *WebSocketClose) Reset() {
*x = WebSocketClose{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1790,7 +1878,7 @@ func (x *WebSocketClose) String() string {
func (*WebSocketClose) ProtoMessage() {}
func (x *WebSocketClose) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1803,7 +1891,7 @@ func (x *WebSocketClose) ProtoReflect() protoreflect.Message {
// Deprecated: Use WebSocketClose.ProtoReflect.Descriptor instead.
func (*WebSocketClose) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{20}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{21}
}
func (x *WebSocketClose) GetStreamId() string {
@@ -1833,7 +1921,7 @@ type EventLog struct {
func (x *EventLog) Reset() {
*x = EventLog{}
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[22]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1845,7 +1933,7 @@ func (x *EventLog) String() string {
func (*EventLog) ProtoMessage() {}
func (x *EventLog) ProtoReflect() protoreflect.Message {
- mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]
+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[22]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1858,7 +1946,7 @@ func (x *EventLog) ProtoReflect() protoreflect.Message {
// Deprecated: Use EventLog.ProtoReflect.Descriptor instead.
func (*EventLog) Descriptor() ([]byte, []int) {
- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{21}
+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{22}
}
func (x *EventLog) GetType() string {
@@ -1963,7 +2051,7 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"file_chunk\x18\f \x01(\v2\x14.tunnel.v1.FileChunkH\x00R\tfileChunk\x12;\n" +
"\fstream_close\x18\r \x01(\v2\x16.tunnel.v1.StreamCloseH\x00R\vstreamClose\x12A\n" +
"\x0ecancel_request\x18\x0e \x01(\v2\x18.tunnel.v1.CancelRequestH\x00R\rcancelRequestB\t\n" +
- "\apayload\"\xb4\x06\n" +
+ "\apayload\"\xf7\x06\n" +
"\x0eManagerMessage\x12;\n" +
"\fhttp_request\x18\x01 \x01(\v2\x16.tunnel.v1.HttpRequestH\x00R\vhttpRequest\x12A\n" +
"\x0eheartbeat_pong\x18\x02 \x01(\v2\x18.tunnel.v1.HeartbeatPongH\x00R\rheartbeatPong\x126\n" +
@@ -1982,7 +2070,8 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\vstream_data\x18\f \x01(\v2\x15.tunnel.v1.StreamDataH\x00R\n" +
"streamData\x125\n" +
"\n" +
- "stream_end\x18\r \x01(\v2\x14.tunnel.v1.StreamEndH\x00R\tstreamEndB\t\n" +
+ "stream_end\x18\r \x01(\v2\x14.tunnel.v1.StreamEndH\x00R\tstreamEnd\x12A\n" +
+ "\x0ecommand_credit\x18\x0e \x01(\v2\x18.tunnel.v1.CommandCreditH\x00R\rcommandCreditB\t\n" +
"\apayload\"\xae\x01\n" +
"\x0fRegisterRequest\x12\x1f\n" +
"\vagent_token\x18\x01 \x01(\tR\n" +
@@ -2024,12 +2113,17 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\n" +
"CommandAck\x12\x1d\n" +
"\n" +
- "command_id\x18\x01 \x01(\tR\tcommandId\"^\n" +
+ "command_id\x18\x01 \x01(\tR\tcommandId\"\xf3\x01\n" +
"\rCommandOutput\x12\x1d\n" +
"\n" +
"command_id\x18\x01 \x01(\tR\tcommandId\x12\x12\n" +
"\x04data\x18\x02 \x01(\fR\x04data\x12\x1a\n" +
- "\bsequence\x18\x03 \x01(\x03R\bsequence\"\x8f\x02\n" +
+ "\bsequence\x18\x03 \x01(\x03R\bsequence\x12\x16\n" +
+ "\x06status\x18\x04 \x01(\x05R\x06status\x12?\n" +
+ "\aheaders\x18\x05 \x03(\v2%.tunnel.v1.CommandOutput.HeadersEntryR\aheaders\x1a:\n" +
+ "\fHeadersEntry\x12\x10\n" +
+ "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8f\x02\n" +
"\x0fCommandComplete\x12\x1d\n" +
"\n" +
"command_id\x18\x01 \x01(\tR\tcommandId\x12\x16\n" +
@@ -2068,7 +2162,11 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +
"\x05error\x18\x02 \x01(\tR\x05error\".\n" +
"\rCancelRequest\x12\x1d\n" +
"\n" +
- "command_id\x18\x01 \x01(\tR\tcommandId\"\xfd\x01\n" +
+ "command_id\x18\x01 \x01(\tR\tcommandId\"D\n" +
+ "\rCommandCredit\x12\x1d\n" +
+ "\n" +
+ "command_id\x18\x01 \x01(\tR\tcommandId\x12\x14\n" +
+ "\x05bytes\x18\x02 \x01(\x03R\x05bytes\"\xfd\x01\n" +
"\vHttpRequest\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x16\n" +
@@ -2145,7 +2243,7 @@ func file_tunnel_v1_tunnel_proto_rawDescGZIP() []byte {
return file_tunnel_v1_tunnel_proto_rawDescData
}
-var file_tunnel_v1_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 30)
+var file_tunnel_v1_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 32)
var file_tunnel_v1_tunnel_proto_goTypes = []any{
(*AgentMessage)(nil), // 0: tunnel.v1.AgentMessage
(*ManagerMessage)(nil), // 1: tunnel.v1.ManagerMessage
@@ -2159,68 +2257,72 @@ var file_tunnel_v1_tunnel_proto_goTypes = []any{
(*StreamOpen)(nil), // 9: tunnel.v1.StreamOpen
(*StreamClose)(nil), // 10: tunnel.v1.StreamClose
(*CancelRequest)(nil), // 11: tunnel.v1.CancelRequest
- (*HttpRequest)(nil), // 12: tunnel.v1.HttpRequest
- (*HttpResponse)(nil), // 13: tunnel.v1.HttpResponse
- (*StreamData)(nil), // 14: tunnel.v1.StreamData
- (*StreamEnd)(nil), // 15: tunnel.v1.StreamEnd
- (*HeartbeatPing)(nil), // 16: tunnel.v1.HeartbeatPing
- (*HeartbeatPong)(nil), // 17: tunnel.v1.HeartbeatPong
- (*WebSocketStart)(nil), // 18: tunnel.v1.WebSocketStart
- (*WebSocketData)(nil), // 19: tunnel.v1.WebSocketData
- (*WebSocketClose)(nil), // 20: tunnel.v1.WebSocketClose
- (*EventLog)(nil), // 21: tunnel.v1.EventLog
- nil, // 22: tunnel.v1.CommandRequest.HeadersEntry
- nil, // 23: tunnel.v1.CommandRequest.MetadataEntry
- nil, // 24: tunnel.v1.CommandComplete.HeadersEntry
- nil, // 25: tunnel.v1.FileChunk.MetadataEntry
- nil, // 26: tunnel.v1.StreamOpen.HeadersEntry
- nil, // 27: tunnel.v1.HttpRequest.HeadersEntry
- nil, // 28: tunnel.v1.HttpResponse.HeadersEntry
- nil, // 29: tunnel.v1.WebSocketStart.HeadersEntry
+ (*CommandCredit)(nil), // 12: tunnel.v1.CommandCredit
+ (*HttpRequest)(nil), // 13: tunnel.v1.HttpRequest
+ (*HttpResponse)(nil), // 14: tunnel.v1.HttpResponse
+ (*StreamData)(nil), // 15: tunnel.v1.StreamData
+ (*StreamEnd)(nil), // 16: tunnel.v1.StreamEnd
+ (*HeartbeatPing)(nil), // 17: tunnel.v1.HeartbeatPing
+ (*HeartbeatPong)(nil), // 18: tunnel.v1.HeartbeatPong
+ (*WebSocketStart)(nil), // 19: tunnel.v1.WebSocketStart
+ (*WebSocketData)(nil), // 20: tunnel.v1.WebSocketData
+ (*WebSocketClose)(nil), // 21: tunnel.v1.WebSocketClose
+ (*EventLog)(nil), // 22: tunnel.v1.EventLog
+ nil, // 23: tunnel.v1.CommandRequest.HeadersEntry
+ nil, // 24: tunnel.v1.CommandRequest.MetadataEntry
+ nil, // 25: tunnel.v1.CommandOutput.HeadersEntry
+ nil, // 26: tunnel.v1.CommandComplete.HeadersEntry
+ nil, // 27: tunnel.v1.FileChunk.MetadataEntry
+ nil, // 28: tunnel.v1.StreamOpen.HeadersEntry
+ nil, // 29: tunnel.v1.HttpRequest.HeadersEntry
+ nil, // 30: tunnel.v1.HttpResponse.HeadersEntry
+ nil, // 31: tunnel.v1.WebSocketStart.HeadersEntry
}
var file_tunnel_v1_tunnel_proto_depIdxs = []int32{
- 13, // 0: tunnel.v1.AgentMessage.http_response:type_name -> tunnel.v1.HttpResponse
- 16, // 1: tunnel.v1.AgentMessage.heartbeat_ping:type_name -> tunnel.v1.HeartbeatPing
- 19, // 2: tunnel.v1.AgentMessage.ws_data:type_name -> tunnel.v1.WebSocketData
- 20, // 3: tunnel.v1.AgentMessage.ws_close:type_name -> tunnel.v1.WebSocketClose
- 14, // 4: tunnel.v1.AgentMessage.stream_data:type_name -> tunnel.v1.StreamData
- 15, // 5: tunnel.v1.AgentMessage.stream_end:type_name -> tunnel.v1.StreamEnd
+ 14, // 0: tunnel.v1.AgentMessage.http_response:type_name -> tunnel.v1.HttpResponse
+ 17, // 1: tunnel.v1.AgentMessage.heartbeat_ping:type_name -> tunnel.v1.HeartbeatPing
+ 20, // 2: tunnel.v1.AgentMessage.ws_data:type_name -> tunnel.v1.WebSocketData
+ 21, // 3: tunnel.v1.AgentMessage.ws_close:type_name -> tunnel.v1.WebSocketClose
+ 15, // 4: tunnel.v1.AgentMessage.stream_data:type_name -> tunnel.v1.StreamData
+ 16, // 5: tunnel.v1.AgentMessage.stream_end:type_name -> tunnel.v1.StreamEnd
2, // 6: tunnel.v1.AgentMessage.register:type_name -> tunnel.v1.RegisterRequest
- 21, // 7: tunnel.v1.AgentMessage.event:type_name -> tunnel.v1.EventLog
+ 22, // 7: tunnel.v1.AgentMessage.event:type_name -> tunnel.v1.EventLog
5, // 8: tunnel.v1.AgentMessage.command_ack:type_name -> tunnel.v1.CommandAck
6, // 9: tunnel.v1.AgentMessage.command_output:type_name -> tunnel.v1.CommandOutput
7, // 10: tunnel.v1.AgentMessage.command_complete:type_name -> tunnel.v1.CommandComplete
8, // 11: tunnel.v1.AgentMessage.file_chunk:type_name -> tunnel.v1.FileChunk
10, // 12: tunnel.v1.AgentMessage.stream_close:type_name -> tunnel.v1.StreamClose
11, // 13: tunnel.v1.AgentMessage.cancel_request:type_name -> tunnel.v1.CancelRequest
- 12, // 14: tunnel.v1.ManagerMessage.http_request:type_name -> tunnel.v1.HttpRequest
- 17, // 15: tunnel.v1.ManagerMessage.heartbeat_pong:type_name -> tunnel.v1.HeartbeatPong
- 18, // 16: tunnel.v1.ManagerMessage.ws_start:type_name -> tunnel.v1.WebSocketStart
- 19, // 17: tunnel.v1.ManagerMessage.ws_data:type_name -> tunnel.v1.WebSocketData
- 20, // 18: tunnel.v1.ManagerMessage.ws_close:type_name -> tunnel.v1.WebSocketClose
+ 13, // 14: tunnel.v1.ManagerMessage.http_request:type_name -> tunnel.v1.HttpRequest
+ 18, // 15: tunnel.v1.ManagerMessage.heartbeat_pong:type_name -> tunnel.v1.HeartbeatPong
+ 19, // 16: tunnel.v1.ManagerMessage.ws_start:type_name -> tunnel.v1.WebSocketStart
+ 20, // 17: tunnel.v1.ManagerMessage.ws_data:type_name -> tunnel.v1.WebSocketData
+ 21, // 18: tunnel.v1.ManagerMessage.ws_close:type_name -> tunnel.v1.WebSocketClose
3, // 19: tunnel.v1.ManagerMessage.register_response:type_name -> tunnel.v1.RegisterResponse
4, // 20: tunnel.v1.ManagerMessage.command_request:type_name -> tunnel.v1.CommandRequest
9, // 21: tunnel.v1.ManagerMessage.stream_open:type_name -> tunnel.v1.StreamOpen
10, // 22: tunnel.v1.ManagerMessage.stream_close:type_name -> tunnel.v1.StreamClose
11, // 23: tunnel.v1.ManagerMessage.cancel_request:type_name -> tunnel.v1.CancelRequest
8, // 24: tunnel.v1.ManagerMessage.file_chunk:type_name -> tunnel.v1.FileChunk
- 14, // 25: tunnel.v1.ManagerMessage.stream_data:type_name -> tunnel.v1.StreamData
- 15, // 26: tunnel.v1.ManagerMessage.stream_end:type_name -> tunnel.v1.StreamEnd
- 22, // 27: tunnel.v1.CommandRequest.headers:type_name -> tunnel.v1.CommandRequest.HeadersEntry
- 23, // 28: tunnel.v1.CommandRequest.metadata:type_name -> tunnel.v1.CommandRequest.MetadataEntry
- 24, // 29: tunnel.v1.CommandComplete.headers:type_name -> tunnel.v1.CommandComplete.HeadersEntry
- 25, // 30: tunnel.v1.FileChunk.metadata:type_name -> tunnel.v1.FileChunk.MetadataEntry
- 26, // 31: tunnel.v1.StreamOpen.headers:type_name -> tunnel.v1.StreamOpen.HeadersEntry
- 27, // 32: tunnel.v1.HttpRequest.headers:type_name -> tunnel.v1.HttpRequest.HeadersEntry
- 28, // 33: tunnel.v1.HttpResponse.headers:type_name -> tunnel.v1.HttpResponse.HeadersEntry
- 29, // 34: tunnel.v1.WebSocketStart.headers:type_name -> tunnel.v1.WebSocketStart.HeadersEntry
- 0, // 35: tunnel.v1.TunnelService.Connect:input_type -> tunnel.v1.AgentMessage
- 1, // 36: tunnel.v1.TunnelService.Connect:output_type -> tunnel.v1.ManagerMessage
- 36, // [36:37] is the sub-list for method output_type
- 35, // [35:36] is the sub-list for method input_type
- 35, // [35:35] is the sub-list for extension type_name
- 35, // [35:35] is the sub-list for extension extendee
- 0, // [0:35] is the sub-list for field type_name
+ 15, // 25: tunnel.v1.ManagerMessage.stream_data:type_name -> tunnel.v1.StreamData
+ 16, // 26: tunnel.v1.ManagerMessage.stream_end:type_name -> tunnel.v1.StreamEnd
+ 12, // 27: tunnel.v1.ManagerMessage.command_credit:type_name -> tunnel.v1.CommandCredit
+ 23, // 28: tunnel.v1.CommandRequest.headers:type_name -> tunnel.v1.CommandRequest.HeadersEntry
+ 24, // 29: tunnel.v1.CommandRequest.metadata:type_name -> tunnel.v1.CommandRequest.MetadataEntry
+ 25, // 30: tunnel.v1.CommandOutput.headers:type_name -> tunnel.v1.CommandOutput.HeadersEntry
+ 26, // 31: tunnel.v1.CommandComplete.headers:type_name -> tunnel.v1.CommandComplete.HeadersEntry
+ 27, // 32: tunnel.v1.FileChunk.metadata:type_name -> tunnel.v1.FileChunk.MetadataEntry
+ 28, // 33: tunnel.v1.StreamOpen.headers:type_name -> tunnel.v1.StreamOpen.HeadersEntry
+ 29, // 34: tunnel.v1.HttpRequest.headers:type_name -> tunnel.v1.HttpRequest.HeadersEntry
+ 30, // 35: tunnel.v1.HttpResponse.headers:type_name -> tunnel.v1.HttpResponse.HeadersEntry
+ 31, // 36: tunnel.v1.WebSocketStart.headers:type_name -> tunnel.v1.WebSocketStart.HeadersEntry
+ 0, // 37: tunnel.v1.TunnelService.Connect:input_type -> tunnel.v1.AgentMessage
+ 1, // 38: tunnel.v1.TunnelService.Connect:output_type -> tunnel.v1.ManagerMessage
+ 38, // [38:39] is the sub-list for method output_type
+ 37, // [37:38] is the sub-list for method input_type
+ 37, // [37:37] is the sub-list for extension type_name
+ 37, // [37:37] is the sub-list for extension extendee
+ 0, // [0:37] is the sub-list for field type_name
}
func init() { file_tunnel_v1_tunnel_proto_init() }
@@ -2258,6 +2360,7 @@ func file_tunnel_v1_tunnel_proto_init() {
(*ManagerMessage_FileChunk)(nil),
(*ManagerMessage_StreamData)(nil),
(*ManagerMessage_StreamEnd)(nil),
+ (*ManagerMessage_CommandCredit)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -2265,7 +2368,7 @@ func file_tunnel_v1_tunnel_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_tunnel_v1_tunnel_proto_rawDesc), len(file_tunnel_v1_tunnel_proto_rawDesc)),
NumEnums: 0,
- NumMessages: 30,
+ NumMessages: 32,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/backend/proto/tunnel/v1/tunnel.proto b/backend/proto/tunnel/v1/tunnel.proto
index 49e028020..5477d6514 100644
--- a/backend/proto/tunnel/v1/tunnel.proto
+++ b/backend/proto/tunnel/v1/tunnel.proto
@@ -50,6 +50,8 @@ message ManagerMessage {
// older agents decode an unknown oneof case as a nil payload.
StreamData stream_data = 12;
StreamEnd stream_end = 13;
+ // Only sent to agents that advertised command-credit-v1.
+ CommandCredit command_credit = 14;
}
}
@@ -93,6 +95,10 @@ message CommandOutput {
string command_id = 1;
bytes data = 2;
int64 sequence = 3;
+ // Set on the first chunk so the manager can stream the response before
+ // command_complete arrives. Older managers ignore these fields.
+ int32 status = 4;
+ map<string, string> headers = 5;
}
message CommandComplete {
@@ -130,6 +136,12 @@ message CancelRequest {
string command_id = 1;
}
+// CommandCredit returns flow-control credit for command output the manager has consumed.
+message CommandCredit {
+ string command_id = 1;
+ int64 bytes = 2;
+}
+
message HttpRequest {
string request_id = 1;
string method = 2;