Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
perf: stream edge responses, pool proxy connections, and clean up edge package (#4354)
Kyle Mendelldf8049171e4e
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex e8b129086..295704c20 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -66,6 +66,7 @@ func Bootstrap(ctx context.Context) error {slog.SetDefault(slog.New(logs.NewSlogHandler(slog.Default().Handler(), ws.LogBroadcaster())))database.SetGormLogger(BuildGormLogger(cfg))slog.InfoContext(ctx, "Arcane is starting...", "version", config.Version)+ startup.ApplyMemoryLimit(ctx)slog.InfoContext(ctx, "Arcane Identity Configuration", "puid", os.Getuid(), "pgid", os.Getgid())appCtx, cancelApp := context.WithCancel(ctx)diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.goindex 5b2e61687..0b4952f81 100644--- a/backend/internal/bootstrap/edge_bootstrap.go+++ b/backend/internal/bootstrap/edge_bootstrap.go@@ -50,7 +50,7 @@ func registerEdgeTunnelRoutes(}server := edge.NewTunnelServerWithRegistry(registry, resolver, statusCallback)- server.SetConfig(&edge.Config{+ server.Config = &edge.Config{EdgeMTLSMode: cfg.EdgeMTLSMode,EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,EdgeMTLSCertFile: cfg.EdgeMTLSCertFile,@@ -59,8 +59,8 @@ func registerEdgeTunnelRoutes(EdgeMTLSAssetsDir: cfg.EdgeMTLSAssetsDir,AppURL: cfg.GetAppURL(),ManagerApiUrl: cfg.ManagerApiUrl,- })- server.SetEnvironmentNameResolver(func(ctx context.Context, envID string) (string, error) {+ }+ server.NameResolver = func(ctx context.Context, envID string) (string, error) {env, err := environmentService.GetEnvironmentByID(ctx, envID)if err != nil {return "", err@@ -69,9 +69,9 @@ func registerEdgeTunnelRoutes(return "", nil}return env.Name, nil- })- server.SetEventCallback(eventCallback)- server.SetEnrollmentCallback(func(ctx context.Context, envID, remoteAddr string, certIssued, caGenerated, reenrolled bool) {+ }+ server.EventCallback = eventCallback+ server.EnrollmentCallback = func(ctx context.Context, envID, remoteAddr string, certIssued, caGenerated, reenrolled bool) {if eventService == nil {return}@@ -93,7 +93,7 @@ func registerEdgeTunnelRoutes(Metadata: database.JSON{"remoteAddr": remoteAddr, "reenrollment": reenrolled},})createEdgeMTLSIssueEventsInternal(ctx, eventService, envIDCopy, envNameCopy, remoteAddr, certIssued, caGenerated, reenrolled)- })+ }var stopCleanup func(context.Context) errorlifecycle.Append(fx.Hook{OnStart: func(context.Context) error {diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.goindex 10d390bbe..5d4ab342a 100644--- a/backend/internal/bootstrap/server_bootstrap.go+++ b/backend/internal/bootstrap/server_bootstrap.go@@ -47,7 +47,7 @@ func NewHTTPServer(lc fx.Lifecycle, p HTTPServerParams) (*http.Server, error) {return nil, err}if p.TunnelServer != nil {- p.TunnelServer.SetConfig(edgeCfg)+ p.TunnelServer.Config = edgeCfg}httpHandler, grpcServer := configureTunnelServerInternal(p.AppCtx, p.Config, p.Router, p.TunnelServer, listenAddr)diff --git a/backend/internal/build/service.go b/backend/internal/build/service.goindex bfd43cb92..d1a358885 100644--- a/backend/internal/build/service.go+++ b/backend/internal/build/service.go@@ -390,7 +390,8 @@ func (s *BuildService) cloneGitContextInternal(ctx context.Context, repositoryUR}if s.gitRepository != nil && s.gitRepository.Client != nil {- return s.gitRepository.Clone(ctx, repositoryURL, ref, authConfig)+ // Builds may run git describe or count commits, so they keep full history.+ return s.gitRepository.Clone(ctx, repositoryURL, ref, authConfig, 0)}return "", errors.New("git repository service not available")diff --git a/backend/internal/gitops/children/sync/service.go b/backend/internal/gitops/children/sync/service.goindex ee25a239a..17a5593d6 100644--- a/backend/internal/gitops/children/sync/service.go+++ b/backend/internal/gitops/children/sync/service.go@@ -107,31 +107,42 @@ func (s *Service) DirectoryProject(ctx context.Context, sync *projectpkg.GitOpsS// ReconcileRevision confirms an interrupted run when the record already holds// the source revision that run pinned.func ReconcileRevision(ctx context.Context, previous scheduler.Run, record *projectpkg.GitOpsSync, outcome scheduler.Outcome) (scheduler.Outcome, error) {- for _, target := range previous.Outcome.Targets {- if target.ID != record.ID || len(target.RecoveryData) == 0 {- continue- }- var revision syncRevisionInternal- if unmarshalErr := json.Unmarshal(target.RecoveryData, &revision); unmarshalErr != nil {- return outcome, unmarshalErr- }- if revision.Commit != "" && record.LastSyncCommit != nil && *record.LastSyncCommit == revision.Commit && record.LastSyncStatus != nil && *record.LastSyncStatus == "success" {- target.Status = scheduler.Succeeded- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {- return outcome, progressErr- }- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil- }+ target, commit, err := recordedRevision(previous, record.ID)+ if err != nil {+ return outcome, err+ }+ if commit == "" || kit.FromPtr(record.LastSyncCommit) != commit || kit.FromPtr(record.LastSyncStatus) != "success" {+ return outcome, nil}- return outcome, nil+ target.Status = scheduler.Succeeded+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ return outcome, progressErr+ }+ return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil}-// preparedSyncSource captures the repository data needed by the sync execution-// paths after the source repository has been cloned and validated.-type syncRevisionInternal struct {+// recordedRevision returns the target and source revision an earlier run recorded for syncID.+func recordedRevision(run scheduler.Run, syncID string) (scheduler.TargetOutcome, string, error) {+ idx := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == syncID && len(target.RecoveryData) > 0+ })+ if idx < 0 {+ return scheduler.TargetOutcome{}, "", nil+ }+ var revision syncRevision+ if err := json.Unmarshal(run.Outcome.Targets[idx].RecoveryData, &revision); err != nil {+ return scheduler.TargetOutcome{}, "", err+ }+ return run.Outcome.Targets[idx], revision.Commit, nil+}++// syncRevision is the recovery data a sync run records so a retry replays the same source revision.+type syncRevision struct {Commit string `json:"commit"`}+// preparedSyncSource captures the repository data needed by the sync execution+// paths after the source repository has been cloned and validated.type preparedSyncSource struct {repoPath stringcommitHash string@@ -177,35 +188,56 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsreturn nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to get authentication config", err.Error())}- repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig)+ // A retry replays the revision its interrupted predecessor recorded, which needs full history.+ pinnedCommit := ""+ if previous, ok := jobcontext.Run(ctx); ok {+ if _, pinnedCommit, err = recordedRevision(previous, sync.ID); err != nil {+ return nil, err+ }+ }+ repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig, kit.Ternary(pinnedCommit == "", 1, 0))if err != nil {return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to clone repository", err.Error())}+ source := &preparedSyncSource{repoPath: repoPath}- commitHash, err := s.repoService.GetCurrentCommit(ctx, repoPath)+ source.commitHash, err = s.repoService.GetCurrentCommit(ctx, repoPath)if err != nil {slog.WarnContext(ctx, "Failed to get commit hash", "error", err)- commitHash = ""}-- commitHash, err = pinSyncRevisionInternal(ctx, sync.ID, repoPath, commitHash)+ if pinnedCommit != "" && pinnedCommit != source.commitHash {+ repo, openErr := git.PlainOpen(repoPath)+ if openErr != nil {+ return source, openErr+ }+ tree, treeErr := repo.Worktree()+ if treeErr != nil {+ return source, treeErr+ }+ if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(pinnedCommit)}); checkoutErr != nil {+ return source, checkoutErr+ }+ source.commitHash = pinnedCommit+ }+ if source.commitHash == "" {+ return source, errors.New("GitOps source revision is unavailable")+ }+ revision, err := json.Marshal(syncRevision{Commit: source.commitHash})if err != nil {- return &preparedSyncSource{repoPath: repoPath}, err+ return source, err+ }+ progress := scheduler.TargetOutcome{ResourceType: "gitops_sync", ID: sync.ID, Status: scheduler.Running, RecoveryData: revision}+ if progressErr := jobcontext.Progress(ctx, progress); progressErr != nil {+ return source, progressErr}+if !s.repoService.FileExists(ctx, repoPath, sync.ComposePath) {errMsg := "compose file not found: " + sync.ComposePath- return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, "Compose file not found at "+sync.ComposePath, errMsg)+ return source, s.failSync(ctx, sync.ID, result, sync, actor, "Compose file not found at "+sync.ComposePath, errMsg)}-- composeContent, err := s.repoService.ReadFile(ctx, repoPath, sync.ComposePath)+ source.composeContent, err = s.repoService.ReadFile(ctx, repoPath, sync.ComposePath)if err != nil {- return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())- }-- source := &preparedSyncSource{- repoPath: repoPath,- commitHash: commitHash,- composeContent: composeContent,+ return source, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())}envPath := filepath.Join(filepath.Dir(sync.ComposePath), ".env")@@ -248,45 +280,6 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsreturn source, nil}-func pinSyncRevisionInternal(ctx context.Context, syncID, repoPath, commitHash string) (string, error) {- if previous, ok := jobcontext.Run(ctx); ok {- for _, target := range previous.Outcome.Targets {- if target.ID != syncID || len(target.RecoveryData) == 0 {- continue- }- var revision syncRevisionInternal- if err := json.Unmarshal(target.RecoveryData, &revision); err != nil {- return "", err- }- if revision.Commit != "" && revision.Commit != commitHash {- repository, err := git.PlainOpen(repoPath)- if err != nil {- return "", err- }- tree, err := repository.Worktree()- if err != nil {- return "", err- }- if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(revision.Commit)}); checkoutErr != nil {- return "", checkoutErr- }- commitHash = revision.Commit- }- }- }- revision, err := json.Marshal(syncRevisionInternal{Commit: commitHash})- if err != nil {- return "", err- }- if commitHash == "" {- return "", errors.New("GitOps source revision is unavailable")- }- if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "gitops_sync", ID: syncID, Status: scheduler.Running, RecoveryData: revision}); progressErr != nil {- return "", progressErr- }- return commitHash, nil-}-// performDirectorySync runs the directory-sync path and only triggers a// redeploy when an already running project's synced contents changed.func (diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.goindex 62e549939..a30501164 100644--- a/backend/internal/gitops/service.go+++ b/backend/internal/gitops/service.go@@ -1171,7 +1171,7 @@ func (s *GitOpsSyncService) BrowseFiles(ctx context.Context, environmentID, id,}// Clone the repository- repoPath, err := s.repoService.Clone(browseCtx, repository.URL, syncRecord.Branch, authConfig)+ repoPath, err := s.repoService.Clone(browseCtx, repository.URL, syncRecord.Branch, authConfig, 1)if err != nil {return nil, fmt.Errorf("failed to clone repository: %w", err)}diff --git a/backend/internal/gitops/service_test.go b/backend/internal/gitops/service_test.goindex c54c78ed8..341ca4e2a 100644--- a/backend/internal/gitops/service_test.go+++ b/backend/internal/gitops/service_test.go@@ -1197,7 +1197,7 @@ func (e *backupTestEnvInternal) reloadInternal(t *testing.T, id string) *project// checkoutRemoteInternal clones the backup branch so the pushed tree can be inspected.func (e *backupTestEnvInternal) checkoutRemoteInternal(t *testing.T) string {t.Helper()- repoPath, err := e.remote.Clone(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"})+ repoPath, err := e.remote.Clone(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"}, 0)require.NoError(t, err)t.Cleanup(func() { _ = e.remote.Cleanup(repoPath) })return repoPathdiff --git a/backend/internal/gitrepo/service.go b/backend/internal/gitrepo/service.goindex 57bcd9aab..926a17906 100644--- a/backend/internal/gitrepo/service.go+++ b/backend/internal/gitrepo/service.go@@ -540,7 +540,7 @@ func (s *GitRepositoryService) BrowseFiles(ctx context.Context, id, branch, path}// Clone the repository- repoPath, err := s.Clone(ctx, repository.URL, branch, authConfig)+ repoPath, err := s.Clone(ctx, repository.URL, branch, authConfig, 1)if err != nil {return nil, fmt.Errorf("failed to clone repository: %w", err)}diff --git a/backend/internal/image/children/attestations/service.go b/backend/internal/image/children/attestations/service.goindex 487b7a06a..aab316a77 100644--- a/backend/internal/image/children/attestations/service.go+++ b/backend/internal/image/children/attestations/service.go@@ -12,6 +12,7 @@ import ("log/slog""net/http""strings"+ "sync""github.com/containerd/platforms""github.com/getarcaneapp/arcane/types/v2/image"@@ -28,6 +29,9 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth")+// zstdDecoder is shared because DecodeAll is safe for concurrent use and a decoder is costly to build.+var zstdDecoder = sync.OnceValues(func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })+// Service reads in-toto attestations attached to local images and registry references.type Service struct {dockerClient func(ctx context.Context) (*client.Client, error)@@ -502,12 +506,11 @@ func decompressAttestationStatementInternal(data []byte) ([]byte, error) {defer func() { _ = reader.Close() }()return io.ReadAll(reader)case len(data) >= 4 && data[0] == 0x28 && data[1] == 0xb5 && data[2] == 0x2f && data[3] == 0xfd:- reader, err := zstd.NewReader(nil)+ decoder, err := zstdDecoder()if err != nil {return nil, err}- defer reader.Close()- return reader.DecodeAll(data, nil)+ return decoder.DecodeAll(data, nil)default:return data, nil}diff --git a/backend/internal/middleware/environment.go b/backend/internal/middleware/environment.goindex b7b200a94..d24466ae3 100644--- a/backend/internal/middleware/environment.go+++ b/backend/internal/middleware/environment.go@@ -19,6 +19,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/container""github.com/getarcaneapp/arcane/types/v2/gitops"+ httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/volume""github.com/labstack/echo/v5"@@ -107,7 +108,7 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(paramName: paramName,resolver: resolver,authValidator: authValidator,- httpClient: &http.Client{Timeout: proxyTimeout},+ httpClient: httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second}),registry: registry,matcher: matcher,checkOrigin: checkOrigin,@@ -774,8 +775,7 @@ func (m *EnvironmentMiddleware) createProxyRequest(c *echo.Context, target strinContentLength: contentLength,}).WithContext(srcReq.Context())- skip := edge.GetSkipHeaders()- edge.CopyRequestHeaders(srcReq.Header, req.Header, skip)+ edge.CopyRequestHeaders(srcReq.Header, req.Header)edge.SetAuthHeader(req, c)edge.SetAgentToken(req, accessToken)edge.SetForwardedHeaders(req, c.RealIP(), srcReq.Host)diff --git a/backend/internal/project/children/sync/service.go b/backend/internal/project/children/sync/service.goindex a60e5bb8f..6365c9e85 100644--- a/backend/internal/project/children/sync/service.go+++ b/backend/internal/project/children/sync/service.go@@ -243,19 +243,37 @@ func ApplyGitSyncEnv(ctx context.Context, projectPath, projectsDirectory string,return update.State.DirectContent, kit.FromPtr(update.EffectiveContent), nil}-// LoadComposeMetadata loads a discovered project's compose file once and-// returns the service count plus compose-go's effective project name.-func LoadComposeMetadata(ctx context.Context, dirPath, dirName, projectsDirectory string, autoInjectEnv bool, pathMapper *projects.PathMapper) (project.ComposeIdentity, error) {+// LoadComposeMetadata returns a discovered project's service count plus+// compose-go's effective project name. Results are cached by dirPath until a+// compose, include or env file changes.+func LoadComposeMetadata(+ ctx context.Context,+ cache project.ComposeCache[project.ComposeIdentity],+ dirPath, dirName, projectsDirectory string,+ autoInjectEnv bool,+ pathMapper *projects.PathMapper,+) (project.ComposeIdentity, error) {normName := projects.NormalizeProjectName(dirName)meta := project.ComposeIdentity{ResolvedProjectName: normName,}+ composeFile, err := projects.DetectComposeFile(ctx, projectsDirectory, dirPath)+ if err != nil {+ return meta, err+ }+ fingerprint := fmt.Sprintf("%q|%q|%q|%t|%#v", composeFile, normName, projectsDirectory, autoInjectEnv, pathMapper)+ if cached, ok := cache.Get(dirPath, fingerprint); ok {+ return cached, nil+ }+// Load unnamed first so COMPOSE_PROJECT_NAME from .env wins; fall back to// the normalized directory name when that fails.- proj, _, err := projects.LoadComposeProjectFromDir(ctx, dirPath, "", projectsDirectory, autoInjectEnv, pathMapper)+ dependencies := project.ComposeDependencies{}+ proj, err := projects.LoadComposeProject(ctx, composeFile, "", projectsDirectory, autoInjectEnv, pathMapper, nil, nil, false, &dependencies, nil, nil)if err != nil {- proj, _, err = projects.LoadComposeProjectFromDir(ctx, dirPath, normName, projectsDirectory, autoInjectEnv, pathMapper)+ dependencies = project.ComposeDependencies{}+ proj, err = projects.LoadComposeProject(ctx, composeFile, normName, projectsDirectory, autoInjectEnv, pathMapper, nil, nil, false, &dependencies, nil, nil)if err != nil {return meta, err}@@ -273,5 +291,10 @@ func LoadComposeMetadata(ctx context.Context, dirPath, dirName, projectsDirectormeta.ComposeProjectName = new(proj.Name)}+ if composeFiles, envFiles, ok := projects.ComposeCacheDependencies(ctx, proj, dependencies, composeFile, projectsDirectory, autoInjectEnv); ok {+ if setErr := cache.Set(dirPath, fingerprint, dirPath, projectsDirectory, composeFile, composeFiles, envFiles, meta); setErr != nil {+ slog.DebugContext(ctx, "failed to cache compose metadata", "path", dirPath, "error", setErr)+ }+ }return meta, nil}diff --git a/backend/internal/project/service.go b/backend/internal/project/service.goindex da4753576..7efcf7816 100644--- a/backend/internal/project/service.go+++ b/backend/internal/project/service.go@@ -115,6 +115,9 @@ type ProjectService struct {// projects, a gitops_syncs lookup — per project, on every list request.// Entries are validated by compose/include/env file mtimes rather than a TTL.metaCache projecttypes.ComposeCache[projects.ArcaneComposeMetadata]+ // composeIdentities caches filesystem-sync compose identities by project path,+ // so a sync only re-parses projects whose compose or env files changed.+ composeIdentities projecttypes.ComposeCache[projecttypes.ComposeIdentity]// FilesChanged fires with the project ID after project files are saved// through Arcane, so Git backups can react without polling.@@ -313,6 +316,7 @@ func NewProjectService(config: cfg,parsedCompose: projects.NewParsedComposeCache(),metaCache: projects.NewComposeCache[projects.ArcaneComposeMetadata](1024, nil),+ composeIdentities: projects.NewComposeCache[projecttypes.ComposeIdentity](2048, nil),FilesChanged: concurrency.NewSignal[string](),}s.initChildren(kvService, buildService)@@ -719,7 +723,7 @@ func (s *ProjectService) refreshComposeProjectName(ctx context.Context, proj *Prif err == nil {dirName := cmp.Or(mo.PointerToOption(proj.DirName).OrEmpty(), proj.Name)autoInjectEnv := kit.ParseOrDefault(cfg.AutoInjectEnv.Value, false, strconv.ParseBool)- meta, err = projectsync.LoadComposeMetadata(ctx, proj.Path, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))+ meta, err = projectsync.LoadComposeMetadata(ctx, s.composeIdentities, proj.Path, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))}if err != nil {if errors.Is(err, common.ErrProjectEnvUnreadable) {@@ -2615,7 +2619,7 @@ func (s *ProjectService) upsertProjectForDir(ctx context.Context, dirName, dirPaprojectsDirectory, serviceCountErr := projects.GetProjectsDirectory(ctx, cfg.ProjectsDirectory.Value)if serviceCountErr == nil {autoInjectEnv := kit.ParseOrDefault(cfg.AutoInjectEnv.Value, false, strconv.ParseBool)- composeMetadata, serviceCountErr = projectsync.LoadComposeMetadata(ctx, dirPath, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))+ composeMetadata, serviceCountErr = projectsync.LoadComposeMetadata(ctx, s.composeIdentities, dirPath, dirName, projectsDirectory, autoInjectEnv, s.projectPathMapper(ctx))}serviceCountLogLevel := kit.Ternary(errors.Is(serviceCountErr, common.ErrProjectEnvUnreadable), slog.LevelDebug, slog.LevelWarn)diff --git a/backend/internal/vulnerability/children/ignores/service.go b/backend/internal/vulnerability/children/ignores/service.goindex 1dd31d7cd..cf9c5d7a0 100644--- a/backend/internal/vulnerability/children/ignores/service.go+++ b/backend/internal/vulnerability/children/ignores/service.go@@ -31,6 +31,8 @@ type Records struct {DeleteIDs func(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error)// Page counts the ignore records of envID and lists one ordered page of them.Page func(ctx context.Context, envID string, order func(*gorm.DB) *gorm.DB, offset, limit int) ([]vulnerability.IgnoredVulnerability, int64, error)+ // Changed runs after an ignore change commits.+ Changed func()}// Service records which vulnerability findings are ignored per environment.@@ -119,6 +121,7 @@ func (s *Service) IgnoreVulnerability(ctx context.Context, envID string, payloadif createIgnoreErr != nil {return nil, fmt.Errorf("failed to create ignore record: %w", createIgnoreErr)}+ s.records.Changed()slog.InfoContext(ctx, "vulnerability ignored","vulnerabilityId", payload.VulnerabilityID,@@ -146,6 +149,7 @@ func (s *Service) UnignoreVulnerability(ctx context.Context, envID, ignoreID strif deleteErr := s.records.DeleteIdentity(ctx, ignore); deleteErr != nil {return fmt.Errorf("failed to delete ignore record: %w", deleteErr)}+ s.records.Changed()slog.InfoContext(ctx, "vulnerability unignored", "ignoreId", ignoreID)return nil@@ -197,6 +201,7 @@ func (s *Service) BulkIgnoreVulnerabilities(ctx context.Context, envID string, fif err != nil {return 0, err}+ s.records.Changed()slog.InfoContext(ctx, "vulnerabilities ignored in bulk", "environmentId", envID, "affected", len(ignores))return len(ignores), nil@@ -228,6 +233,7 @@ func (s *Service) BulkUnignoreVulnerabilities(ctx context.Context, envID string,if err != nil {return 0, err}+ s.records.Changed()slog.InfoContext(ctx, "vulnerabilities unignored in bulk", "environmentId", envID, "affected", affected)return int(affected), nildiff --git a/backend/internal/vulnerability/children/query/service.go b/backend/internal/vulnerability/children/query/service.goindex f51685e22..1dd27faa1 100644--- a/backend/internal/vulnerability/children/query/service.go+++ b/backend/internal/vulnerability/children/query/service.go@@ -15,6 +15,8 @@ import ("sort""strconv""strings"+ "sync"+ "sync/atomic""time""github.com/getarcaneapp/arcane/types/v2"@@ -23,6 +25,7 @@ import ("github.com/moby/moby/api/types/container""github.com/samber/mo""go.getarcane.app/kit/pkg"+ "golang.org/x/sync/singleflight""gorm.io/gorm""gorm.io/gorm/clause"@@ -40,6 +43,11 @@ const (scansTableInternal = "vulnerability_scans"scanItemsTableInternal = "vulnerability_scan_items"threatIntelTableInternal = "vulnerability_threat_intel"++ // riskOverviewCacheTTL bounds staleness from changes that do not invalidate+ // the cache, such as containers starting or threat intel refreshing.+ riskOverviewCacheTTL = 5 * time.Second+ riskOverviewBuildTimeout = 30 * time.Second)// Service answers vulnerability result, listing, export, and risk queries.@@ -52,6 +60,18 @@ type Service struct {refreshSummary func(ctx context.Context, summary *vulnerability.ScanSummary)threatIntelStatus func(ctx context.Context) vulnerability.ThreatIntelStatusfilterIgnored func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error)++ // riskFlight and riskCache share one risk overview build per environment;+ // riskGen is bumped by InvalidateRiskOverview to drop cached overviews.+ riskFlight singleflight.Group+ riskCache sync.Map // envID -> *riskOverviewCacheEntryInternal+ riskGen atomic.Uint64+}++type riskOverviewCacheEntryInternal struct {+ overview *vulnerability.RiskOverview+ builtAt time.Time+ gen uint64}func NewService(@@ -1035,11 +1055,64 @@ func (s *Service) riskNowInternal() time.Time {}// GetRiskOverview returns the environment patch priority with its trend, drivers, prioritized images and top findings.+// Concurrent callers share one build, and the result is reused briefly. Callers must not mutate it.func (s *Service) GetRiskOverview(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {return nil, common.ErrFeatureDisabled}+ if overview, ok := s.cachedRiskOverviewInternal(envID); ok {+ return overview, nil+ }++ // Keying the build by generation keeps a request that follows an invalidation off an older build.+ gen := s.riskGen.Load()+ results := s.riskFlight.DoChan(envID+":"+strconv.FormatUint(gen, 10), func() (any, error) {+ // A caller that queued behind the winner finds the fresh entry here.+ if overview, ok := s.cachedRiskOverviewInternal(envID); ok {+ return overview, nil+ }+ buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), riskOverviewBuildTimeout)+ defer cancel()+ overview, buildErr := s.buildRiskOverviewInternal(buildCtx, envID)+ if buildErr != nil {+ return nil, buildErr+ }+ s.riskCache.Store(envID, &riskOverviewCacheEntryInternal{overview: overview, builtAt: time.Now(), gen: gen})+ return overview, nil+ })+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case result := <-results:+ if result.Err != nil {+ return nil, result.Err+ }+ overview, ok := result.Val.(*vulnerability.RiskOverview)+ if !ok {+ return nil, errors.New("risk overview cache returned unexpected type")+ }+ return overview, nil+ }+}++// InvalidateRiskOverview drops cached risk overviews. Call it after a scan or ignore change commits.+func (s *Service) InvalidateRiskOverview() {+ s.riskGen.Add(1)+}++func (s *Service) cachedRiskOverviewInternal(envID string) (*vulnerability.RiskOverview, bool) {+ value, ok := s.riskCache.Load(envID)+ if !ok {+ return nil, false+ }+ entry, ok := value.(*riskOverviewCacheEntryInternal)+ if !ok || entry.gen != s.riskGen.Load() || time.Since(entry.builtAt) >= riskOverviewCacheTTL {+ return nil, false+ }+ return entry.overview, true+}+func (s *Service) buildRiskOverviewInternal(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {overview, err := s.computeRiskInternal(ctx, envID)if err != nil {return nil, errdiff --git a/backend/internal/vulnerability/service.go b/backend/internal/vulnerability/service.goindex 488bf67a2..10d83d071 100644--- a/backend/internal/vulnerability/service.go+++ b/backend/internal/vulnerability/service.go@@ -92,6 +92,7 @@ func NewVulnerabilityService(DeleteIdentity: s.deleteIgnoreIdentityInternal,DeleteIDs: deleteIgnoreIDsInternal,Page: s.ignorePageInternal,+ Changed: s.query.InvalidateRiskOverview,}, s.query.ActiveItemsQuery)s.scan = scan.NewService(scan.Dependencies{DB: db,@@ -109,9 +110,16 @@ func NewVulnerabilityService(Events: eventService,Notifications: notificationService,Activity: activityService,- SaveResult: s.store.SaveResult,- EnsureSummary: store.EnsureSummary,- FilterIgnored: s.ignores.FilterForImage,+ // The risk cache is cleared only once the scan's transaction has committed.+ SaveResult: func(ctx context.Context, result *vulnerability.ScanResult) error {+ if err := s.store.SaveResult(ctx, result); err != nil {+ return err+ }+ s.query.InvalidateRiskOverview()+ return nil+ },+ EnsureSummary: store.EnsureSummary,+ FilterIgnored: s.ignores.FilterForImage,})return s}@@ -303,7 +311,11 @@ func (s *VulnerabilityService) saveRecordStatusInternal(ctx context.Context, res}func (s *VulnerabilityService) deleteRecordsInternal(ctx context.Context, imageIDs []string) error {- return s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error+ if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {+ return err+ }+ s.query.InvalidateRiskOverview()+ return nil}func (s *VulnerabilityService) deleteReportsInternal(ctx context.Context, imageIDs []string) error {diff --git a/backend/pkg/gitutil/git.go b/backend/pkg/gitutil/git.goindex 19a60f338..c1ca6a8ae 100644--- a/backend/pkg/gitutil/git.go+++ b/backend/pkg/gitutil/git.go@@ -319,8 +319,9 @@ func getKnownHostsPathsInternal(}}-// Clone clones a repository to a temporary directory-func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig) (string, error) {+// Clone clones a repository to a temporary directory. A depth of 0 fetches the+// full history; read-only callers pass 1 to fetch only the branch tip.+func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig, depth int) (string, error) {if _, hasDeadline := ctx.Deadline(); !hasDeadline {var cancel context.CancelFuncctx, cancel = context.WithTimeout(ctx, 5*time.Minute)@@ -362,6 +363,7 @@ func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig)cloneOptions := &git.CloneOptions{URL: url,Progress: nil,+ Depth: depth,}if authMethod != nil {@@ -374,6 +376,13 @@ func (c *Client) Clone(ctx context.Context, url, branch string, auth AuthConfig)}_, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)+ if err != nil && depth > 0 && strings.Contains(err.Error(), "shallow") {+ // Servers without shallow support, go-git's own included, get a full clone.+ cloneOptions.Depth = 0+ if err = os.RemoveAll(tmpDir); err == nil {+ _, err = git.PlainCloneContext(ctx, tmpDir, false, cloneOptions)+ }+ }if err != nil {_ = os.RemoveAll(tmpDir)return "", fmt.Errorf("failed to clone repository: %w", err)@@ -639,7 +648,7 @@ func (c *Client) TestConnection(ctx context.Context, url, branch string, auth Aureturn err}- tmpDir, err := c.Clone(ctx, url, branch, auth)+ tmpDir, err := c.Clone(ctx, url, branch, auth, 1)if err != nil {return err}diff --git a/backend/pkg/gitutil/git_write.go b/backend/pkg/gitutil/git_write.goindex caed9309c..c05d5859d 100644--- a/backend/pkg/gitutil/git_write.go+++ b/backend/pkg/gitutil/git_write.go@@ -10,6 +10,7 @@ import ("os""path""path/filepath"+ "slices""sort""strings""time"@@ -18,7 +19,6 @@ import ("github.com/go-git/go-git/v5""github.com/go-git/go-git/v5/config""github.com/go-git/go-git/v5/plumbing"- "github.com/go-git/go-git/v5/plumbing/format/diff""github.com/go-git/go-git/v5/plumbing/object""github.com/go-git/go-git/v5/plumbing/transport""go.getarcane.app/acfs"@@ -215,7 +215,7 @@ func (c *Client) CheckoutForWrite(ctx context.Context, url, branch string, authreturn nil, err}- repoPath, err := c.Clone(ctx, normalized, branch, auth)+ repoPath, err := c.Clone(ctx, normalized, branch, auth, 0)if err == nil {repo, openErr := git.PlainOpen(repoPath)if openErr != nil {@@ -269,7 +269,7 @@ func (c *Client) initEmptyCheckoutInternal(url, branch string) (*WriteCheckout,}func (c *Client) checkoutNewBranchInternal(ctx context.Context, url, branch string, auth AuthConfig) (*WriteCheckout, error) {- repoPath, err := c.Clone(ctx, url, "", auth)+ repoPath, err := c.Clone(ctx, url, "", auth, 0)if err != nil {return nil, err}@@ -448,10 +448,16 @@ func (c *Client) DirectoryHistory(ctx context.Context, repoPath, directory strin}return nil, fmt.Errorf("failed to iterate history: %w", nextErr)}- files, nextErr := commitDirectoryFilesInternal(commit, prefix)+ changes, nextErr := commitChangesInternal(ctx, commit, object.DefaultDiffTreeOptions)if nextErr != nil {return nil, nextErr}+ var files []string+ for _, change := range changes {+ if _, relative, ok := changePathInDirectoryInternal(change, prefix); ok {+ files = append(files, relative)+ }+ }entries = append(entries, historyEntryInternal(commit, files))if limit > 0 && len(entries) >= limit {break@@ -481,22 +487,31 @@ func (c *Client) CommitDiff(ctx context.Context, repoPath, commitHash, directoryreturn HistoryEntry{}, nil, fmt.Errorf("commit not found: %w", err)}prefix := directoryPrefixInternal(directory)- patch, err := commitPatchInternal(commit)+ // Files are listed rename-aware; each file's patch comes from the plain diff.+ listed, err := commitChangesInternal(ctx, commit, object.DefaultDiffTreeOptions)+ if err != nil {+ return HistoryEntry{}, nil, err+ }+ plain, err := commitChangesInternal(ctx, commit, nil)if err != nil {return HistoryEntry{}, nil, err}var files []stringvar diffs []FileDiff- for _, filePatch := range patch.FilePatches() {- name, relative, ok := patchPathInDirectoryInternal(filePatch, prefix)+ for _, change := range listed {+ name, relative, ok := changePathInDirectoryInternal(change, prefix)if !ok {continue}files = append(files, relative)- single, commitFilePatchErr := commitFilePatchInternal(commit, name)- if commitFilePatchErr != nil {- return HistoryEntry{}, nil, commitFilePatchErr+ single := ""+ if idx := slices.IndexFunc(plain, func(c *object.Change) bool { return c.From.Name == name || c.To.Name == name }); idx >= 0 {+ patch, patchErr := plain[idx].Patch()+ if patchErr != nil {+ return HistoryEntry{}, nil, fmt.Errorf("failed to build patch for %s: %w", name, patchErr)+ }+ single = patch.String()}diffs = append(diffs, FileDiff{Path: relative, Patch: single})}@@ -523,7 +538,9 @@ func historyEntryInternal(commit *object.Commit, files []string) HistoryEntry {}}-func commitPatchInternal(commit *object.Commit) (*object.Patch, error) {+// commitChangesInternal diffs a commit against its first parent. Nil opts+// disables rename detection.+func commitChangesInternal(ctx context.Context, commit *object.Commit, opts *object.DiffTreeOptions) (object.Changes, error) {var parentTree *object.Treeif commit.NumParents() > 0 {parent, err := commit.Parent(0)@@ -539,71 +556,23 @@ func commitPatchInternal(commit *object.Commit) (*object.Patch, error) {if err != nil {return nil, fmt.Errorf("failed to load commit tree: %w", err)}- patch, err := parentTree.Patch(tree)+ changes, err := object.DiffTreeWithOptions(ctx, parentTree, tree, opts)if err != nil {return nil, fmt.Errorf("failed to diff commit: %w", err)}- return patch, nil+ return changes, nil}-func commitDirectoryFilesInternal(commit *object.Commit, prefix string) ([]string, error) {- patch, err := commitPatchInternal(commit)- if err != nil {- return nil, err- }- var files []string- for _, filePatch := range patch.FilePatches() {- if _, relative, ok := patchPathInDirectoryInternal(filePatch, prefix); ok {- files = append(files, relative)- }- }- return files, nil-}--// patchPathInDirectoryInternal picks the side of a file patch that lives under+// changePathInDirectoryInternal picks the side of a change that lives under// prefix, so renames into or out of the directory are still reported.-func patchPathInDirectoryInternal(filePatch diff.FilePatch, prefix string) (string, string, bool) {- from, to := filePatch.Files()- for _, file := range []diff.File{to, from} {- if file == nil {+func changePathInDirectoryInternal(change *object.Change, prefix string) (string, string, bool) {+ for _, name := range []string{change.To.Name, change.From.Name} {+ if name == "" {continue}- if relative, ok := strings.CutPrefix(file.Path(), prefix); ok {- return file.Path(), relative, true+ if relative, ok := strings.CutPrefix(name, prefix); ok {+ return name, relative, true}}return "", "", false}--func commitFilePatchInternal(commit *object.Commit, name string) (string, error) {- var parentTree *object.Tree- if commit.NumParents() > 0 {- parent, err := commit.Parent(0)- if err != nil {- return "", fmt.Errorf("failed to load parent commit: %w", err)- }- parentTree, err = parent.Tree()- if err != nil {- return "", fmt.Errorf("failed to load parent tree: %w", err)- }- }- tree, err := commit.Tree()- if err != nil {- return "", fmt.Errorf("failed to load commit tree: %w", err)- }- changes, err := object.DiffTree(parentTree, tree)- if err != nil {- return "", fmt.Errorf("failed to diff commit: %w", err)- }- for _, change := range changes {- if change.From.Name != name && change.To.Name != name {- continue- }- patch, patchErr := change.Patch()- if patchErr != nil {- return "", fmt.Errorf("failed to build patch for %s: %w", name, patchErr)- }- return patch.String(), nil- }- return "", nil-}diff --git a/backend/pkg/gitutil/git_write_test.go b/backend/pkg/gitutil/git_write_test.goindex 47c30099c..c1ca8a37f 100644--- a/backend/pkg/gitutil/git_write_test.go+++ b/backend/pkg/gitutil/git_write_test.go@@ -117,7 +117,7 @@ func TestCheckoutForWrite_CreatesMissingBranchFromDefault(t *testing.T) {require.Equal(t, head, remoteHead)verify := NewClient(t.TempDir())- repoPath, err := verify.Clone(ctx, url, "feature", noAuthInternal())+ repoPath, err := verify.Clone(ctx, url, "feature", noAuthInternal(), 0)require.NoError(t, err)t.Cleanup(func() { _ = verify.Cleanup(repoPath) })assert.FileExists(t, filepath.Join(repoPath, "unrelated", "keep.txt"))diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.goindex a601fe99c..2c50f2795 100644--- a/backend/pkg/libarcane/edge/client.go+++ b/backend/pkg/libarcane/edge/client.go@@ -10,6 +10,7 @@ import ("log/slog""net""net/http"+ "slices""strings""sync""time"@@ -403,7 +404,7 @@ func (c *TunnelClient) grpcFailureStreakInternal() int {func (c *TunnelClient) registerMessageInternal() *TunnelMessage {capabilities := AdvertisedEdgeCommands()- capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity)+ capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity, tunnelCapabilityCommandCredit)registration, _ := c.registration.Load()return &TunnelMessage{Type: MessageTypeRegister,@@ -457,7 +458,10 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn Tunneif !result.msg.Accepted {return nil, fmt.Errorf("manager rejected tunnel registration: %s", result.msg.Error)}- c.registration.Store(clientRegistrationInternal{sessionID: result.msg.SessionID})+ c.registration.Store(clientRegistrationInternal{+ sessionID: result.msg.SessionID,+ commandCredit: slices.Contains(result.msg.Capabilities, tunnelCapabilityCommandCreditGrant),+ })return result.msg, nil}}@@ -583,6 +587,16 @@ func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, wc.handleStreamClose(ctx, msg)case MessageTypeCancelRequest:slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)+ case MessageTypeCommandCredit:+ if value, ok := c.commandRecorders.Load(msg.ID); ok {+ if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {+ recorder.inFlight.Add(-msg.Credit)+ select {+ case recorder.credited <- struct{}{}:+ default:+ }+ }+ }case MessageTypeResponse, MessageTypeHeartbeat, MessageTypeStreamEnd, MessageTypeEvent, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete:slog.DebugContext(ctx, "Ignoring message type on agent", "type", msg.Type)case MessageTypeHeartbeatAck:@@ -708,7 +722,20 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnreturn}- recorder := newCommandResponseRecorderInternal(msg.ID, msg.Command, conn)+ recorder := &commandResponseRecorder{+ commandID: msg.ID,+ commandName: msg.Command,+ conn: conn,+ headers: make(http.Header),+ statusCode: http.StatusOK,+ ctx: reqCtx,+ credited: make(chan struct{}, 1),+ }+ if registration, _ := c.registration.Load(); registration.commandCredit {+ recorder.window = commandCreditWindow+ c.commandRecorders.Store(msg.ID, recorder)+ defer c.commandRecorders.Delete(msg.ID)+ }c.handler.ServeHTTP(recorder, req)if closeErr := recorder.Close(); closeErr != nil {slog.WarnContext(reqCtx, "Failed to finalize command response", "id", msg.ID, "command", msg.Command, "error", closeErr)@@ -1188,16 +1215,6 @@ func (c *TunnelClient) sendCommandCompleteInternal(conn TunnelConnection, comman})}-func newCommandResponseRecorderInternal(commandID, commandName string, conn TunnelConnection) *commandResponseRecorder {- return &commandResponseRecorder{- commandID: commandID,- commandName: commandName,- conn: conn,- headers: make(http.Header),- statusCode: http.StatusOK,- }-}-func (r *commandResponseRecorder) Header() http.Header {return r.headers}@@ -1207,6 +1224,9 @@ func (r *commandResponseRecorder) Write(b []byte) (int, error) {defer r.mu.Unlock()originalLen := len(b)+ if r.flushErr != nil {+ return 0, r.flushErr+ }if len(b) == 0 {return 0, nil}@@ -1225,16 +1245,9 @@ func (r *commandResponseRecorder) Write(b []byte) (int, error) {if len(chunk) > defaultCommandChunkSize {chunk = chunk[:defaultCommandChunkSize]}- if err := r.conn.Send(&TunnelMessage{- ID: r.commandID,- Type: MessageTypeCommandOutput,- Body: append([]byte(nil), chunk...),- Sequence: r.sequence,- Command: r.commandName,- }); err != nil {+ if err := r.sendOutputLocked(chunk); err != nil {return 0, err}- r.sequence++b = b[len(chunk):]}@@ -1252,7 +1265,15 @@ func (r *commandResponseRecorder) Flush() {r.mu.Lock()defer r.mu.Unlock()r.streaming = true- _ = r.flushBufferLocked()+ if r.flushErr != nil {+ return+ }+ // An empty first flush still sends the status and headers so the manager can start the response.+ if r.sequence == 0 && r.buffer.Len() == 0 {+ r.flushErr = r.sendOutputLocked(nil)+ return+ }+ r.flushErr = r.flushBufferLocked()}func (r *commandResponseRecorder) Close() error {@@ -1261,6 +1282,9 @@ func (r *commandResponseRecorder) Close() error {if r.closed {return nil}+ if r.flushErr != nil {+ return r.flushErr+ }if !r.streaming {err := r.conn.Send(&TunnelMessage{@@ -1301,17 +1325,40 @@ func (r *commandResponseRecorder) flushBufferLocked() error {if r.buffer.Len() == 0 {return nil}- if err := r.conn.Send(&TunnelMessage{+ if err := r.sendOutputLocked(r.buffer.Bytes()); err != nil {+ return err+ }+ r.buffer.Reset()+ return nil+}++// sendOutputLocked sends one output chunk, first waiting for manager credit when the window is+// full. The first chunk also carries the status and headers so the manager can stream the response.+func (r *commandResponseRecorder) sendOutputLocked(chunk []byte) error {+ for r.window > 0 && r.inFlight.Load() > 0 && r.inFlight.Load()+int64(len(chunk)) > r.window {+ select {+ case <-r.credited:+ case <-r.ctx.Done():+ return r.ctx.Err()+ }+ }+ r.inFlight.Add(int64(len(chunk)))++ msg := &TunnelMessage{ID: r.commandID,Type: MessageTypeCommandOutput,- Body: append([]byte(nil), r.buffer.Bytes()...),+ Body: append([]byte(nil), chunk...),Sequence: r.sequence,Command: r.commandName,- }); err != nil {+ }+ if r.sequence == 0 {+ msg.Status = r.statusCode+ msg.Headers = flattenResponseHeadersInternal(r.headers)+ }+ if err := r.conn.Send(msg); err != nil {return err}r.sequence++- r.buffer.Reset()return nil}@@ -1439,7 +1486,7 @@ func (r *streamingResponseRecorder) writeHeaderLocked(statusCode int) error {respHeaders[k] = v[0]}}- respHeaders["X-Arcane-Tunnel-Stream"] = "1"+ respHeaders[tunnelStreamHeader] = "1"if err := r.conn.Send(&TunnelMessage{ID: r.requestID,@@ -1498,7 +1545,7 @@ func (c *TunnelClient) connectAndServeWebSocketInternal(ctx context.Context) err}transport := &http.Transport{Proxy: http.ProxyFromEnvironment}if strings.HasPrefix(strings.ToLower(managerWSURL), "wss://") {- tlsConfig, err := buildManagerClientTLSConfigInternal(c.cfg)+ tlsConfig, err := buildManagerClientTLSConfig(c.cfg)if err != nil {return fmt.Errorf("failed to configure edge websocket TLS: %w", err)}diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.goindex 9da64aaf9..436b6039a 100644--- a/backend/pkg/libarcane/edge/client_test.go+++ b/backend/pkg/libarcane/edge/client_test.go@@ -2274,7 +2274,7 @@ func (f *fakeTunnelConn) Send(msg *TunnelMessage) error {}copyMsg := *msgif msg.Headers != nil {- copyMsg.Headers = cloneHeaderMap(msg.Headers)+ copyMsg.Headers = maps.Clone(msg.Headers)}if msg.Body != nil {copyMsg.Body = append([]byte(nil), msg.Body...)diff --git a/backend/pkg/libarcane/edge/client_transport_grpc.go b/backend/pkg/libarcane/edge/client_transport_grpc.goindex 6544c4ea5..5f8f7ea85 100644--- a/backend/pkg/libarcane/edge/client_transport_grpc.go+++ b/backend/pkg/libarcane/edge/client_transport_grpc.go@@ -50,7 +50,7 @@ func (c *TunnelClient) connectAndServeGRPC(ctx context.Context) error {}if c.useTLSForManagerGRPC() {- tlsConfig, err := buildManagerClientTLSConfigInternal(c.cfg)+ tlsConfig, err := buildManagerClientTLSConfig(c.cfg)if err != nil {return fmt.Errorf("failed to configure edge gRPC TLS: %w", err)}diff --git a/backend/pkg/libarcane/edge/command_client.go b/backend/pkg/libarcane/edge/command_client.goindex 298a1edc6..3b1ff9e12 100644--- a/backend/pkg/libarcane/edge/command_client.go+++ b/backend/pkg/libarcane/edge/command_client.go@@ -4,6 +4,8 @@ import ("context""errors""fmt"+ "log/slog"+ "math""net/http""slices""time"@@ -17,7 +19,13 @@ const (// manager->agent, cancel_request agent->manager) instead of the legacy// re-encoded forms.tunnelCapabilityProtoParity = "proto-parity-v1"- bodyTransferMetadataKey = "body_transfer_id"+ // Credit-based flow control for command output keeps at most commandCreditWindow bytes+ // unconsumed by the manager. Agents offer it; only the manager sends the grant, which older+ // managers that echo agent capabilities cannot produce.+ tunnelCapabilityCommandCredit = "command-credit-v1"+ tunnelCapabilityCommandCreditGrant = "command-credit-grant-v1"+ commandCreditWindow = 8 << 20+ bodyTransferMetadataKey = "body_transfer_id")func NewCommandClient() *CommandClient {@@ -101,8 +109,20 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C}}- status, headers, body, err := collectCommandResponseInternal(ctx, tunnel, pending, req.Method)+ // Output the collector has consumed is credited back, so the agent cannot outrun a slow reader.+ var credit func(int)+ if slices.Contains(tunnel.Capabilities, tunnelCapabilityCommandCredit) {+ credit = func(consumed int) {+ tunnel.grantCommandCredit(ctx, requestID, int64(consumed))+ }+ }++ status, headers, body, err := collectCommandResponseInternal(ctx, tunnel, pending, req.Method, req.Output, credit)if err != nil {+ if credit != nil {+ // Release an agent still waiting to send output that nobody will read.+ credit(math.MaxInt32)+ }return nil, err}@@ -156,6 +176,42 @@ func (c *CommandClient) OpenStream(ctx context.Context, tunnel *AgentTunnel, reqvar DefaultCommandClient = NewCommandClient()+// grantCommandCredit queues credit for a command without blocking. A per-tunnel sender coalesces+// queued credits and sends them, so a stalled connection never holds up the response reader.+func (t *AgentTunnel) grantCommandCredit(ctx context.Context, commandID string, consumed int64) {+ t.creditOnce.Do(func() {+ t.credits = make(map[string]int64)+ t.creditSignal = make(chan struct{}, 1)+ logCtx := context.WithoutCancel(ctx)+ go func() {+ for {+ select {+ case <-t.done:+ return+ case <-t.creditSignal:+ }+ t.creditMu.Lock()+ queued := t.credits+ t.credits = make(map[string]int64)+ t.creditMu.Unlock()+ for id, credit := range queued {+ if err := t.Conn.Send(&TunnelMessage{ID: id, Type: MessageTypeCommandCredit, Credit: credit}); err != nil {+ slog.DebugContext(logCtx, "Failed to send edge command credit", "id", id, "error", err)+ }+ }+ }+ }()+ })++ t.creditMu.Lock()+ t.credits[commandID] += consumed+ t.creditMu.Unlock()+ select {+ case t.creditSignal <- struct{}{}:+ default:+ }+}+func validateConnectedTunnelInternal(tunnel *AgentTunnel) error {if tunnel == nil || tunnel.Conn == nil || tunnel.Conn.IsClosed() {return errors.New("edge tunnel is not connected")diff --git a/backend/pkg/libarcane/edge/commands_test.go b/backend/pkg/libarcane/edge/commands_test.goindex 6d3dc5c2a..a1cda8158 100644--- a/backend/pkg/libarcane/edge/commands_test.go+++ b/backend/pkg/libarcane/edge/commands_test.go@@ -125,7 +125,7 @@ func TestCollectCommandResponse(t *testing.T) {pending.ResponseCh <- &TunnelMessage{ID: "cmd-1", Type: MessageTypeCommandComplete, Status: 200, Headers: map[string]string{"Content-Type": "text/plain"}, Body: []byte("world")}require.NoError(t, tunnel.CloseWithReason(""))- status, headers, body, err := collectCommandResponseInternal(t.Context(), tunnel, pending, "")+ status, headers, body, err := collectCommandResponseInternal(t.Context(), tunnel, pending, "", nil, nil)require.NoError(t, err)require.Equal(t, 200, status)require.Equal(t, "text/plain", headers["Content-Type"])diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.goindex 89c55e331..79e38de68 100644--- a/backend/pkg/libarcane/edge/poll_control.go+++ b/backend/pkg/libarcane/edge/poll_control.go@@ -237,7 +237,7 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {// In proxy-terminated mTLS deployments, the client certificate is consumed// by the TLS terminator before this request reaches Arcane. The token is// still needed as the poll protocol's environment lookup claim.- token, source := tokenFromHeadersWithSourceInternal(req)+ token, source := agentToken(req.Header.Values)if token == "" {slog.WarnContext(ctx, "Edge poll request without token")return c.JSON(http.StatusUnauthorized, map[string]any{"error": "agent token required"})@@ -259,9 +259,9 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {)return c.JSON(http.StatusUnauthorized, map[string]any{"error": "invalid agent token"})}- if requireRequestCertificateIdentityErr := s.requireRequestCertificateIdentityInternal(req, envID); requireRequestCertificateIdentityErr != nil {- slog.WarnContext(ctx, "Rejected edge poll request with mismatched client certificate", "environmentId", envID, "error", requireRequestCertificateIdentityErr)- return c.JSON(http.StatusUnauthorized, map[string]any{"error": requireRequestCertificateIdentityErr.Error()})+ if identityErr := s.requireCertificateIdentity(req.TLS, envID); identityErr != nil {+ slog.WarnContext(ctx, "Rejected edge poll request with mismatched client certificate", "environmentId", envID, "error", identityErr)+ return c.JSON(http.StatusUnauthorized, map[string]any{"error": identityErr.Error()})}pollInterval := DefaultTunnelPollIntervaldiff --git a/backend/pkg/libarcane/edge/poll_control_test.go b/backend/pkg/libarcane/edge/poll_control_test.goindex 93d2a58fa..45acc1542 100644--- a/backend/pkg/libarcane/edge/poll_control_test.go+++ b/backend/pkg/libarcane/edge/poll_control_test.go@@ -179,10 +179,10 @@ func TestTunnelServer_HandlePoll_AcceptsTokenAfterProxyTerminatedMTLS(t *testing}return "env-proxy-mtls", nil}, nil)- server.SetConfig(&Config{+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired,AppURL: "https://manager.example.com",- })+ }router := echo.New()router.POST("/api/tunnel/poll", server.HandlePoll)diff --git a/backend/pkg/libarcane/edge/proxy.go b/backend/pkg/libarcane/edge/proxy.goindex 0b88db0ef..5053a6bfc 100644--- a/backend/pkg/libarcane/edge/proxy.go+++ b/backend/pkg/libarcane/edge/proxy.go@@ -15,13 +15,17 @@ import ()const (- // DefaultProxyTimeout is the default timeout for proxied requests- DefaultProxyTimeout = 5 * time.Minute- // DefaultTunnelAcquirePollEvery is how frequently the manager checks for a- // newly activated edge tunnel while waiting for poll mode to connect.+ DefaultProxyTimeout = 5 * time.MinuteDefaultTunnelAcquirePollEvery = 100 * time.Millisecond+ tunnelStreamHeader = "X-Arcane-Tunnel-Stream")+// browserSecurityHeaders are browser-enforced headers that must not cross the edge tunnel.+var browserSecurityHeaders = map[string]struct{}{+ "Origin": {}, "Referer": {}, "Cookie": {}, "Access-Control-Request-Method": {},+ "Access-Control-Request-Headers": {}, "Sec-Fetch-Mode": {}, "Sec-Fetch-Site": {}, "Sec-Fetch-Dest": {},+}+// DefaultTunnelAcquireTimeout returns a poll-aware wait timeout for acquiring// an on-demand edge tunnel.func DefaultTunnelAcquireTimeout() time.Duration {@@ -58,20 +62,30 @@ func registerPendingRequestInternal(tunnel *AgentTunnel, requestID string) (*Penreturn pending, nil}-func collectCommandResponseInternal(ctx context.Context, tunnel *AgentTunnel, pending *PendingRequest, method string) (int, map[string]string, []byte, error) {- state := &grpcResponseState{}+// collectCommandResponseInternal waits for a command response. When out is set+// the response is written to it as it arrives and the returned body is nil.+// credit, when set, returns flow-control credit for each consumed output chunk.+func collectCommandResponseInternal(+ ctx context.Context,+ tunnel *AgentTunnel,+ pending *PendingRequest,+ method string,+ out http.ResponseWriter,+ credit func(consumed int),+) (int, map[string]string, []byte, error) {+ state := &grpcResponseState{out: out, credit: credit}for {select {case <-ctx.Done():return 0, nil, nil, ctx.Err()case <-tunnel.done:- if done, status, headers, body, err := state.drainTerminalResponseInternal(pending.ResponseCh, method); done {+ if done, status, headers, body, err := state.drainTerminalResponseInternal(ctx, pending, method); done {return status, headers, body, err}return 0, nil, nil, fmt.Errorf("edge tunnel closed while waiting for response: %w", ErrTunnelConnectionClosed)case err := <-pending.failureCh:- if done, status, headers, body, drainTerminalResponseErr := state.drainTerminalResponseInternal(pending.ResponseCh, method); done {+ if done, status, headers, body, drainTerminalResponseErr := state.drainTerminalResponseInternal(ctx, pending, method); done {return status, headers, body, drainTerminalResponseErr}return 0, nil, nil, err@@ -86,53 +100,71 @@ func collectCommandResponseInternal(ctx context.Context, tunnel *AgentTunnel, pe}}+// handleTunnelMessageInternal applies one response message. A failed write to the output+// ends the response with that error.func (s *grpcResponseState) handleTunnelMessageInternal(method string, incoming *TunnelMessage) (bool, int, map[string]string, []byte, error) {if incoming == nil {return false, 0, nil, nil, nil}+ var done bool+ var status int+ var headers map[string]string+ var body []byte+ var err errorswitch incoming.Type {case MessageTypeResponse:- done, status, headers, body := s.handleResponse(method, incoming)- return done, status, headers, body, nil+ done, status, headers, body = s.handleResponse(method, incoming)case MessageTypeCommandOutput, MessageTypeStreamData, MessageTypeFileChunk:s.handleStreamData(incoming)+ if incoming.Type == MessageTypeCommandOutput && s.credit != nil && len(incoming.Body) > 0 && s.outErr == nil {+ s.credit(len(incoming.Body))+ }case MessageTypeCommandComplete:- status, headers, body, err := s.handleCommandComplete(incoming)- return true, status, headers, body, err+ done = true+ status, headers, body, err = s.handleCommandComplete(incoming)case MessageTypeStreamEnd:- done, status, headers, body := s.handleStreamEnd()- return done, status, headers, body, nil- case MessageTypeRequest,- MessageTypeHeartbeat,- MessageTypeHeartbeatAck,- MessageTypeWebSocketStart,- MessageTypeWebSocketData,- MessageTypeWebSocketClose,- MessageTypeRegister,- MessageTypeRegisterResponse,- MessageTypeEvent,- MessageTypeCommandRequest,- MessageTypeCommandAck,- MessageTypeStreamOpen,- MessageTypeStreamClose,- MessageTypeCancelRequest:- }- return false, 0, nil, nil, nil+ done, status, headers, body = s.handleStreamEnd()+ }+ if s.outErr != nil {+ return true, s.status, nil, nil, fmt.Errorf("failed to write proxied response: %w", s.outErr)+ }+ return done, status, headers, body, err}-func (s *grpcResponseState) drainTerminalResponseInternal(respCh <-chan *TunnelMessage, method string) (bool, int, map[string]string, []byte, error) {+// drainTerminalResponseInternal applies messages already received for pending, including any still+// queued behind a full channel, until the response completes or nothing more is queued.+func (s *grpcResponseState) drainTerminalResponseInternal(ctx context.Context, pending *PendingRequest, method string) (bool, int, map[string]string, []byte, error) {for {+ if err := ctx.Err(); err != nil {+ return true, 0, nil, nil, err+ }+ // Checking idle before reading ensures every queued delivery is already in the channel.+ pending.mu.Lock()+ idle := pending.failed || (!pending.draining && len(pending.backlog) == 0)+ pending.mu.Unlock()++ var incoming *TunnelMessage+ ok := trueselect {- case incoming, ok := <-respCh:- if !ok {- return true, 0, nil, nil, fmt.Errorf("edge tunnel response channel closed before a response was received: %w", ErrTunnelConnectionClosed)+ case incoming, ok = <-pending.ResponseCh:+ default:+ if idle {+ return false, 0, nil, nil, nil}- if done, status, headers, body, err := s.handleTunnelMessageInternal(method, incoming); done {- return true, status, headers, body, err+ select {+ case incoming, ok = <-pending.ResponseCh:+ case <-ctx.Done():+ return true, 0, nil, nil, ctx.Err()+ case <-time.After(pendingDeliveryTimeout):+ return false, 0, nil, nil, nil}- default:- return false, 0, nil, nil, nil+ }+ if !ok {+ return true, 0, nil, nil, fmt.Errorf("edge tunnel response channel closed before a response was received: %w", ErrTunnelConnectionClosed)+ }+ if done, status, headers, body, err := s.handleTunnelMessageInternal(method, incoming); done {+ return true, status, headers, body, err}}}@@ -144,36 +176,40 @@ func (s *grpcResponseState) handleResponse(method string, incoming *TunnelMessags.respHeaders = incoming.Headers}- if s.respHeaders["X-Arcane-Tunnel-Stream"] == "1" {- if len(incoming.Body) > 0 {- s.respBody.Write(incoming.Body)- }+ if s.respHeaders[tunnelStreamHeader] == "1" {+ s.commitInternal()+ s.writeBodyInternal(incoming.Body)return false, 0, nil, nil}if len(incoming.Body) > 0 {- s.respBody.Write(incoming.Body)- return true, s.status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes()+ s.writeBodyInternal(incoming.Body)+ return s.finishInternal()}if method == http.MethodHead || s.status == http.StatusNoContent || s.status == http.StatusNotModified {- return true, s.status, stripInternalTunnelHeaders(s.respHeaders), nil+ return s.finishInternal()}return false, 0, nil, nil}func (s *grpcResponseState) handleStreamData(incoming *TunnelMessage) {- if len(incoming.Body) > 0 {- s.respBody.Write(incoming.Body)+ // Agents that support streaming put the status and headers on the first output chunk.+ if !s.gotResponse && incoming.Type == MessageTypeCommandOutput && incoming.Status != 0 {+ s.gotResponse = true+ s.status = incoming.Status+ s.respHeaders = incoming.Headers+ s.commitInternal()}+ s.writeBodyInternal(incoming.Body)}func (s *grpcResponseState) handleStreamEnd() (bool, int, map[string]string, []byte) {if !s.gotResponse {return false, 0, nil, nil}- return true, s.status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes()+ return s.finishInternal()}func (s *grpcResponseState) handleCommandComplete(incoming *TunnelMessage) (int, map[string]string, []byte, error) {@@ -182,13 +218,59 @@ func (s *grpcResponseState) handleCommandComplete(incoming *TunnelMessage) (int,s.status = incoming.Statuss.respHeaders = incoming.Headers}- if len(incoming.Body) > 0 {- s.respBody.Write(incoming.Body)- }+ s.writeBodyInternal(incoming.Body)if incoming.Error != "" && incoming.Status >= http.StatusBadRequest {return incoming.Status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes(), errors.New(incoming.Error)}- return incoming.Status, stripInternalTunnelHeaders(s.respHeaders), s.respBody.Bytes(), nil+ _, _, headers, body := s.finishInternal()+ return incoming.Status, headers, body, nil+}++// commitInternal writes the status, headers and any buffered body to out.+func (s *grpcResponseState) commitInternal() {+ if s.out == nil || s.committed {+ return+ }+ s.committed = true+ header := s.out.Header()+ for k, v := range s.respHeaders {+ if !isHopByHopHeader(k) && http.CanonicalHeaderKey(k) != tunnelStreamHeader {+ header.Set(k, v)+ }+ }+ s.out.WriteHeader(s.status)+ s.writeBodyInternal(s.respBody.Bytes())+ s.respBody.Reset()+}++// writeBodyInternal buffers body until the response is committed, then writes and flushes it.+// The first output error is kept and stops later writes.+func (s *grpcResponseState) writeBodyInternal(body []byte) {+ if !s.committed {+ s.respBody.Write(body)+ return+ }+ if s.outErr != nil {+ return+ }+ if len(body) > 0 {+ if _, s.outErr = s.out.Write(body); s.outErr != nil {+ return+ }+ }+ if flushErr := http.NewResponseController(s.out).Flush(); flushErr != nil && !errors.Is(flushErr, http.ErrNotSupported) {+ s.outErr = flushErr+ }+}++// finishInternal completes the response, committing it to out when set.+func (s *grpcResponseState) finishInternal() (bool, int, map[string]string, []byte) {+ headers := stripInternalTunnelHeaders(s.respHeaders)+ if s.out != nil {+ s.commitInternal()+ return true, s.status, headers, nil+ }+ return true, s.status, headers, s.respBody.Bytes()}// ProxyHTTPRequest is a helper that proxies an echo context through a tunnel@@ -227,13 +309,25 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) e"bodyLength", len(body),)- status, respHeaders, respBody, err := ProxyRequest(proxyCtx, tunnel, req.Method, targetPath, req.URL.RawQuery, headers, body)+ _, err := DefaultCommandClient.Execute(proxyCtx, tunnel, &CommandRequest{+ Method: req.Method,+ Path: targetPath,+ Query: req.URL.RawQuery,+ Headers: headers,+ Body: body,+ Output: c.Response(),+ })if err != nil {slog.ErrorContext(ctx, "Edge tunnel proxy failed","environmentId", tunnel.EnvironmentID,"error", err,)+ // Headers are already sent, so abort the connection rather than end a truncated+ // response as if it were complete. Echo's recover middleware re-panics this sentinel.+ if resp, unwrapErr := echo.UnwrapResponse(c.Response()); unwrapErr == nil && resp.Committed {+ panic(http.ErrAbortHandler)+ }if proxyCtx.Err() != nil {return c.JSON(http.StatusGatewayTimeout, map[string]any{"error": "request timed out"})}@@ -241,28 +335,13 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) ereturn c.JSON(http.StatusBadGateway, map[string]any{"error": "failed to proxy request through tunnel"})}- for k, v := range respHeaders {- if !isHopByHopHeader(k) {- c.Response().Header().Set(k, v)- }- }-- return c.Blob(status, respHeaders["Content-Type"], respBody)+ return nil}// isHopByHopHeader returns true if the header should not be forwardedfunc isHopByHopHeader(header string) bool {- hopByHop := map[string]bool{- "Connection": true,- "Keep-Alive": true,- "Proxy-Authenticate": true,- "Proxy-Authorization": true,- "Te": true,- "Trailers": true,- "Transfer-Encoding": true,- "Upgrade": true,- }- return hopByHop[http.CanonicalHeaderKey(header)]+ _, ok := hopByHopHeaders[http.CanonicalHeaderKey(header)]+ return ok}// isBrowserSecurityHeader returns true for headers that are browser-enforced@@ -271,17 +350,8 @@ func isHopByHopHeader(header string) bool {// its allowed origins, returning 403. The agent authenticates via// X-Arcane-Agent-Token instead of browser cookies/origin checks.func isBrowserSecurityHeader(header string) bool {- browserHeaders := map[string]bool{- "Origin": true,- "Referer": true,- "Cookie": true,- "Access-Control-Request-Method": true,- "Access-Control-Request-Headers": true,- "Sec-Fetch-Mode": true,- "Sec-Fetch-Site": true,- "Sec-Fetch-Dest": true,- }- return browserHeaders[http.CanonicalHeaderKey(header)]+ _, ok := browserSecurityHeaders[http.CanonicalHeaderKey(header)]+ return ok}func stripInternalTunnelHeaders(headers map[string]string) map[string]string {@@ -290,7 +360,7 @@ func stripInternalTunnelHeaders(headers map[string]string) map[string]string {}cleaned := make(map[string]string, len(headers))for k, v := range headers {- if http.CanonicalHeaderKey(k) == "X-Arcane-Tunnel-Stream" {+ if http.CanonicalHeaderKey(k) == tunnelStreamHeader {continue}cleaned[k] = vdiff --git a/backend/pkg/libarcane/edge/remenv.go b/backend/pkg/libarcane/edge/remenv.goindex 3d8f03057..81c4d6a76 100644--- a/backend/pkg/libarcane/edge/remenv.go+++ b/backend/pkg/libarcane/edge/remenv.go@@ -2,6 +2,7 @@ package edgeimport ("io"+ "maps""net/http""strings"@@ -12,20 +13,33 @@ import ()const (- HeaderAPIKey = "X-Api-Key" // #nosec G101: header name, not a credential- HeaderAuthorization = "Authorization"- HeaderCookie = "Cookie"- HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential- HeaderUpgrade = "Upgrade"- HeaderConnection = "Connection"-+ HeaderAPIKey = "X-Api-Key" // #nosec G101: header name, not a credential+ HeaderAuthorization = "Authorization"+ HeaderCookie = "Cookie"+ HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential+ HeaderUpgrade = "Upgrade"+ HeaderConnection = "Connection"ConnectionUpgradeToken = "upgrade")-func CopyRequestHeaders(from, to http.Header, skip map[string]struct{}) {+var (+ // hopByHopHeaders is shared and read-only; BuildHopByHopHeaders returns a private copy.+ hopByHopHeaders = map[string]struct{}{+ "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {}, "Proxy-Authorization": {},+ "Te": {}, "Trailers": {}, "Trailer": {}, "Transfer-Encoding": {}, "Upgrade": {},+ }+ skipHeaders = map[string]struct{}{+ "Host": {}, "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {},+ "Proxy-Authorization": {}, "Te": {}, "Trailer": {}, "Transfer-Encoding": {},+ "Upgrade": {}, "Content-Length": {}, "Accept-Encoding": {}, "Origin": {}, "Referer": {},+ "Access-Control-Request-Method": {}, "Access-Control-Request-Headers": {}, "Cookie": {},+ }+)++func CopyRequestHeaders(from, to http.Header) {for k, vs := range from {ck := http.CanonicalHeaderKey(k)- if _, ok := skip[ck]; ok || ck == http.CanonicalHeaderKey(HeaderAuthorization) || ck == http.CanonicalHeaderKey(HeaderAPIKey) {+ if _, ok := skipHeaders[ck]; ok || ck == http.CanonicalHeaderKey(HeaderAuthorization) || ck == http.CanonicalHeaderKey(HeaderAPIKey) {continue}for _, v := range vs {@@ -122,23 +136,8 @@ func SetForwardedHeaders(req *http.Request, clientIP, host string) {req.Header.Set("X-Forwarded-Host", host)}-func GetHopByHopHeaders() map[string]struct{} {- return map[string]struct{}{- http.CanonicalHeaderKey("Connection"): {},- http.CanonicalHeaderKey("Keep-Alive"): {},- http.CanonicalHeaderKey("Proxy-Authenticate"): {},- http.CanonicalHeaderKey("Proxy-Authorization"): {},- http.CanonicalHeaderKey("TE"): {},- http.CanonicalHeaderKey("Trailers"): {},- http.CanonicalHeaderKey("Trailer"): {},- http.CanonicalHeaderKey("Transfer-Encoding"): {},- http.CanonicalHeaderKey("Upgrade"): {},- }-}-func BuildHopByHopHeaders(respHeader http.Header) map[string]struct{} {- hop := GetHopByHopHeaders()-+ hop := maps.Clone(hopByHopHeaders)for _, connVal := range respHeader.Values("Connection") {for token := range strings.SplitSeq(connVal, ",") {if t := strings.TrimSpace(token); t != "" {@@ -161,12 +160,3 @@ func CopyResponseHeaders(from, to http.Header, hop map[string]struct{}) {}}}--func GetSkipHeaders() map[string]struct{} {- return map[string]struct{}{- "Host": {}, "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {},- "Proxy-Authorization": {}, "Te": {}, "Trailer": {}, "Transfer-Encoding": {},- "Upgrade": {}, "Content-Length": {}, "Accept-Encoding": {}, "Origin": {}, "Referer": {},- "Access-Control-Request-Method": {}, "Access-Control-Request-Headers": {}, "Cookie": {},- }-}diff --git a/backend/pkg/libarcane/edge/remenv_test.go b/backend/pkg/libarcane/edge/remenv_test.goindex 0a1f86ce5..21f44b8ce 100644--- a/backend/pkg/libarcane/edge/remenv_test.go+++ b/backend/pkg/libarcane/edge/remenv_test.go@@ -76,7 +76,6 @@ func (w *noFlushResponseWriter) Write(b []byte) (int, error) {func (w *noFlushResponseWriter) WriteHeader(statusCode int) { w.status = statusCode }func TestCopyRequestHeaders_SkipsExpectedHeaders(t *testing.T) {- skip := GetSkipHeaders()from := http.Header{}from.Add("X-Test", "a")from.Add("X-Test", "b")@@ -87,7 +86,7 @@ func TestCopyRequestHeaders_SkipsExpectedHeaders(t *testing.T) {from.Set("Transfer-Encoding", "chunked")to := http.Header{}- CopyRequestHeaders(from, to, skip)+ CopyRequestHeaders(from, to)require.Equal(t, []string{"a", "b"}, to.Values("X-Test"))require.Empty(t, to.Get(HeaderAuthorization))@@ -151,7 +150,7 @@ func TestSetForwardedHeaders(t *testing.T) {}func TestGetHopByHopHeaders_IncludesStandardHeaders(t *testing.T) {- h := GetHopByHopHeaders()+ h := BuildHopByHopHeaders(http.Header{})_, ok := h[http.CanonicalHeaderKey("Connection")]require.True(t, ok)_, ok = h[http.CanonicalHeaderKey("Transfer-Encoding")]@@ -189,7 +188,7 @@ func TestCopyResponseHeaders_SkipsHopByHopAndConnectionNamedHeaders(t *testing.T}func TestGetSkipHeaders_ContainsExpectedEntries(t *testing.T) {- skip := GetSkipHeaders()+ skip := skipHeadersrequire.Contains(t, skip, "Host")require.Contains(t, skip, "Connection")require.Contains(t, skip, "Transfer-Encoding")diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.goindex 6ae5b4e8f..f8f9469a4 100644--- a/backend/pkg/libarcane/edge/server.go+++ b/backend/pkg/libarcane/edge/server.go@@ -1,13 +1,17 @@package edgeimport (+ "cmp""context""crypto/tls""errors""fmt""io"+ "io/fs""log/slog""net/http"+ "os"+ "path/filepath""slices""strings""time"@@ -15,7 +19,7 @@ import (certgen "github.com/getarcaneapp/arcane/cli/v2/pkg/generate""github.com/labstack/echo/v5"- "github.com/samber/mo"+ "go.getarcane.app/acfs/atomic"kit "go.getarcane.app/kit/pkg""google.golang.org/grpc""google.golang.org/grpc/codes"@@ -27,19 +31,20 @@ import (wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws""github.com/getarcaneapp/arcane/backend/v2/pkg/remenv""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1")const (// TunnelStaleTimeout is how long before a tunnel is considered stale.TunnelStaleTimeout = 2 * time.Minute- // tunnelStaleSweepInterval is how often stale tunnels are reaped. The sweep- // is a backstop behind read-deadline liveness, so it runs well under- // TunnelStaleTimeout to keep the dead-but-selectable window short.+ // tunnelStaleSweepInterval backs up read-deadline liveness, so it runs well under TunnelStaleTimeout.tunnelStaleSweepInterval = time.Minute- // streamDeliveryTimeout bounds per-message delivery wait to a pending consumer.- // This prevents silent data loss while avoiding indefinite blocking.- streamDeliveryTimeout = 5 * time.Second+ // pendingDeliveryTimeout is how long one queued message may wait for a slow consumer.+ pendingDeliveryTimeout = 5 * time.Second+ // maxPendingBacklogBytes and maxPendingBacklogMessages bound how far a consumer may fall behind.+ maxPendingBacklogBytes = 32 << 20+ maxPendingBacklogMessages = 4096)// EnvironmentResolver resolves an agent token to an environment ID.@@ -55,11 +60,8 @@ type StatusUpdateCallback func(ctx context.Context, environmentID string, connec// EventCallback is called when an edge agent publishes an event.type EventCallback func(ctx context.Context, environmentID string, event *TunnelEvent) error-// EnrollmentCallback is called after a successful manager-side mTLS enrollment-// of an edge agent, allowing the host to record audit events or metrics.-// remoteAddr is the client socket address as seen by the manager.-// reenrolled is true when an environment that had already enrolled receives-// assets again after the enrollment cooldown.+// EnrollmentCallback is called after a successful manager-side edge mTLS enrollment.+// reenrolled is true when the environment had already enrolled before the cooldown.type EnrollmentCallback func(ctx context.Context, environmentID, remoteAddr string, certIssued, caGenerated, reenrolled bool)// NewTunnelServerWithRegistry creates a new tunnel server using an injected tunnel registry.@@ -76,46 +78,66 @@ func NewTunnelServerWithRegistry(registry *TunnelRegistry, resolver EnvironmentR}}-// SetEnvironmentNameResolver configures environment name lookup for manager-generated assets.-func (s *TunnelServer) SetEnvironmentNameResolver(resolver EnvironmentNameResolver) {- if s == nil {- return+// GRPCServerOptions returns the receive limit and stream interceptors for the tunnel service.+// It is served through ServeHTTP, so transport keepalive options belong on net/http.+func (s *TunnelServer) GRPCServerOptions() []grpc.ServerOption {+ recovery := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {+ defer func() {+ if panicErr := utils.PanicToError(recover()); panicErr != nil {+ slog.ErrorContext(ss.Context(), "panic in gRPC tunnel stream", "method", info.FullMethod, "error", panicErr)+ err = status.Error(codes.Internal, "internal tunnel error")+ }+ }()+ return handler(srv, ss)}- s.nameResolver = resolver-}-// GRPCServerOptions returns the receive guardrail and stream interceptor chain-// used by the tunnel service. Arcane serves this server through ServeHTTP, so-// transport-level server keepalive and MaxConcurrentStreams options would be-// inert here; those belong on net/http unless the server moves to grpc.Serve.-func (s *TunnelServer) GRPCServerOptions() []grpc.ServerOption {- return []grpc.ServerOption{- grpc.MaxRecvMsgSize(maxGRPCTunnelMessageSize),- grpc.ChainStreamInterceptor(- s.recoveryStreamInterceptorInternal(),- s.loggingStreamInterceptorInternal(),- s.authStreamInterceptorInternal(),- ),+ logging := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {+ start := time.Now()+ err := handler(srv, ss)+ duration := time.Since(start)+ switch {+ case err == nil:+ slog.DebugContext(ss.Context(), "gRPC stream completed", "method", info.FullMethod, "duration", duration)+ case isExpectedReceiveError(err):+ slog.DebugContext(ss.Context(), "gRPC stream closed", "method", info.FullMethod, "duration", duration, "error", err)+ default:+ slog.WarnContext(ss.Context(), "gRPC stream failed", "method", info.FullMethod, "duration", duration, "error", err)+ }+ return err}-}-// SetEventCallback configures the manager callback invoked for agent events.-func (s *TunnelServer) SetEventCallback(callback EventCallback) {- s.eventCallback = callback-}+ auth := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {+ // TunnelService only exposes Connect; keep this gate aligned with tunnel.proto.+ if info.FullMethod != tunnelpb.TunnelService_Connect_FullMethodName {+ return handler(srv, ss)+ }-// SetEnrollmentCallback configures the manager callback invoked after-// a successful edge mTLS enrollment. Passing nil clears the callback.-func (s *TunnelServer) SetEnrollmentCallback(callback EnrollmentCallback) {- if s == nil {- return+ ctx := ss.Context()+ md, _ := metadata.FromIncomingContext(ctx)+ token, _ := agentToken(md.Get)+ envID, err := s.resolveEnvironment(ctx, token)+ if err != nil {+ return status.Error(codes.Unauthenticated, "invalid agent token")+ }++ var state *tls.ConnectionState+ if p, ok := peer.FromContext(ctx); ok {+ if tlsInfo, isTLS := p.AuthInfo.(credentials.TLSInfo); isTLS {+ state = &tlsInfo.State+ }+ }+ if identityErr := s.requireCertificateIdentity(state, envID); identityErr != nil {+ return status.Error(codes.Unauthenticated, identityErr.Error())+ }++ identity := streamIdentity{environmentID: envID, securityMode: kit.Ternary(hasVerifiedPeerCertificate(state), "mtls", "token")}+ return handler(srv, &contextualServerStream{ServerStream: ss, ctx: context.WithValue(ctx, streamIdentityKey{}, identity)})}- s.enrollmentCallback = callback-}-// SetConfig attaches edge tunnel runtime config to the manager-side server.-func (s *TunnelServer) SetConfig(cfg *Config) {- s.cfg = cfg+ return []grpc.ServerOption{+ grpc.MaxRecvMsgSize(maxGRPCTunnelMessageSize),+ grpc.ChainStreamInterceptor(recovery, logging, auth),+ }}// HandleConnect is the WebSocket handler for edge agent connections.@@ -123,10 +145,8 @@ func (s *TunnelServer) SetConfig(cfg *Config) {func (s *TunnelServer) HandleConnect(c *echo.Context) error {req := c.Request()ctx := req.Context()- callbackCtx := context.WithoutCancel(ctx)- // Upgrade to WebSocket. Agents authenticate with tokens/mTLS, not browser- // cookies, so no Origin check is needed here.+ // Agents authenticate with tokens or mTLS rather than browser cookies, so no Origin check is needed.conn, err := wshub.Accept(c.Response(), req, nil)if err != nil {slog.ErrorContext(ctx, "Failed to upgrade edge tunnel connection", "error", err)@@ -134,67 +154,60 @@ func (s *TunnelServer) HandleConnect(c *echo.Context) error {}tunnelConn := NewTunnelConn(conn)- firstMsg, err := tunnelConn.Receive()- if err != nil {- slog.WarnContext(ctx, "Failed to receive websocket edge tunnel registration", "error", err)+ reject := func(reason string) error {+ if reason != "" {+ _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Error: reason})+ }_ = tunnelConn.Close()return nil}- if firstMsg == nil || firstMsg.Type != MessageTypeRegister {++ registerMsg, err := tunnelConn.Receive()+ if err != nil {+ slog.WarnContext(ctx, "Failed to receive websocket edge tunnel registration", "error", err)+ return reject("")+ }+ if registerMsg == nil || registerMsg.Type != MessageTypeRegister {slog.WarnContext(ctx, "Websocket edge tunnel missing register message")- _ = tunnelConn.Close()- return nil+ return reject("")}- token := tokenFromHeadersInternal(req)- if token == "" {- // Header auth can be unavailable after the WebSocket upgrade path, so- // the register message remains a fallback environment lookup claim.- // In proxy-terminated mTLS deployments, the client certificate is the- // auth credential and is consumed before the request reaches Arcane.- token = strings.TrimSpace(firstMsg.AgentToken)- }+ // Headers can be lost on the upgrade path, and proxy-terminated mTLS consumes the client+ // certificate, so the register message's token is the fallback environment claim.+ token, _ := agentToken(req.Header.Values)+ token = cmp.Or(token, strings.TrimSpace(registerMsg.AgentToken))if token == "" {slog.WarnContext(ctx, "Edge tunnel connection attempt without token")- _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Accepted: false, Error: "agent token required"})- _ = tunnelConn.Close()- return nil+ return reject("agent token required")}envID, err := s.resolveEnvironment(ctx, token)if err != nil {slog.WarnContext(ctx, "Failed to resolve agent token", "error", err)- _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Accepted: false, Error: "invalid agent token"})- _ = tunnelConn.Close()- return nil+ return reject("invalid agent token")}- if requireRequestCertificateIdentityErr := s.requireRequestCertificateIdentityInternal(req, envID); requireRequestCertificateIdentityErr != nil {- slog.WarnContext(ctx, "Rejected websocket edge tunnel with mismatched client certificate", "environmentId", envID, "error", requireRequestCertificateIdentityErr)- _ = tunnelConn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, Accepted: false, Error: "client certificate does not match environment"})- _ = tunnelConn.Close()- return nil+ if identityErr := s.requireCertificateIdentity(req.TLS, envID); identityErr != nil {+ slog.WarnContext(ctx, "Rejected websocket edge tunnel with mismatched client certificate", "environmentId", envID, "error", identityErr)+ return reject("client certificate does not match environment")}- tunnel := NewAgentTunnelWithConn(envID, tunnelConn)- s.populateSessionMetadata(tunnel, firstMsg, requestSecurityModeInternal(req))- s.manageConnectedTunnel(ctx, callbackCtx, tunnel)+ s.manageConnectedTunnel(ctx, envID, tunnelConn, registerMsg, kit.Ternary(hasVerifiedPeerCertificate(req.TLS), "mtls", "token"))return nil}-// HandleMTLSEnroll returns manager-generated edge client certificates for the-// calling environment when generated edge mTLS is enabled. The response includes-// private key material, so response-body logging must not be enabled here.+// HandleMTLSEnroll returns manager-generated edge client certificates for the calling environment.+// The response contains private keys, so response-body logging must stay disabled here.func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {req := c.Request()ctx := req.Context()c.Response().Header().Set("Cache-Control", "no-store")c.Response().Header().Set("Pragma", "no-cache")- if s == nil || s.cfg == nil || !shouldUseGeneratedManagerCAInternal(s.cfg) {+ if !usesGeneratedManagerCA(s.Config) {return c.JSON(http.StatusNotFound, map[string]any{"error": "edge mTLS enrollment is not available"})}- token := tokenFromHeadersInternal(req)+ token, _ := agentToken(req.Header.Values)if token == "" {return c.JSON(http.StatusUnauthorized, map[string]any{"error": "agent token required"})}@@ -206,8 +219,8 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {}envName := ""- if s.nameResolver != nil {- resolvedName, resolveErr := s.nameResolver(ctx, envID)+ if s.NameResolver != nil {+ resolvedName, resolveErr := s.NameResolver(ctx, envID)if resolveErr != nil {slog.WarnContext(ctx, "Failed to resolve environment name for edge mTLS enrollment", "environmentId", envID, "error", resolveErr)} else {@@ -215,14 +228,37 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {}}- now := time.Now()- previouslyEnrolled, enrollmentLimited, err := managerMTLSEnrollmentStateInternal(s.cfg, envID, now)- if err != nil {- slog.ErrorContext(ctx, "Failed to read edge mTLS enrollment state", "environmentId", envID, "error", err)+ // The marker holds the last enrollment time; within the cooldown the existing assets are re-served.+ failState := func(stateErr error) error {+ slog.ErrorContext(ctx, "Failed to read edge mTLS enrollment state", "environmentId", envID, "error", stateErr)return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to read edge mTLS enrollment state"})}+ assetsDir, err := edgeMTLSAssetsDir(s.Config, managerMTLSDirName)+ if err != nil {+ return failState(err)+ }+ safeEnvID := sanitizedEnvID(envID)+ if safeEnvID == "" {+ return failState(errors.New("environment ID is required"))+ }+ markerPath := filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID, generatedMTLSEnrolledName)+ marker, readErr := os.ReadFile(markerPath)+ if readErr != nil && !errors.Is(readErr, fs.ErrNotExist) {+ return failState(readErr)+ }+ now := time.Now()+ previouslyEnrolled := readErr == nil+ enrollmentLimited := false+ if trimmed := strings.TrimSpace(string(marker)); trimmed != "" {+ enrolledAt, parseErr := time.Parse(time.RFC3339Nano, trimmed)+ if parseErr != nil {+ return failState(fmt.Errorf("failed to parse edge mTLS enrollment marker %s: %w", markerPath, parseErr))+ }+ enrollmentLimited = now.Sub(enrolledAt) < managerMTLSReenrollCooldown+ }+if enrollmentLimited {- cachedAssets, cacheErr := GenerateManagerClientMTLSAssetsWithContext(ctx, s.cfg, envID, envName)+ cachedAssets, cacheErr := GenerateManagerClientMTLSAssetsWithContext(ctx, s.Config, envID, envName)if cacheErr == nil && cachedAssets != nil {slog.InfoContext(ctx, "Served cached edge mTLS enrollment during cooldown", "environmentId", envID, "remoteAddr", c.RealIP())return c.JSON(http.StatusOK, enrollMTLSResponse{Files: cachedAssets.Files})@@ -234,7 +270,7 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {return c.JSON(http.StatusTooManyRequests, map[string]any{"error": "edge mTLS enrollment was recently completed; retry later"})}- assets, err := GenerateManagerClientMTLSAssetsWithContext(ctx, s.cfg, envID, envName)+ assets, err := GenerateManagerClientMTLSAssetsWithContext(ctx, s.Config, envID, envName)if err != nil {slog.ErrorContext(ctx, "Failed to generate edge mTLS enrollment assets", "environmentId", envID, "error", err)return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to generate edge mTLS assets"})@@ -243,8 +279,8 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {return c.JSON(http.StatusNotFound, map[string]any{"error": "edge mTLS enrollment assets unavailable"})}assets.Reenrolled = previouslyEnrolled- if recordManagerMTLSEnrollmentErr := recordManagerMTLSEnrollmentInternal(s.cfg, envID, now); recordManagerMTLSEnrollmentErr != nil {- slog.ErrorContext(ctx, "Failed to record edge mTLS enrollment state", "environmentId", envID, "error", recordManagerMTLSEnrollmentErr)+ if recordErr := atomic.WriteFile(markerPath, []byte(now.UTC().Format(time.RFC3339Nano)+"\n"), 0o600); recordErr != nil {+ slog.ErrorContext(ctx, "Failed to record edge mTLS enrollment state", "environmentId", envID, "error", recordErr)return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to record edge mTLS enrollment state"})}if assets.Reenrolled {@@ -253,8 +289,8 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {slog.InfoContext(ctx, "Edge mTLS certificate assets enrolled", "environmentId", envID, "remoteAddr", c.RealIP(), "certIssued", assets.CertIssued)}- if s.enrollmentCallback != nil {- s.enrollmentCallback(context.WithoutCancel(ctx), envID, c.RealIP(), assets.CertIssued, assets.CAGenerated, assets.Reenrolled)+ if s.EnrollmentCallback != nil {+ s.EnrollmentCallback(context.WithoutCancel(ctx), envID, c.RealIP(), assets.CertIssued, assets.CAGenerated, assets.Reenrolled)}return c.JSON(http.StatusOK, enrollMTLSResponse{Files: assets.Files})@@ -263,7 +299,6 @@ func (s *TunnelServer) HandleMTLSEnroll(c *echo.Context) error {// Connect is the gRPC bidi stream handler for edge agent connections.func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMessage, tunnelpb.ManagerMessage]) error {ctx := stream.Context()- callbackCtx := context.WithoutCancel(ctx)firstMsg, err := stream.Recv()if err != nil {@@ -273,51 +308,23 @@ func (s *TunnelServer) Connect(stream grpc.BidiStreamingServer[tunnelpb.AgentMesreturn err}- register := firstMsg.GetRegister()- if register == nil {+ registerMsg, err := agentProtoToTunnelMessage(firstMsg)+ if err != nil || registerMsg.Type != MessageTypeRegister {return status.Error(codes.Unauthenticated, "first message must be register")}- envID, ok := resolvedEnvironmentIDFromContextInternal(ctx).Get()- if !ok || envID == "" {+ // The auth interceptor already verified the token and client certificate for this stream.+ identity, _ := ctx.Value(streamIdentityKey{}).(streamIdentity)+ if strings.TrimSpace(identity.environmentID) == "" {return status.Error(codes.Unauthenticated, "authenticated environment is missing from stream context")}- if requireCertificateIdentityFromContextErr := s.requireCertificateIdentityFromContextInternal(ctx, envID); requireCertificateIdentityFromContextErr != nil {- return status.Error(codes.Unauthenticated, requireCertificateIdentityFromContextErr.Error())- }managerConn := NewGRPCManagerTunnelConn(stream)- managerConn.SetParityEncoding(slices.Contains(register.GetCapabilities(), tunnelCapabilityProtoParity))- tunnel := NewAgentTunnelWithConn(envID, managerConn)- s.populateSessionMetadata(tunnel, &TunnelMessage{- AgentInstance: register.GetAgentInstanceId(),- Capabilities: append([]string(nil), register.GetCapabilities()...),- ResumeSession: register.GetResumeSessionId(),- }, securityModeFromGRPCContextInternal(ctx))- s.manageConnectedTunnel(ctx, callbackCtx, tunnel)+ managerConn.parity = slices.Contains(registerMsg.Capabilities, tunnelCapabilityProtoParity)+ s.manageConnectedTunnel(ctx, identity.environmentID, managerConn, registerMsg, identity.securityMode)return nil}-func (s *TunnelServer) populateSessionMetadata(tunnel *AgentTunnel, registerMsg *TunnelMessage, securityMode string) {- if tunnel == nil {- return- }-- tunnel.SessionID = uuid.New().String()- tunnel.SecurityMode = securityMode- if tunnel.SecurityMode == "" {- tunnel.SecurityMode = "token"- }- if registerMsg != nil {- tunnel.AgentInstance = strings.TrimSpace(registerMsg.AgentInstance)- tunnel.Capabilities = append([]string(nil), registerMsg.Capabilities...)- }-- if tunnel.Conn != nil {- tunnel.Transport = tunnel.Conn.Transport()- }-}-func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (string, error) {if s.resolver == nil {return "", errors.New("edge resolver is not configured")@@ -329,257 +336,218 @@ func (s *TunnelServer) resolveEnvironment(ctx context.Context, token string) (streturn s.resolver(ctx, token)}-// agentTokenHeaderPriority is the shared credential fallback order for both-// header- and metadata-based token extraction, so the transports cannot drift.-var agentTokenHeaderPriority = []string{HeaderAgentToken, HeaderAPIKey}--func tokenFromMetadataInternal(ctx context.Context) string {- md, ok := metadata.FromIncomingContext(ctx)- if !ok {- return ""- }- for _, header := range agentTokenHeaderPriority {- values := md.Get(strings.ToLower(header))- for _, value := range values {- trimmed := strings.TrimSpace(value)- if trimmed != "" {- return trimmed- }+// agentToken returns the first agent token from X-Arcane-Agent-Token, X-API-Key, then+// "Authorization: Bearer", for proxies that strip custom headers, plus the header that supplied it.+func agentToken(values func(key string) []string) (string, string) {+ for _, header := range []string{HeaderAgentToken, HeaderAPIKey} {+ if tokens := kit.TrimNonEmpty(values(header)); len(tokens) > 0 {+ return tokens[0], header}}- for _, value := range md.Get(strings.ToLower(HeaderAuthorization)) {+ for _, value := range values(HeaderAuthorization) {if token := remenv.ExtractBearerToken(value); token != "" {- return token- }- }- return ""-}--func tokenFromHeadersInternal(req *http.Request) string {- token, _ := tokenFromHeadersWithSourceInternal(req)- return token-}--// tokenFromHeadersWithSourceInternal returns the agent token along with the-// canonical header name that supplied it. Falls back from X-Arcane-Agent-Token-// to X-API-Key to "Authorization: Bearer <token>" so deployments behind-// reverse proxies that strip custom X- headers can still authenticate.-func tokenFromHeadersWithSourceInternal(req *http.Request) (string, string) {- if req == nil {- return "", ""- }- for _, header := range agentTokenHeaderPriority {- if token := strings.TrimSpace(req.Header.Get(header)); token != "" {- return token, header+ return token, HeaderAuthorization}}- if token := remenv.ExtractBearerToken(req.Header.Get(HeaderAuthorization)); token != "" {- return token, HeaderAuthorization- }return "", ""}-func (s *TunnelServer) manageConnectedTunnel(ctx, callbackCtx context.Context, tunnel *AgentTunnel) {+// manageConnectedTunnel registers an authenticated agent session and serves its messages until it disconnects.+func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, envID string, conn TunnelConnection, registerMsg *TunnelMessage, securityMode string) {+ callbackCtx := context.WithoutCancel(ctx)+ tunnel := NewAgentTunnelWithConn(envID, conn)+ tunnel.SessionID = uuid.New().String()+ tunnel.SecurityMode = securityMode+ tunnel.AgentInstance = strings.TrimSpace(registerMsg.AgentInstance)+ tunnel.Capabilities = registerMsg.Capabilities+ tunnel.Transport = conn.Transport()+accepted, drainPrevious, rejectReason, err := s.registry.RegisterSession(callbackCtx, tunnel, TunnelStaleTimeout)if err != nil {- slog.ErrorContext(ctx, "Failed to register edge agent session",- "environmentId", tunnel.EnvironmentID,- "agentInstanceId", tunnel.AgentInstance,- "error", err,- )+ slog.ErrorContext(ctx, "Failed to register edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "error", err)_ = tunnel.CloseWithReason("edge agent session registration unavailable")return}if !accepted {- slog.WarnContext(ctx, "Rejected duplicate edge agent session",- "environmentId", tunnel.EnvironmentID,- "agentInstanceId", tunnel.AgentInstance,- "reason", rejectReason,- )- _ = tunnel.Conn.Send(&TunnelMessage{- Type: MessageTypeRegisterResponse,- Accepted: false,- EnvironmentID: tunnel.EnvironmentID,- Error: rejectReason,- })+ slog.WarnContext(ctx, "Rejected duplicate edge agent session", "environmentId", envID, "agentInstanceId", tunnel.AgentInstance, "reason", rejectReason)+ _ = conn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse, EnvironmentID: envID, Error: rejectReason})_ = tunnel.CloseWithReason(rejectReason)return}- slog.InfoContext(ctx, "Edge agent connected",- "environmentId", tunnel.EnvironmentID,- "sessionId", tunnel.SessionID,- "securityMode", tunnel.SecurityMode,- )+ defer func() {+ removed, active := s.registry.UnregisterCurrent(callbackCtx, envID, tunnel)+ if !removed {+ return+ }+ slog.InfoContext(ctx, "Edge agent disconnected", "environmentId", envID, "sessionId", tunnel.SessionID)+ if !active {+ s.updateConnectionStatus(callbackCtx, tunnel, false)+ }+ }()- // Echo the agent's capabilities and append the manager's own so the agent- // learns what this manager supports (older agents ignore extra strings).- capabilities := kit.Unique(append(slices.Clone(tunnel.Capabilities), tunnelCapabilityProtoParity, tunnelCapabilityChunkedRequest))+ slog.InfoContext(ctx, "Edge agent connected", "environmentId", envID, "sessionId", tunnel.SessionID, "securityMode", tunnel.SecurityMode)- if sendErr := tunnel.Conn.Send(&TunnelMessage{+ // Echo the agent's capabilities plus the manager's own; older agents ignore extra strings.+ capabilities := kit.Unique(append(slices.Clone(tunnel.Capabilities), tunnelCapabilityProtoParity, tunnelCapabilityChunkedRequest, tunnelCapabilityCommandCreditGrant))+ if sendErr := conn.Send(&TunnelMessage{Type: MessageTypeRegisterResponse,Accepted: true,- EnvironmentID: tunnel.EnvironmentID,+ EnvironmentID: envID,SessionID: tunnel.SessionID,SecurityMode: tunnel.SecurityMode,Capabilities: capabilities,DrainPrevious: drainPrevious,}); sendErr != nil {- slog.WarnContext(ctx, "Failed to send register response", "environmentId", tunnel.EnvironmentID, "error", sendErr)+ slog.WarnContext(ctx, "Failed to send register response", "environmentId", envID, "error", sendErr)_ = tunnel.CloseWithReason("")- removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)- if removed && !active {- s.updateConnectionStatusInternal(callbackCtx, tunnel, false)- }return}+ s.updateConnectionStatus(callbackCtx, tunnel, true)- s.updateConnectionStatusInternal(callbackCtx, tunnel, true)-- defer func() {- removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)- if !removed {+ deliver := func(msg *TunnelMessage, closeStream bool) {+ value, _ := tunnel.Pending.Load(msg.ID)+ if pending, ok := value.(*PendingRequest); ok {+ pending.enqueue(ctx, tunnel, msg, closeStream)return}- slog.InfoContext(ctx, "Edge agent disconnected", "environmentId", tunnel.EnvironmentID, "sessionId", tunnel.SessionID)- if !active {- s.updateConnectionStatusInternal(callbackCtx, tunnel, false)- }- }()-- s.messageLoop(ctx, tunnel)-}--// messageLoop processes incoming messages from the agent.-func (s *TunnelServer) messageLoop(ctx context.Context, tunnel *AgentTunnel) {- // One reusable timer for stream delivery deadlines. time.After per message- // left a live 5s runtime timer for every frame, so a chatty log tail through- // a tunnel accumulated thousands of them. This loop is the only user, so the- // timer is never touched concurrently.- deliveryTimer := time.NewTimer(streamDeliveryTimeout)- deliveryTimer.Stop()- defer deliveryTimer.Stop()+ slog.DebugContext(ctx, "Received message for unknown request", "id", msg.ID, "type", msg.Type)+ }- for {- select {- case <-ctx.Done():- return- default:- msg, err := tunnel.Conn.Receive()- if err != nil {- if !tunnel.Conn.IsExpectedReceiveError(err) {- slog.WarnContext(ctx, "Error receiving from edge tunnel", "environmentId", tunnel.EnvironmentID, "error", err)- }- return+ for ctx.Err() == nil {+ msg, receiveErr := conn.Receive()+ if receiveErr != nil {+ if !conn.IsExpectedReceiveError(receiveErr) {+ slog.WarnContext(ctx, "Error receiving from edge tunnel", "environmentId", envID, "error", receiveErr)}-- s.handleTunnelMessage(ctx, tunnel, msg, deliveryTimer)+ return}- }-}-func (s *TunnelServer) handleTunnelMessage(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, deliveryTimer *time.Timer) {- switch msg.Type {- case MessageTypeHeartbeat:- s.handleHeartbeat(ctx, tunnel, msg)- case MessageTypeResponse:- s.deliverResponse(ctx, tunnel, msg)- case MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete, MessageTypeFileChunk:- s.deliverResponse(ctx, tunnel, msg)- case MessageTypeEvent:- s.handleEvent(ctx, tunnel, msg)- case MessageTypeStreamData, MessageTypeStreamEnd, MessageTypeWebSocketData, MessageTypeWebSocketClose, MessageTypeStreamClose:- s.deliverStream(ctx, tunnel, msg, deliveryTimer)- case MessageTypeRequest, MessageTypeHeartbeatAck, MessageTypeWebSocketStart, MessageTypeRegisterResponse, MessageTypeCommandRequest, MessageTypeStreamOpen, MessageTypeCancelRequest:- slog.DebugContext(ctx, "Ignoring message type from agent", "type", msg.Type, "environmentId", tunnel.EnvironmentID)- case MessageTypeRegister:- slog.DebugContext(ctx, "Ignoring duplicate register message from agent", "environmentId", tunnel.EnvironmentID)- default:- slog.WarnContext(ctx, "Unknown message type from agent", "type", msg.Type, "environmentId", tunnel.EnvironmentID)+ switch msg.Type {+ case MessageTypeHeartbeat:+ tunnel.UpdateHeartbeat()+ if ackErr := conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeHeartbeatAck}); ackErr != nil {+ slog.WarnContext(ctx, "Failed to send heartbeat ack", "error", ackErr)+ }+ case MessageTypeResponse, MessageTypeCommandAck, MessageTypeCommandOutput, MessageTypeCommandComplete, MessageTypeFileChunk:+ deliver(msg, false)+ case MessageTypeStreamData, MessageTypeStreamEnd, MessageTypeWebSocketData, MessageTypeWebSocketClose, MessageTypeStreamClose:+ deliver(msg, true)+ case MessageTypeEvent:+ if msg.Event == nil {+ slog.WarnContext(ctx, "Received event message without payload", "environmentId", envID)+ continue+ }+ if s.EventCallback == nil {+ continue+ }+ event := cloneTunnelEvent(msg.Event)+ go func() {+ eventCtx, cancel := context.WithTimeout(callbackCtx, 15*time.Second)+ defer cancel()+ if eventErr := s.EventCallback(eventCtx, envID, event); eventErr != nil {+ slog.WarnContext(eventCtx, "Failed to process edge event", "environmentId", envID, "type", event.Type, "error", eventErr)+ }+ }()+ case MessageTypeRequest, MessageTypeHeartbeatAck, MessageTypeWebSocketStart, MessageTypeRegisterResponse,+ MessageTypeCommandRequest, MessageTypeStreamOpen, MessageTypeCancelRequest, MessageTypeRegister, MessageTypeCommandCredit:+ slog.DebugContext(ctx, "Ignoring message type from agent", "type", msg.Type, "environmentId", envID)+ default:+ slog.WarnContext(ctx, "Unknown message type from agent", "type", msg.Type, "environmentId", envID)+ }}}-func (s *TunnelServer) handleHeartbeat(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage) {- tunnel.UpdateHeartbeat()- ack := &TunnelMessage{- ID: msg.ID,- Type: MessageTypeHeartbeatAck,- }- if err := tunnel.Conn.Send(ack); err != nil {- slog.WarnContext(ctx, "Failed to send heartbeat ack", "error", err)+// enqueue hands msg to the consumer without blocking the tunnel. Behind a full channel, messages+// wait in order for up to pendingDeliveryTimeout each. A consumer that stalls or falls past the+// backlog limits is failed once, and a stream also tells the agent to stop. Queued messages+// outlive a tunnel close so the collector can still drain a fully received response.+func (p *PendingRequest) enqueue(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, closeStream bool) {+ retained := func(m *TunnelMessage) int {+ size := len(m.Body) + len(m.Error)+ for k, v := range m.Headers {+ size += len(k) + len(v)+ }+ for k, v := range m.Metadata {+ size += len(k) + len(v)+ }+ return size}-}-func (s *TunnelServer) deliverResponse(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage) {- if req, ok := tunnel.Pending.Load(msg.ID); ok {- pending, isPending := req.(*PendingRequest)- if !isPending {+ fail := func(stalled error) {+ // A consumer that already finished has nothing left to fail.+ if !tunnel.Pending.CompareAndDelete(msg.ID, p) {return}select {- case pending.ResponseCh <- msg:+ case p.failureCh <- stalled:default:- err := fmt.Errorf("response delivery failed because pending request %s is not consuming messages", msg.ID)- tunnel.Pending.Delete(msg.ID)- pending.failureCh <- err- slog.WarnContext(ctx, "Failed pending request with full response channel", "id", msg.ID)}- return- }- slog.WarnContext(ctx, "Received response for unknown request", "id", msg.ID)-}--func (s *TunnelServer) deliverStream(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage, deliveryTimer *time.Timer) {- if req, ok := tunnel.Pending.Load(msg.ID); ok {- pending, isPending := req.(*PendingRequest)- if !isPending {+ slog.WarnContext(ctx, "Failed slow pending request consumer", "id", msg.ID, "type", msg.Type, "error", stalled)+ if !closeStream {return}+ if closeErr := tunnel.Conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeStreamClose, Error: stalled.Error()}); closeErr != nil {+ slog.DebugContext(ctx, "Failed to close slow edge stream", "id", msg.ID, "error", closeErr)+ }+ }- // Go 1.23+ timers drop any stale value on Stop/Reset, so no drain is needed.- deliveryTimer.Stop()- deliveryTimer.Reset(streamDeliveryTimeout)- defer deliveryTimer.Stop()-+ p.mu.Lock()+ if p.failed {+ p.mu.Unlock()+ return+ }+ if !p.draining {select {- case pending.ResponseCh <- msg:- case <-ctx.Done():+ case p.ResponseCh <- msg:+ p.mu.Unlock()return- case <-deliveryTimer.C:- err := fmt.Errorf("stream delivery timed out for pending request %s", msg.ID)- tunnel.Pending.Delete(msg.ID)- pending.failureCh <- err- slog.WarnContext(ctx, "Failed slow pending stream consumer",- "id", msg.ID,- "type", msg.Type,- "timeout", streamDeliveryTimeout,- )- if sendErr := tunnel.Conn.Send(&TunnelMessage{ID: msg.ID, Type: MessageTypeStreamClose, Error: err.Error()}); sendErr != nil {- slog.DebugContext(ctx, "Failed to close slow edge stream", "id", msg.ID, "error", sendErr)- }+ default:}- return+ p.draining = true+ go func() {+ timer := time.NewTimer(pendingDeliveryTimeout)+ defer timer.Stop()+ for {+ p.mu.Lock()+ if p.failed || len(p.backlog) == 0 {+ p.draining = false+ p.mu.Unlock()+ return+ }+ next := p.backlog[0]+ p.backlog[0] = nil+ p.backlog = p.backlog[1:]+ p.backlogBytes -= retained(next)+ p.mu.Unlock()++ timer.Reset(pendingDeliveryTimeout)+ select {+ case p.ResponseCh <- next:+ case <-timer.C:+ p.mu.Lock()+ stalled := !p.failed+ p.failed, p.backlog = true, nil+ p.mu.Unlock()+ if stalled {+ fail(fmt.Errorf("consumer for pending request %s stalled for %s", next.ID, pendingDeliveryTimeout))+ }+ return+ }+ }+ }()}- slog.DebugContext(ctx, "Received stream message for unknown request", "id", msg.ID, "type", msg.Type)-}-func (s *TunnelServer) handleEvent(ctx context.Context, tunnel *AgentTunnel, msg *TunnelMessage) {- if msg.Event == nil {- slog.WarnContext(ctx, "Received event message without payload", "environmentId", tunnel.EnvironmentID)- return+ p.backlog = append(p.backlog, msg)+ p.backlogBytes += retained(msg)+ overflow := p.backlogBytes > maxPendingBacklogBytes || len(p.backlog) > maxPendingBacklogMessages+ if overflow {+ p.failed, p.backlog = true, nil}- if s.eventCallback == nil {- return+ p.mu.Unlock()+ if overflow {+ fail(fmt.Errorf("consumer for pending request %s fell more than %d messages or %d bytes behind", msg.ID, maxPendingBacklogMessages, maxPendingBacklogBytes))}-- eventCopy := cloneTunnelEvent(msg.Event)- go func() {- eventCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)- defer cancel()- if err := s.eventCallback(eventCtx, tunnel.EnvironmentID, eventCopy); err != nil {- slog.WarnContext(eventCtx, "Failed to process edge event", "environmentId", tunnel.EnvironmentID, "type", eventCopy.Type, "error", err)- }- }()}// StartCleanupLoop periodically cleans up stale tunnels.@@ -595,7 +563,7 @@ func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {case <-ticker.C:removed := s.registry.CleanupStale(ctx, TunnelStaleTimeout)for _, tunnel := range removed {- s.updateConnectionStatusInternal(context.WithoutCancel(ctx), tunnel, false)+ s.updateConnectionStatus(context.WithoutCancel(ctx), tunnel, false)}if len(removed) > 0 {slog.InfoContext(ctx, "Cleaned up stale tunnels", "count", len(removed))@@ -604,8 +572,10 @@ func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {}}-func (s *TunnelServer) updateConnectionStatusInternal(ctx context.Context, tunnel *AgentTunnel, connected bool) {- if s == nil || s.statusCallback == nil || tunnel == nil {+// updateConnectionStatus reports a connect or disconnect, skipping a disconnect+// when a newer live tunnel already serves the environment.+func (s *TunnelServer) updateConnectionStatus(ctx context.Context, tunnel *AgentTunnel, connected bool) {+ if s.statusCallback == nil {return}@@ -624,135 +594,40 @@ func (s *TunnelServer) WaitForCleanupDone() {<-s.cleanupDone}-func (s *TunnelServer) authStreamInterceptorInternal() grpc.StreamServerInterceptor {- // TunnelService currently exposes only Connect. Keep this explicit method- // gate aligned with tunnel.proto if new RPCs are added.- return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {- if info == nil || info.FullMethod != tunnelpb.TunnelService_Connect_FullMethodName {- return handler(srv, ss)- }-- token := tokenFromMetadataInternal(ss.Context())- envID, err := s.resolveEnvironment(ss.Context(), token)- if err != nil {- return status.Error(codes.Unauthenticated, "invalid agent token")- }- if requireCertificateIdentityFromContextErr := s.requireCertificateIdentityFromContextInternal(ss.Context(), envID); requireCertificateIdentityFromContextErr != nil {- return status.Error(codes.Unauthenticated, requireCertificateIdentityFromContextErr.Error())- }-- ctx := context.WithValue(ss.Context(), resolvedEnvironmentIDKey{}, envID)- return handler(srv, &contextualServerStream{ServerStream: ss, ctx: ctx})- }-}--func (s *TunnelServer) loggingStreamInterceptorInternal() grpc.StreamServerInterceptor {- return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {- start := time.Now()- err := handler(srv, ss)- duration := time.Since(start)-- if err != nil {- if isExpectedGRPCReceiveErrorInternal(err) {- slog.DebugContext(ss.Context(), "gRPC stream closed", "method", info.FullMethod, "duration", duration, "error", err)- } else {- slog.WarnContext(ss.Context(), "gRPC stream failed", "method", info.FullMethod, "duration", duration, "error", err)- }- return err- }-- slog.DebugContext(ss.Context(), "gRPC stream completed", "method", info.FullMethod, "duration", duration)- return nil- }-}--func (s *TunnelServer) recoveryStreamInterceptorInternal() grpc.StreamServerInterceptor {- return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {- defer func() {- if panicErr := utils.PanicToError(recover()); panicErr != nil {- slog.ErrorContext(ss.Context(), "panic in gRPC tunnel stream",- "method", info.FullMethod,- "error", panicErr,- )- err = status.Error(codes.Internal, "internal tunnel error")- }- }()-- return handler(srv, ss)- }-}-func (s *contextualServerStream) Context() context.Context {return s.ctx}-func resolvedEnvironmentIDFromContextInternal(ctx context.Context) mo.Option[string] {- if ctx == nil {- return mo.None[string]()- }-- envID, ok := ctx.Value(resolvedEnvironmentIDKey{}).(string)- if !ok || strings.TrimSpace(envID) == "" {- return mo.None[string]()+// requireCertificateIdentity checks a direct-TLS client certificate against envID. A nil state+// means TLS ended before Arcane, such as at a proxy, so only the token applies.+func (s *TunnelServer) requireCertificateIdentity(state *tls.ConnectionState, envID string) error {+ mode := EdgeMTLSModeDisabled+ if s.Config != nil {+ mode = NormalizeEdgeMTLSMode(s.Config.EdgeMTLSMode)}-- return mo.Some(envID)-}--func (s *TunnelServer) requireCertificateIdentityInternal(state *tls.ConnectionState, envID string) error {- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeDisabled {+ if mode == EdgeMTLSModeDisabled || state == nil {return nil}- return verifiedPeerCertificateEnvironmentIDMatchesInternal(state, envID, certgen.EdgeMTLSTrustDomain(edgeMTLSAppURLInternal(s.cfg)))-}--func (s *TunnelServer) requireRequestCertificateIdentityInternal(req *http.Request, envID string) error {- if req == nil || req.TLS == nil {- return nil- }- if !hasVerifiedPeerCertificateInternal(req.TLS) {- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeRequired {+ if !hasVerifiedPeerCertificate(state) {+ if mode == EdgeMTLSModeRequired {return errors.New("verified edge mTLS client certificate is required")}return nil}- return s.requireCertificateIdentityInternal(req.TLS, envID)-}-func (s *TunnelServer) requireCertificateIdentityFromContextInternal(ctx context.Context, envID string) error {- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeDisabled {- return nil+ safeEnvID := sanitizedEnvID(envID)+ if safeEnvID == "" {+ return errors.New("environment ID is required for edge mTLS certificate identity check")}- p, ok := peer.FromContext(ctx)- if !ok {- return nil+ appURL := cmp.Or(strings.TrimSpace(s.Config.AppURL), strings.TrimSpace(httpx.ManagerBaseURL(s.Config.ManagerApiUrl)))+ expected := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)+ if expected == nil {+ return errors.New("edge mTLS trust domain is required for certificate identity check")}- tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo)- if !ok {- return nil- }- if !hasVerifiedPeerCertificateInternal(&tlsInfo.State) {- if NormalizeEdgeMTLSMode(s.edgeMTLSModeInternal()) == EdgeMTLSModeRequired {- return errors.New("verified edge mTLS client certificate is required")- }- return nil- }- return verifiedPeerCertificateEnvironmentIDMatchesInternal(&tlsInfo.State, envID, certgen.EdgeMTLSTrustDomain(edgeMTLSAppURLInternal(s.cfg)))-}--func (s *TunnelServer) edgeMTLSModeInternal() string {- if s == nil || s.cfg == nil {- return EdgeMTLSModeDisabled- }- return s.cfg.EdgeMTLSMode-}--func securityModeFromGRPCContextInternal(ctx context.Context) string {- p, ok := peer.FromContext(ctx)- if !ok {- return "token"+ if !certificateHasURISAN(state.VerifiedChains[0][0], expected) {+ return fmt.Errorf("verified edge mTLS client certificate does not match environment %s", strings.TrimSpace(envID))}- return grpcContextSecurityModeInternal(*p)+ return nil}// IsInternalTunnelRequest reports whether a request is being dispatched by thediff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.goindex 26ce4a991..5d11d88d3 100644--- a/backend/pkg/libarcane/edge/server_test.go+++ b/backend/pkg/libarcane/edge/server_test.go@@ -10,6 +10,7 @@ import ("net/http""net/http/httptest""os"+ "path/filepath""strings""testing""time"@@ -174,10 +175,10 @@ func TestTunnelServer_HandleConnect_AcceptsTokenAfterProxyTerminatedMTLS(t *test}return "", errors.New("invalid token")}, nil)- server.SetConfig(&Config{+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired,AppURL: "https://manager.example.com",- })+ }router := echo.New()router.GET("/connect", server.HandleConnect)@@ -334,14 +335,14 @@ func TestTunnelServer_HandleMTLSEnroll(t *testing.T) {}return "env-mtls", nil}, nil)- server.SetEnvironmentNameResolver(func(ctx context.Context, environmentID string) (string, error) {+ server.NameResolver = func(ctx context.Context, environmentID string) (string, error) {require.Equal(t, "env-mtls", environmentID)return "Lab Server", nil- })- server.SetConfig(&Config{+ }+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired,EdgeMTLSAssetsDir: t.TempDir(),- })+ }router := echo.New()router.POST("/enroll", server.HandleMTLSEnroll)@@ -378,10 +379,10 @@ func TestTunnelServer_HandleMTLSEnroll_ServesCachedAssetsDuringCooldown(t *testi}return "env-repeat", nil}, nil)- server.SetConfig(&Config{+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired,EdgeMTLSAssetsDir: t.TempDir(),- })+ }router := echo.New()router.POST("/enroll", server.HandleMTLSEnroll)@@ -418,13 +419,13 @@ func TestTunnelServer_HandleMTLSEnroll_AllowsRepeatAfterCooldownAndMarksReenroll}return "env-repeat-old", nil}, nil)- server.SetConfig(cfg)+ server.Config = cfgreenrolled := make(chan bool, 2)- server.SetEnrollmentCallback(func(ctx context.Context, environmentID, remoteAddr string, certIssued, caGenerated, wasReenrolled bool) {+ server.EnrollmentCallback = func(ctx context.Context, environmentID, remoteAddr string, certIssued, caGenerated, wasReenrolled bool) {require.Equal(t, "env-repeat-old", environmentID)reenrolled <- wasReenrolled- })+ }router := echo.New()router.POST("/enroll", server.HandleMTLSEnroll)@@ -436,8 +437,7 @@ func TestTunnelServer_HandleMTLSEnroll_AllowsRepeatAfterCooldownAndMarksReenrollrequire.Equal(t, http.StatusOK, w.Code)require.False(t, <-reenrolled)- markerPath, err := managerMTLSEnrollmentMarkerPathInternal(cfg, "env-repeat-old")- require.NoError(t, err)+ markerPath := filepath.Join(cfg.EdgeMTLSAssetsDir, generatedClientMTLSSubdir, "env-repeat-old", generatedMTLSEnrolledName)oldEnrollment := time.Now().Add(-(managerMTLSReenrollCooldown + time.Minute)).UTC().Format(time.RFC3339Nano) + "\n"require.NoError(t, os.WriteFile(markerPath, []byte(oldEnrollment), 0o600))@@ -494,65 +494,71 @@ func TestTunnelServer_resolveEnvironment_Errors(t *testing.T) {})}-func TestTokenFromMetadata(t *testing.T) {+func TestAgentToken(t *testing.T) {t.Run("prefers agent token and trims whitespace", func(t *testing.T) {- ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs(+ md := metadata.Pairs(strings.ToLower(HeaderAgentToken), " agent-token ",strings.ToLower(HeaderAPIKey), "api-token",- ))+ )- assert.Equal(t, "agent-token", tokenFromMetadataInternal(ctx))+ token, source := agentToken(md.Get)+ assert.Equal(t, "agent-token", token)+ assert.Equal(t, HeaderAgentToken, source)})t.Run("falls back to api key", func(t *testing.T) {- ctx := metadata.NewIncomingContext(t.Context(), metadata.Pairs(- strings.ToLower(HeaderAPIKey), "api-token",- ))+ md := metadata.Pairs(strings.ToLower(HeaderAPIKey), "api-token")- assert.Equal(t, "api-token", tokenFromMetadataInternal(ctx))+ token, source := agentToken(md.Get)+ assert.Equal(t, "api-token", token)+ assert.Equal(t, HeaderAPIKey, source)})t.Run("returns empty when metadata missing", func(t *testing.T) {- assert.Empty(t, tokenFromMetadataInternal(t.Context()))+ var md metadata.MD+ token, source := agentToken(md.Get)+ assert.Empty(t, token)+ assert.Empty(t, source)})}func TestIsExpectedTunnelReceiveError(t *testing.T) {t.Run("nil is not expected", func(t *testing.T) {- assert.False(t, isExpectedGRPCReceiveErrorInternal(nil))+ assert.False(t, isExpectedReceiveError(nil))})t.Run("io eof expected", func(t *testing.T) {- assert.True(t, isExpectedGRPCReceiveErrorInternal(io.EOF))+ assert.True(t, isExpectedReceiveError(io.EOF))})t.Run("context canceled expected", func(t *testing.T) {- assert.True(t, isExpectedGRPCReceiveErrorInternal(context.Canceled))+ assert.True(t, isExpectedReceiveError(context.Canceled))})t.Run("context deadline expected", func(t *testing.T) {- assert.True(t, isExpectedGRPCReceiveErrorInternal(context.DeadlineExceeded))+ assert.True(t, isExpectedReceiveError(context.DeadlineExceeded))})t.Run("grpc canceled expected", func(t *testing.T) {err := status.Error(codes.Canceled, "context canceled")- assert.True(t, isExpectedGRPCReceiveErrorInternal(err))+ assert.True(t, isExpectedReceiveError(err))})t.Run("grpc deadline exceeded expected", func(t *testing.T) {err := status.Error(codes.DeadlineExceeded, "deadline exceeded")- assert.True(t, isExpectedGRPCReceiveErrorInternal(err))+ assert.True(t, isExpectedReceiveError(err))})t.Run("unexpected error not expected", func(t *testing.T) {- assert.False(t, isExpectedGRPCReceiveErrorInternal(errors.New("boom")))+ assert.False(t, isExpectedReceiveError(errors.New("boom")))})}func TestTunnelServer_HandleEventCallback(t *testing.T) {called := make(chan struct{}, 1)- server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)- server.SetEventCallback(func(ctx context.Context, environmentID string, event *TunnelEvent) error {+ registry := NewTunnelRegistry()+ server := NewTunnelServerWithRegistry(registry, nil, nil)+ server.EventCallback = func(ctx context.Context, environmentID string, event *TunnelEvent) error {assert.Equal(t, "env-edge", environmentID)require.NotNil(t, event)assert.Equal(t, "container.start", event.Type)@@ -562,32 +568,43 @@ func TestTunnelServer_HandleEventCallback(t *testing.T) {default:}return nil- })+ }- tunnel := NewAgentTunnelWithConn("env-edge", &fakeServerTunnelConn{})- deliveryTimer := time.NewTimer(streamDeliveryTimeout)- deliveryTimer.Stop()- defer deliveryTimer.Stop()- server.handleTunnelMessage(t.Context(), tunnel, &TunnelMessage{+ conn := &fakeServerTunnelConn{messages: []*TunnelMessage{{Type: MessageTypeEvent,Event: &TunnelEvent{Type: "container.start",Title: "Container started",},- }, deliveryTimer)+ }}}+ server.manageConnectedTunnel(t.Context(), "env-edge", conn, &TunnelMessage{Type: MessageTypeRegister}, "token")select {case <-called:case <-time.After(2 * time.Second):require.FailNow(t, "timeout waiting for event callback")}++ stopCtx, cancelStop := context.WithTimeout(t.Context(), time.Second)+ defer cancelStop()+ require.NoError(t, registry.Stop(stopCtx))}-type fakeServerTunnelConn struct{}+// fakeServerTunnelConn delivers messages in order, then reports EOF.+type fakeServerTunnelConn struct {+ messages []*TunnelMessage+}func (f *fakeServerTunnelConn) Send(msg *TunnelMessage) error { return nil }-func (f *fakeServerTunnelConn) Receive() (*TunnelMessage, error) { return nil, io.EOF }+func (f *fakeServerTunnelConn) Receive() (*TunnelMessage, error) {+ if len(f.messages) == 0 {+ return nil, io.EOF+ }+ msg := f.messages[0]+ f.messages = f.messages[1:]+ return msg, nil+}func (f *fakeServerTunnelConn) IsExpectedReceiveError(error) bool { return false }@@ -638,23 +655,23 @@ func TestTunnelServer_ManageConnectedTunnel_RegistersBeforeSendingGRPCRegisterRereturn nil}- tunnel := NewAgentTunnelWithConn(envID, conn)- server.manageConnectedTunnel(t.Context(), t.Context(), tunnel)+ server.manageConnectedTunnel(t.Context(), envID, conn, &TunnelMessage{Type: MessageTypeRegister}, "token")_, ok := server.registry.Get(envID).Get()assert.False(t, ok)}-func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCancellationInternal(t *testing.T) {+func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCancellation(t *testing.T) {registry := NewTunnelRegistry()server := NewTunnelServerWithRegistry(registry, nil, nil)- tunnel := NewAgentTunnelWithConn("env-cancelled-disconnect", ®isterResponseOrderConn{recvErr: context.Canceled})+ envID := "env-cancelled-disconnect"+ conn := ®isterResponseOrderConn{recvErr: context.Canceled}connectionCtx, cancelConnection := context.WithCancel(t.Context())cancelConnection()- server.manageConnectedTunnel(connectionCtx, context.WithoutCancel(connectionCtx), tunnel)+ server.manageConnectedTunnel(connectionCtx, envID, conn, &TunnelMessage{Type: MessageTypeRegister}, "token")- _, ok := registry.Get(tunnel.EnvironmentID).Get()+ _, ok := registry.Get(envID).Get()require.False(t, ok)stopCtx, cancelStop := context.WithTimeout(t.Context(), time.Second)diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.goindex b7d8871ad..a6f386e43 100644--- a/backend/pkg/libarcane/edge/tls.go+++ b/backend/pkg/libarcane/edge/tls.go@@ -1,7 +1,6 @@package edgeimport (- "bytes""cmp""context""crypto"@@ -16,7 +15,6 @@ import ("errors""fmt""io"- "io/fs""log/slog""net/http""net/url"@@ -24,6 +22,7 @@ import ("path""path/filepath""regexp"+ "slices""strconv""strings""syscall"@@ -34,18 +33,15 @@ import ("go.getarcane.app/acfs/atomic"kit "go.getarcane.app/kit/pkg"libcrypto "go.getarcane.app/sys/crypto"- "google.golang.org/grpc/credentials"- "google.golang.org/grpc/peer""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx")const (- defaultGeneratedMTLSDir = "data/edge-mtls"- defaultAgentMTLSDir = "data/edge-mtls-agent"- generatedMTLSContainerDir = "/app/data/edge-mtls-agent"- generatedMTLSCertValidity = 5 * 365 * 24 * time.Hour+ managerMTLSDirName = "edge-mtls"+ agentMTLSDirName = "edge-mtls-agent"+ generatedMTLSContainerDir = "/app/data/" + agentMTLSDirNamegeneratedClientMTLSSubdir = "clients"generatedMTLSCACertFileName = "ca.crt"generatedMTLSCAKeyFileName = "ca.key"@@ -58,15 +54,16 @@ const (managerCALockTimeout = 2 * time.MinutemanagerCALockPollInterval = 100 * time.Millisecond- // caKeyEncryptedPrefix marks files written with libcrypto envelope encryption.- // The payload after the prefix is the base64 ciphertext returned by- // libcrypto.Encrypt of the plain PEM-encoded CA private key.+ // caKeyEncryptedPrefix marks a CA key file holding the libcrypto.Encrypt output of the PEM key.caKeyEncryptedPrefix = "ARCANE-ENC-V1:")-var generatedAssetNameSanitizer = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)--var managerCALocks utils.KeyedMutex+var (+ generatedAssetNameSanitizer = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)+ managerCALocks utils.KeyedMutex+ // caKeyEncrypt is a variable so tests can force encryption failures.+ caKeyEncrypt = libcrypto.Encrypt+)// BuildManagerServerTLSConfig returns the manager TLS configuration needed to// support optional edge mTLS on the shared Arcane listener.@@ -75,19 +72,13 @@ func BuildManagerServerTLSConfig(cfg *Config) (*tls.Config, error) {return nil, nil}- caPool, err := loadCertPoolInternal(strings.TrimSpace(cfg.EdgeMTLSCAFile))- if err != nil {+ caPool := x509.NewCertPool()+ if err := appendCAFile(caPool, cfg.EdgeMTLSCAFile); err != nil {return nil, fmt.Errorf("failed to load edge mTLS CA file: %w", err)}- // ClientAuth is intentionally VerifyClientCertIfGiven even when- // EdgeMTLSMode is "required". Enforcement of certificate identity is done- // per-request at the application layer so that the mTLS enrollment endpoint,- // which agents must reach before they own a client certificate, remains- // accessible. In proxy-terminated deployments the TLS state is not visible- // here; identity is enforced by the upstream proxy.- // If the handshake were set to RequireAndVerifyClientCert, bootstrap would fail.- // TODO: reload ClientCAs when CA rotation support is added.+ // Even "required" mode only verifies certificates when given: agents enroll before owning one,+ // so identity is enforced per request. TODO: reload ClientCAs when CA rotation support is added.return &tls.Config{MinVersion: tls.VersionTLS12,ClientAuth: tls.VerifyClientCertIfGiven,@@ -103,7 +94,7 @@ func NewManagerHTTPClient(cfg *Config, timeout time.Duration) (*http.Client, errreturn nil, errors.New("http.DefaultTransport is not *http.Transport")}transport := baseTransport.Clone()- tlsConfig, err := buildManagerClientTLSConfigInternal(cfg)+ tlsConfig, err := buildManagerClientTLSConfig(cfg)if err != nil {return nil, err}@@ -118,40 +109,25 @@ func NewManagerHTTPClient(cfg *Config, timeout time.Duration) (*http.Client, errreturn client, nil}-// PrepareManagerMTLSAssetsWithContext ensures Arcane-managed edge mTLS assets exist when-// edge mTLS is enabled and no explicit manager CA file is configured.+// PrepareManagerMTLSAssetsWithContext generates the manager CA when edge mTLS is+// enabled and no explicit manager CA file is configured.func PrepareManagerMTLSAssetsWithContext(ctx context.Context, cfg *Config) error {- if !shouldAutoGenerateManagerCAInternal(cfg) {+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled || strings.TrimSpace(cfg.EdgeMTLSCAFile) != "" {return nil}- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)if err != nil {return err}-- if _, _, _, ensureManagerCAErr := ensureManagerCAInternal(ctx, assetsDir); ensureManagerCAErr != nil {- return ensureManagerCAErr+ if _, _, caErr := ensureManagerCA(ctx, assetsDir); caErr != nil {+ return caErr}cfg.EdgeMTLSCAFile = filepath.Join(assetsDir, generatedMTLSCACertFileName)return nil}-func generatedManagerMTLSCAPathInternal(cfg *Config) (string, error) {- if cfg == nil {- return "", errors.New("edge config is required")- }- if configured := strings.TrimSpace(cfg.EdgeMTLSCAFile); configured != "" {- return configured, nil- }- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)- if err != nil {- return "", err- }- return filepath.Join(assetsDir, generatedMTLSCACertFileName), nil-}-// AvailableManagerMTLSCAPath resolves an existing manager CA certificate.func AvailableManagerMTLSCAPath(cfg *Config) (string, error) {if cfg == nil {@@ -161,50 +137,123 @@ func AvailableManagerMTLSCAPath(cfg *Config) (string, error) {return "", errors.New("edge mTLS is disabled")}- caPath, err := generatedManagerMTLSCAPathInternal(cfg)- if err != nil {- return "", fmt.Errorf("resolve edge mTLS CA path: %w", err)+ caPath := strings.TrimSpace(cfg.EdgeMTLSCAFile)+ if caPath == "" {+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)+ if err != nil {+ return "", fmt.Errorf("resolve edge mTLS CA path: %w", err)+ }+ caPath = filepath.Join(assetsDir, generatedMTLSCACertFileName)}- // os.* rather than acfs: the assets dir may be user-configured to anywhere on- // the host, so no confinement root handle is in scope for this probe.+ // os rather than acfs: the assets dir may be configured anywhere on the host.if _, statErr := os.Stat(caPath); statErr != nil {return "", fmt.Errorf("stat edge mTLS CA: %w", statErr)}return caPath, nil}-// GenerateManagerClientMTLSAssetsWithContext creates or loads the generated CA and per-environment client certificate bundle.+// GenerateManagerClientMTLSAssetsWithContext ensures the generated CA and the environment's client+// certificate, reusing a client certificate that is still valid for the environment's identity.func GenerateManagerClientMTLSAssetsWithContext(ctx context.Context, cfg *Config, envID, envName string) (*GeneratedMTLSAssets, error) {- if !shouldUseGeneratedManagerCAInternal(cfg) {+ if !usesGeneratedManagerCA(cfg) {return nil, nil}- if strings.TrimSpace(envID) == "" {+ safeEnvID := sanitizedEnvID(envID)+ if safeEnvID == "" {return nil, errors.New("environment ID is required")}- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)if err != nil {return nil, err}- caCertPath, _, caGenerated, err := ensureManagerCAInternal(ctx, assetsDir)+ ca, caGenerated, err := ensureManagerCA(ctx, assetsDir)if err != nil {return nil, err}- appURL := edgeMTLSAppURLInternal(cfg)- clientCertPath, clientKeyPath, certIssued, err := ensureClientCertificateInternal(ctx, assetsDir, envID, envName, appURL)++ clientLogicalDir := path.Join("/", generatedClientMTLSSubdir, safeEnvID)+ clientDir := filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID)+ if mkdirErr := acfs.MkdirAll(ctx, assetsDir, clientLogicalDir, utils.DirPerm); mkdirErr != nil {+ return nil, fmt.Errorf("failed to create client cert dir: %w", mkdirErr)+ }+ unlock, err := lockEdgeMTLSPath(ctx, clientDir, ".client.lock")if err != nil {return nil, err}+ defer unlock()++ certPath := filepath.Join(clientDir, generatedMTLSClientCertName)+ keyPath := filepath.Join(clientDir, generatedMTLSClientKeyName)+ appURL := cmp.Or(strings.TrimSpace(cfg.AppURL), strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)))+ uriSAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)++ // The URI SAN is the stable identity, so renaming an environment keeps its certificate.+ existing, reason := loadClientCertificate(certPath, keyPath, time.Now())+ certIssued := reason != "" || !supportedGeneratedKey(existing.Leaf.PublicKey) ||+ existing.Leaf.CheckSignatureFrom(ca.Leaf) != nil || (uriSAN != nil && !certificateHasURISAN(existing.Leaf, uriSAN))+ if certIssued {+ // Client keys follow the CA: P-384 under a legacy ECDSA CA, otherwise ML-DSA-87.+ var clientKey crypto.Signer+ if _, ecdsaCA := ca.PrivateKey.(*ecdsa.PrivateKey); ecdsaCA {+ clientKey, err = certgen.GenerateP384PrivateKey()+ } else {+ clientKey, err = certgen.GenerateMLDSA87PrivateKey()+ }+ if err != nil {+ return nil, fmt.Errorf("failed to generate client private key: %w", err)+ }- caPEM, err := readGeneratedAssetInternal(ctx, assetsDir, caCertPath)+ // The common name is display metadata: "<name>-<id>", capped at 64 characters.+ safeEnvName := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envName), "-")+ commonName := safeEnvID+ if name := strings.Trim(safeEnvName, "-_"); name != "" && len(safeEnvID) < 63 {+ if name = strings.Trim(name[:min(len(name), 63-len(safeEnvID))], "-_"); name != "" {+ commonName = name + "-" + safeEnvID+ }+ }+ dnsSANs := []string{"arcane-agent"}+ if name := strings.Trim(safeEnvName, "-_."); name != "" && uriSAN != nil {+ dnsSANs = append(dnsSANs, name+".agent."+uriSAN.Host)+ }++ template, templateErr := certgen.NewEdgeMTLSClientTemplate(commonName, uriSAN, dnsSANs)+ if templateErr != nil {+ return nil, templateErr+ }+ certDER, createErr := x509.CreateCertificate(rand.Reader, template, ca.Leaf, clientKey.Public(), ca.PrivateKey)+ if createErr != nil {+ return nil, fmt.Errorf("failed to create client certificate: %w", createErr)+ }+ if writeErr := atomic.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), utils.FilePerm); writeErr != nil {+ return nil, writeErr+ }++ // P-384 keys keep SEC1 framing so agents from before the ML-DSA migration can parse them.+ keyPEM := &pem.Block{Type: "PRIVATE KEY"}+ if ecKey, ok := clientKey.(*ecdsa.PrivateKey); ok {+ keyPEM.Type = "EC PRIVATE KEY"+ keyPEM.Bytes, err = x509.MarshalECPrivateKey(ecKey)+ } else {+ keyPEM.Bytes, err = x509.MarshalPKCS8PrivateKey(clientKey)+ }+ if err != nil {+ return nil, fmt.Errorf("failed to marshal client private key: %w", err)+ }+ if writeErr := atomic.WriteFile(keyPath, pem.EncodeToMemory(keyPEM), 0o600); writeErr != nil {+ return nil, writeErr+ }+ }++ caPEM, err := acfs.ReadFile(ctx, assetsDir, "/"+generatedMTLSCACertFileName)if err != nil {return nil, fmt.Errorf("failed to read generated CA certificate: %w", err)}- clientCertPEM, err := readGeneratedAssetInternal(ctx, assetsDir, clientCertPath)+ clientCertPEM, err := acfs.ReadFile(ctx, assetsDir, path.Join(clientLogicalDir, generatedMTLSClientCertName))if err != nil {return nil, fmt.Errorf("failed to read generated client certificate: %w", err)}- clientKeyPEM, err := readGeneratedAssetInternal(ctx, assetsDir, clientKeyPath)+ clientKeyPEM, err := acfs.ReadFile(ctx, assetsDir, path.Join(clientLogicalDir, generatedMTLSClientKeyName))if err != nil {return nil, fmt.Errorf("failed to read generated client key: %w", err)}@@ -214,185 +263,113 @@ func GenerateManagerClientMTLSAssetsWithContext(ctx context.Context, cfg *ConfigCertIssued: certIssued,CAGenerated: caGenerated,Files: []GeneratedMTLSFile{- {Name: generatedMTLSCACertFileName, Content: string(caPEM), ContainerPath: filepath.ToSlash(filepath.Join(generatedMTLSContainerDir, generatedMTLSCACertFileName)), Permissions: "0644"},- {Name: generatedMTLSClientCertName, Content: string(clientCertPEM), ContainerPath: filepath.ToSlash(filepath.Join(generatedMTLSContainerDir, generatedMTLSClientCertName)), Permissions: "0644"},- {Name: generatedMTLSClientKeyName, Content: string(clientKeyPEM), ContainerPath: filepath.ToSlash(filepath.Join(generatedMTLSContainerDir, generatedMTLSClientKeyName)), Permissions: "0600"},+ {Name: generatedMTLSCACertFileName, Content: string(caPEM), ContainerPath: path.Join(generatedMTLSContainerDir, generatedMTLSCACertFileName), Permissions: "0644"},+ {Name: generatedMTLSClientCertName, Content: string(clientCertPEM), ContainerPath: path.Join(generatedMTLSContainerDir, generatedMTLSClientCertName), Permissions: "0644"},+ {Name: generatedMTLSClientKeyName, Content: string(clientKeyPEM), ContainerPath: path.Join(generatedMTLSContainerDir, generatedMTLSClientKeyName), Permissions: "0600"},},}, nil}-// readGeneratedAssetInternal reads a file Arcane itself wrote under the-// generated-assets directory, confined to that root.-func readGeneratedAssetInternal(ctx context.Context, assetsDir, absPath string) ([]byte, error) {- logicalPath, err := acfs.LogicalPath(assetsDir, absPath)- if err != nil {- return nil, err- }- return acfs.ReadFile(ctx, assetsDir, logicalPath)-}--func managerMTLSEnrollmentStateInternal(cfg *Config, envID string, now time.Time) (bool, bool, error) {- markerPath, err := managerMTLSEnrollmentMarkerPathInternal(cfg, envID)- if err != nil {- return false, false, err- }- enrolledAt, err := readMTLSEnrollmentMarkerInternal(markerPath)- if err != nil {- return false, false, kit.Ternary(errors.Is(err, fs.ErrNotExist), nil, err)- }- if now.IsZero() {- now = time.Now()- }- if enrolledAt.IsZero() {- return true, false, nil- }- return true, now.Sub(enrolledAt) < managerMTLSReenrollCooldown, nil-}--func recordManagerMTLSEnrollmentInternal(cfg *Config, envID string, now time.Time) error {- markerPath, err := managerMTLSEnrollmentMarkerPathInternal(cfg, envID)- if err != nil {- return err- }- if now.IsZero() {- now = time.Now()- }- return atomic.WriteFile(markerPath, []byte(now.UTC().Format(time.RFC3339Nano)+"\n"), 0o600)-}--func managerMTLSEnrollmentMarkerPathInternal(cfg *Config, envID string) (string, error) {- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)- if err != nil {- return "", err- }- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")- if safeEnvID == "" {- return "", errors.New("environment ID is required")- }- return filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID, generatedMTLSEnrolledName), nil-}--func readMTLSEnrollmentMarkerInternal(localPath string) (time.Time, error) {- // Path-only helper; callers derive the path from the assets root, kept on- // os.* with the other path-only readers.- content, err := os.ReadFile(localPath)- if err != nil {- return time.Time{}, err- }- trimmed := strings.TrimSpace(string(content))- if trimmed == "" {- return time.Time{}, nil- }- enrolledAt, err := time.Parse(time.RFC3339Nano, trimmed)- if err != nil {- return time.Time{}, fmt.Errorf("failed to parse edge mTLS enrollment marker %s: %w", localPath, err)- }- return enrolledAt, nil-}-// GeneratedManagerClientMTLSCertPath returns the manager-side generated client certificate path for an environment.func GeneratedManagerClientMTLSCertPath(cfg *Config, envID string) (string, error) {- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)if err != nil {return "", err}-- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")+ safeEnvID := sanitizedEnvID(envID)if safeEnvID == "" {return "", errors.New("environment ID is required")}-- return filepath.Join(assetsDir, "clients", safeEnvID, generatedMTLSClientCertName), nil+ return filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID, generatedMTLSClientCertName), nil}-// EnsureAgentMTLSAssets downloads manager-generated client certificates when-// edge mTLS is enabled and explicit client cert/key files are not configured.+// EnsureAgentMTLSAssets downloads manager-generated client certificates when edge mTLS is+// enabled without configured client cert/key files, and renews them before they expire.func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {- if !shouldAutoEnrollAgentMTLSInternal(cfg) {- return nil- }- if hasClientCertificateInternal(cfg) {+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled ||+ (strings.TrimSpace(cfg.EdgeMTLSCertFile) != "" && strings.TrimSpace(cfg.EdgeMTLSKeyFile) != "") {return nil}- assetsDir, err := edgeAgentMTLSAssetsDirInternal(cfg)+ assetsDir, err := edgeMTLSAssetsDir(cfg, agentMTLSDirName)if err != nil {return err}certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)- if fileExistsInternal(certPath) && fileExistsInternal(keyPath) {- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath, time.Now())- if !needsEnrollment {- if !fileExistsInternal(filepath.Join(assetsDir, generatedMTLSEnrolledName)) {- if writeErr := acfs.Write(ctx, assetsDir, "/"+generatedMTLSEnrolledName, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), acfs.WriteOptions{Mode: 0o600}); writeErr != nil {- return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr)- }++ _, reason := loadClientCertificate(certPath, keyPath, time.Now())+ if reason != "" {+ if fileExists(certPath) {+ slog.WarnContext(ctx, "Existing edge mTLS assets need renewal; enrolling new assets", "reason", reason, "certPath", certPath)+ }+ if enrollErr := enrollAgentMTLSAssets(ctx, cfg, assetsDir, certPath, keyPath); enrollErr != nil {+ if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired {+ slog.WarnContext(ctx, "Edge mTLS enrollment failed; proceeding without client certificate", "error", enrollErr)+ return nil}- setAgentMTLSAssetPathsInternal(cfg, assetsDir)- return nil+ return enrollErr}- slog.WarnContext(ctx, "Existing edge mTLS assets need renewal; enrolling new assets", "reason", reason, "certPath", certPath)}- if enrollAgentMTLSAssetsErr := enrollAgentMTLSAssetsInternal(ctx, cfg, assetsDir, certPath, keyPath); enrollAgentMTLSAssetsErr != nil {- if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired {- slog.WarnContext(ctx, "Edge mTLS enrollment failed; proceeding without client certificate", "error", enrollAgentMTLSAssetsErr)- return nil+ if reason != "" || !fileExists(filepath.Join(assetsDir, generatedMTLSEnrolledName)) {+ marker := []byte(time.Now().UTC().Format(time.RFC3339) + "\n")+ if writeErr := acfs.Write(ctx, assetsDir, "/"+generatedMTLSEnrolledName, marker, acfs.WriteOptions{Mode: 0o600}); writeErr != nil {+ return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr)}- return enrollAgentMTLSAssetsErr+ }+ cfg.EdgeMTLSCertFile = certPath+ cfg.EdgeMTLSKeyFile = keyPath+ if caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName); fileExists(caPath) && strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {+ cfg.EdgeMTLSCAFile = caPath}return nil}-func enrollAgentMTLSAssetsInternal(ctx context.Context, cfg *Config, assetsDir, certPath, keyPath string) error {- if cfg == nil {- return errors.New("MANAGER_API_URL is required to enroll edge mTLS assets")- }+// enrollAgentMTLSAssets downloads and writes the manager-issued client assets. It stays separate+// from EnsureAgentMTLSAssets to keep that function under the cognitive-complexity limit.+func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPath, keyPath string) error {managerBaseURL := strings.TrimRight(strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl)), "/")if managerBaseURL == "" {return errors.New("MANAGER_API_URL is required to enroll edge mTLS assets")}- if !managerUsesTLSInternal(cfg) {+ if !managerUsesTLS(cfg) {return errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https for certificate enrollment")}- httpClient, err := NewManagerHTTPClient(cfg, 30*time.Second)- if err != nil {- return fmt.Errorf("failed to configure edge mTLS enrollment client: %w", err)+ httpClient, clientErr := NewManagerHTTPClient(cfg, 30*time.Second)+ if clientErr != nil {+ return fmt.Errorf("failed to configure edge mTLS enrollment client: %w", clientErr)}-- req, err := http.NewRequestWithContext(ctx, http.MethodPost, managerBaseURL+"/api/tunnel/mtls/enroll", http.NoBody)- if err != nil {- return fmt.Errorf("failed to create edge mTLS enrollment request: %w", err)+ req, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, managerBaseURL+"/api/tunnel/mtls/enroll", http.NoBody)+ if reqErr != nil {+ return fmt.Errorf("failed to create edge mTLS enrollment request: %w", reqErr)+ }+ for header, value := range agentAuthCredentialsInternal(cfg.AgentToken) {+ req.Header.Set(header, value)}- req.Header.Set(HeaderAgentToken, cfg.AgentToken)- req.Header.Set(HeaderAPIKey, cfg.AgentToken)- req.Header.Set(HeaderAuthorization, "Bearer "+cfg.AgentToken)- resp, err := httpClient.Do(req)- if err != nil {- return fmt.Errorf("edge mTLS enrollment request failed: %w", err)+ resp, doErr := httpClient.Do(req)+ if doErr != nil {+ return fmt.Errorf("edge mTLS enrollment request failed: %w", doErr)}defer func() { _ = resp.Body.Close() }()-if resp.StatusCode != http.StatusOK {body, _ := io.ReadAll(io.LimitReader(resp.Body, maxEnrollResponseBytes))return fmt.Errorf("edge mTLS enrollment failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))}var enrollResp enrollMTLSResponse- if unmarshalReadErr := json.UnmarshalRead(io.LimitReader(resp.Body, maxEnrollResponseBytes), &enrollResp); unmarshalReadErr != nil {- return fmt.Errorf("failed to decode edge mTLS enrollment response: %w", unmarshalReadErr)+ if decodeErr := json.UnmarshalRead(io.LimitReader(resp.Body, maxEnrollResponseBytes), &enrollResp); decodeErr != nil {+ return fmt.Errorf("failed to decode edge mTLS enrollment response: %w", decodeErr)}if len(enrollResp.Files) == 0 {return errors.New("edge mTLS enrollment response did not include any files")}- // The assets dir is the confinement root for the writes below, so it is- // created through os before acfs opens it.- if mkdirAllErr := os.MkdirAll(assetsDir, utils.DirPerm); mkdirAllErr != nil {- return fmt.Errorf("failed to create edge mTLS asset dir: %w", mkdirAllErr)+ // The assets dir is the acfs confinement root for the writes below, so os creates it.+ if mkdirErr := os.MkdirAll(assetsDir, utils.DirPerm); mkdirErr != nil {+ return fmt.Errorf("failed to create edge mTLS asset dir: %w", mkdirErr)}for _, file := range enrollResp.Files {perm := kit.Ternary(strings.TrimSpace(file.Permissions) == "0600", 0o600, utils.FilePerm)@@ -400,309 +377,142 @@ func enrollAgentMTLSAssetsInternal(ctx context.Context, cfg *Config, assetsDir,return fmt.Errorf("failed to write edge mTLS asset %s: %w", file.Name, writeErr)}}- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath, time.Now())- if needsEnrollment {- return fmt.Errorf("edge mTLS enrollment wrote unusable assets: %s", reason)+ if _, writtenReason := loadClientCertificate(certPath, keyPath, time.Now()); writtenReason != "" {+ return fmt.Errorf("edge mTLS enrollment wrote unusable assets: %s", writtenReason)}- if writeErr2 := acfs.Write(ctx, assetsDir, "/"+generatedMTLSEnrolledName, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), acfs.WriteOptions{Mode: 0o600}); writeErr2 != nil {- return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr2)- }-- setAgentMTLSAssetPathsInternal(cfg, assetsDir)return nil}-func buildManagerClientTLSConfigInternal(cfg *Config) (*tls.Config, error) {- ctx := context.Background() //nolint:forbidigo // TLS configuration is built at startup without a request context.- if cfg == nil || !managerUsesTLSInternal(cfg) {+// buildManagerClientTLSConfig returns the agent's TLS config for an https manager URL, adding the+// configured CA to the system roots and the configured client certificate.+func buildManagerClientTLSConfig(cfg *Config) (*tls.Config, error) {+ if cfg == nil || !managerUsesTLS(cfg) {return nil, nil}+ ctx := context.Background() //nolint:forbidigo // TLS configuration is built at startup without a request context.+ tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: strings.TrimSpace(cfg.EdgeMTLSServerName)}- tlsConfig := &tls.Config{- MinVersion: tls.VersionTLS12,- }-- if serverName := strings.TrimSpace(cfg.EdgeMTLSServerName); serverName != "" {- tlsConfig.ServerName = serverName- }-- if caFile := strings.TrimSpace(cfg.EdgeMTLSCAFile); caFile != "" {- pool, err := loadSystemOrCustomCertPoolInternal(caFile)- if err != nil {- return nil, fmt.Errorf("failed to load edge mTLS CA file: %w", err)+ if strings.TrimSpace(cfg.EdgeMTLSCAFile) != "" {+ pool, err := x509.SystemCertPool()+ if err != nil || pool == nil {+ slog.WarnContext(ctx, "Failed to load system certificate pool; falling back to configured edge mTLS CA only", "error", err)+ pool = x509.NewCertPool()+ }+ if appendErr := appendCAFile(pool, cfg.EdgeMTLSCAFile); appendErr != nil {+ return nil, fmt.Errorf("failed to load edge mTLS CA file: %w", appendErr)}tlsConfig.RootCAs = pool}- if hasClientCertificateInternal(cfg) {- mode := NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode)- certPath := strings.TrimSpace(cfg.EdgeMTLSCertFile)- keyPath := strings.TrimSpace(cfg.EdgeMTLSKeyFile)- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath, time.Now())- if needsEnrollment {- err := fmt.Errorf("edge mTLS client certificate is unusable: %s", reason)- if mode == EdgeMTLSModeOptional {- slog.WarnContext(ctx, "Ignoring unusable optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "error", err.Error())- return tlsConfig, nil- }- return nil, err- }- cert, err := tls.LoadX509KeyPair(certPath, keyPath)- if err != nil {- if mode == EdgeMTLSModeOptional {- slog.WarnContext(ctx, "Failed to load optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "error", err.Error())- return tlsConfig, nil- }- return nil, fmt.Errorf("failed to load edge mTLS client certificate: %w", err)- }- if cert.Leaf != nil {- if _, ok := cert.Leaf.PublicKey.(*mldsa.PublicKey); ok {- tlsConfig.MinVersion = tls.VersionTLS13- }+ certPath, keyPath := strings.TrimSpace(cfg.EdgeMTLSCertFile), strings.TrimSpace(cfg.EdgeMTLSKeyFile)+ if certPath == "" || keyPath == "" {+ return tlsConfig, nil+ }+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now())+ if reason != "" {+ if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeOptional {+ slog.WarnContext(ctx, "Ignoring unusable optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "reason", reason)+ return tlsConfig, nil}- tlsConfig.Certificates = []tls.Certificate{cert}+ return nil, fmt.Errorf("edge mTLS client certificate is unusable: %s", reason)}-+ // ML-DSA client certificates require TLS 1.3.+ if _, ok := cert.Leaf.PublicKey.(*mldsa.PublicKey); ok {+ tlsConfig.MinVersion = tls.VersionTLS13+ }+ tlsConfig.Certificates = []tls.Certificate{cert}return tlsConfig, nil}// ValidateAgentMTLSConfig validates the edge agent TLS configuration before the// reverse tunnel client starts.func ValidateAgentMTLSConfig(cfg *Config) error {- if cfg == nil {- return nil- }-- mode := NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode)- if mode == EdgeMTLSModeDisabled {+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {return nil}-- if !managerUsesTLSInternal(cfg) {+ if !managerUsesTLS(cfg) {return errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https")}-- _, err := buildManagerClientTLSConfigInternal(cfg)+ _, err := buildManagerClientTLSConfig(cfg)return err}-// ValidateManagerMTLSConfig validates the manager-side mTLS configuration used-// by edge tunnel endpoints.+// ValidateManagerMTLSConfig validates a configured manager CA file. An empty CA file is+// allowed because PrepareManagerMTLSAssetsWithContext generates one.func ValidateManagerMTLSConfig(cfg *Config) error {- if cfg == nil {- return nil- }-- mode := NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode)- if mode == EdgeMTLSModeDisabled {- return nil- }-- if strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {+ if cfg == nil || strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {return nil}-- _, err := loadCertPoolInternal(cfg.EdgeMTLSCAFile)- if err != nil {- return fmt.Errorf("failed to load edge mTLS CA file: %w", err)- }- return nil-}--func loadCertPoolInternal(caFile string) (*x509.CertPool, error) {- caFile = strings.TrimSpace(caFile)- if caFile == "" {- return nil, errors.New("CA file is required")- }-- // os.* rather than acfs: the CA path may be user-configured to anywhere on- // the host, so no confinement root exists for it.- pemBytes, err := os.ReadFile(caFile)- if err != nil {- return nil, err- }-- pool := x509.NewCertPool()- if !pool.AppendCertsFromPEM(pemBytes) {- return nil, errors.New("failed to parse PEM certificates")- }-- return pool, nil+ _, err := BuildManagerServerTLSConfig(cfg)+ return err}-func loadSystemOrCustomCertPoolInternal(caFile string) (*x509.CertPool, error) {- pool, err := x509.SystemCertPool()- if err != nil || pool == nil {- ctx := context.Background() //nolint:forbidigo // TLS configuration is built at startup without a request context.- slog.WarnContext(ctx, "Failed to load system certificate pool; falling back to configured edge mTLS CA only",- "error", err)- pool = x509.NewCertPool()- }-+// appendCAFile adds the PEM certificates in caFile to pool. The path may be configured+// anywhere on the host, so it is read with os rather than acfs.+func appendCAFile(pool *x509.CertPool, caFile string) error {caFile = strings.TrimSpace(caFile)if caFile == "" {- return pool, nil+ return errors.New("CA file is required")}-- // os.* rather than acfs: the CA path may be user-configured to anywhere on- // the host, so no confinement root exists for it.pemBytes, err := os.ReadFile(caFile)if err != nil {- return nil, err+ return err}if !pool.AppendCertsFromPEM(pemBytes) {- return nil, errors.New("failed to parse PEM certificates")+ return errors.New("failed to parse PEM certificates")}- return pool, nil+ return nil}-func hasClientCertificateInternal(cfg *Config) bool {- if cfg == nil {- return false- }-- return strings.TrimSpace(cfg.EdgeMTLSCertFile) != "" && strings.TrimSpace(cfg.EdgeMTLSKeyFile) != ""+func managerUsesTLS(cfg *Config) bool {+ return strings.HasPrefix(strings.ToLower(strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))), "https://")}-func managerUsesTLSInternal(cfg *Config) bool {- if cfg == nil {- return false- }-- baseURL := strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))- return strings.HasPrefix(strings.ToLower(baseURL), "https://")-}--func shouldAutoGenerateManagerCAInternal(cfg *Config) bool {- if cfg == nil {- return false- }-- return NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeDisabled &&- strings.TrimSpace(cfg.EdgeMTLSCAFile) == ""-}--func shouldUseGeneratedManagerCAInternal(cfg *Config) bool {- if cfg == nil {- return false- }- if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {+// usesGeneratedManagerCA reports whether edge mTLS runs on the Arcane-generated CA. The configured+// CA also counts once PrepareManagerMTLSAssetsWithContext has pointed it at the generated file.+func usesGeneratedManagerCA(cfg *Config) bool {+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {return false}configuredCA := strings.TrimSpace(cfg.EdgeMTLSCAFile)if configuredCA == "" {return true}- assetsDir, err := edgeMTLSAssetsDirInternal(cfg)- if err != nil {- return false- }- generatedCA := filepath.Join(assetsDir, generatedMTLSCACertFileName)- return filepath.Clean(configuredCA) == filepath.Clean(generatedCA)-}--func shouldAutoEnrollAgentMTLSInternal(cfg *Config) bool {- if cfg == nil {- return false- }-- return NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeDisabled &&- !hasClientCertificateInternal(cfg)-}--func setAgentMTLSAssetPathsInternal(cfg *Config, assetsDir string) {- if cfg == nil {- return- }-- cfg.EdgeMTLSCertFile = filepath.Join(assetsDir, generatedMTLSClientCertName)- cfg.EdgeMTLSKeyFile = filepath.Join(assetsDir, generatedMTLSClientKeyName)-- caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)- if fileExistsInternal(caPath) && strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {- cfg.EdgeMTLSCAFile = caPath- }-}--func requestSecurityModeInternal(req *http.Request) string {- return kit.Ternary(hasVerifiedEdgeMTLSRequestInternal(req), "mtls", "token")-}--func hasVerifiedEdgeMTLSRequestInternal(req *http.Request) bool {- if req == nil {- return false- }- return req.TLS != nil && hasVerifiedPeerCertificateInternal(req.TLS)+ assetsDir, err := edgeMTLSAssetsDir(cfg, managerMTLSDirName)+ return err == nil && filepath.Clean(configuredCA) == filepath.Join(assetsDir, generatedMTLSCACertFileName)}-func grpcContextSecurityModeInternal(pctx peer.Peer) string {- if tlsInfo, ok := pctx.AuthInfo.(credentials.TLSInfo); ok && hasVerifiedPeerCertificateInternal(&tlsInfo.State) {- return "mtls"- }- return "token"+func hasVerifiedPeerCertificate(state *tls.ConnectionState) bool {+ return state != nil && len(state.PeerCertificates) > 0 && len(state.VerifiedChains) > 0}-func hasVerifiedPeerCertificateInternal(state *tls.ConnectionState) bool {- if state == nil {- return false- }- return len(state.PeerCertificates) > 0 && len(state.VerifiedChains) > 0+// certificateHasURISAN reports whether cert carries expected as a URI SAN, ignoring case and a trailing dot in the host.+func certificateHasURISAN(cert *x509.Certificate, expected *url.URL) bool {+ return slices.ContainsFunc(cert.URIs, func(uri *url.URL) bool {+ return uri != nil && strings.EqualFold(uri.Scheme, expected.Scheme) && uri.Path == expected.Path &&+ strings.EqualFold(strings.TrimSuffix(uri.Host, "."), strings.TrimSuffix(expected.Host, "."))+ })}-func verifiedPeerCertificateEnvironmentIDMatchesInternal(state *tls.ConnectionState, envID, trustDomain string) error {- if !hasVerifiedPeerCertificateInternal(state) {- return nil- }- expectedPath := expectedEdgeMTLSURIPathInternal(envID)- if expectedPath == "" {- return errors.New("environment ID is required for edge mTLS certificate identity check")- }- trustDomain = strings.TrimSpace(strings.ToLower(strings.TrimSuffix(trustDomain, ".")))- if trustDomain == "" {- return errors.New("edge mTLS trust domain is required for certificate identity check")- }- leaf := state.VerifiedChains[0][0]- for _, uri := range leaf.URIs {- if uri == nil {- continue- }- if uri.Scheme == "spiffe" && strings.EqualFold(strings.TrimSuffix(uri.Host, "."), trustDomain) && uri.Path == expectedPath {- return nil- }- }- return fmt.Errorf("verified edge mTLS client certificate does not match environment %s", strings.TrimSpace(envID))+// sanitizedEnvID makes an environment ID safe for asset paths and certificate identities.+func sanitizedEnvID(envID string) string {+ return generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")}-func edgeMTLSAppURLInternal(cfg *Config) string {- if cfg == nil {- return ""- }- if appURL := strings.TrimSpace(cfg.AppURL); appURL != "" {- return appURL- }- return strings.TrimSpace(httpx.ManagerBaseURL(cfg.ManagerApiUrl))-}--func expectedEdgeMTLSURIPathInternal(envID string) string {- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")- return kit.Ternary(safeEnvID == "", "", "/edge/"+safeEnvID)-}--func edgeMTLSAssetsDirInternal(cfg *Config) (string, error) {+// edgeMTLSAssetsDir returns the configured assets dir, else data/<name> under /app when it exists.+func edgeMTLSAssetsDir(cfg *Config, name string) (string, error) {if cfg == nil {return "", errors.New("edge config is required")}-if configured := strings.TrimSpace(cfg.EdgeMTLSAssetsDir); configured != "" {return configured, nil}- baseDir := defaultGeneratedMTLSDir+ baseDir := filepath.Join("data", name)// Probes the fixed system path /app/data, which is not under any acfs root.if _, err := os.Stat("/app/data"); err == nil {- baseDir = "/app/data/edge-mtls"+ baseDir = "/app/data/" + name}-resolved, err := filepath.Abs(baseDir)if err != nil {return "", fmt.Errorf("failed to resolve edge mTLS assets dir: %w", err)@@ -710,598 +520,179 @@ func edgeMTLSAssetsDirInternal(cfg *Config) (string, error) {return resolved, nil}-func edgeAgentMTLSAssetsDirInternal(cfg *Config) (string, error) {- if cfg == nil {- return "", errors.New("edge config is required")- }- if configured := strings.TrimSpace(cfg.EdgeMTLSAssetsDir); configured != "" {- return configured, nil- }-- baseDir := defaultAgentMTLSDir- // Probes the fixed system path /app/data, which is not under any acfs root.- if _, err := os.Stat("/app/data"); err == nil {- baseDir = "/app/data/edge-mtls-agent"- }-- resolved, err := filepath.Abs(baseDir)- if err != nil {- return "", fmt.Errorf("failed to resolve edge agent mTLS assets dir: %w", err)- }- return resolved, nil-}--func ensureManagerCAInternal(ctx context.Context, assetsDir string) (string, string, bool, error) {- // The assets dir is the confinement root for everything below, so it is- // created through os before acfs opens it.+// ensureManagerCA returns the generated CA, creating it unless the existing one decrypts,+// matches its key, and uses a supported algorithm.+func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, bool, error) {+ // The assets dir is the acfs confinement root for callers, so os creates it.if err := os.MkdirAll(assetsDir, utils.DirPerm); err != nil {- return "", "", false, fmt.Errorf("failed to create edge mTLS assets dir: %w", err)+ return tls.Certificate{}, false, fmt.Errorf("failed to create edge mTLS assets dir: %w", err)}-- unlock, err := lockEdgeMTLSPathInternal(ctx, assetsDir, ".ca.lock")+ unlock, err := lockEdgeMTLSPath(ctx, assetsDir, ".ca.lock")if err != nil {- return "", "", false, err+ return tls.Certificate{}, false, err}defer unlock()- caCertPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)- caKeyPath := filepath.Join(assetsDir, generatedMTLSCAKeyFileName)- if generatedCAReadyInternal(caCertPath, caKeyPath) {- return caCertPath, caKeyPath, false, nil+ certPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)+ keyPath := filepath.Join(assetsDir, generatedMTLSCAKeyFileName)+ // os rather than acfs: the assets dir may be configured anywhere on the host.+ certPEM, certErr := os.ReadFile(certPath)+ rawKey, keyErr := os.ReadFile(keyPath)+ keyPEM := ""+ if ciphertext, encrypted := strings.CutPrefix(strings.TrimSpace(string(rawKey)), caKeyEncryptedPrefix); certErr == nil && keyErr == nil && encrypted {+ keyPEM, _ = libcrypto.Decrypt(ciphertext)+ }+ if ca, pairErr := tls.X509KeyPair(certPEM, []byte(keyPEM)); pairErr == nil && ca.Leaf.IsCA && supportedGeneratedKey(ca.Leaf.PublicKey) {+ return ca, false, nil}- _ = acfs.Remove(ctx, assetsDir, "/"+generatedMTLSCACertFileName)- _ = acfs.Remove(ctx, assetsDir, "/"+generatedMTLSCAKeyFileName)privateKey, err := certgen.GenerateMLDSA87PrivateKey()if err != nil {- return "", "", false, fmt.Errorf("failed to generate CA private key: %w", err)+ return tls.Certificate{}, false, fmt.Errorf("failed to generate CA private key: %w", err)}-template, err := certgen.NewEdgeMTLSCATemplate()if err != nil {- return "", "", false, err+ return tls.Certificate{}, false, err}-certDER, err := x509.CreateCertificate(rand.Reader, template, template, privateKey.PublicKey(), privateKey)if err != nil {- return "", "", false, fmt.Errorf("failed to create CA certificate: %w", err)+ return tls.Certificate{}, false, fmt.Errorf("failed to create CA certificate: %w", err)}-- if writePEMFileErr := writePEMFileInternal(caCertPath, "CERTIFICATE", certDER, utils.FilePerm); writePEMFileErr != nil {- return "", "", false, writePEMFileErr- }- caKeyDER, err := x509.MarshalPKCS8PrivateKey(privateKey)+ leaf, err := x509.ParseCertificate(certDER)if err != nil {- return "", "", false, fmt.Errorf("failed to marshal CA private key: %w", err)- }- if writeCAKeyFileErr := writeCAKeyFileInternal(caKeyPath, caKeyDER); writeCAKeyFileErr != nil {- return "", "", false, writeCAKeyFileErr+ return tls.Certificate{}, false, fmt.Errorf("failed to parse CA certificate: %w", err)}-- slog.InfoContext(ctx, "generated edge mTLS CA", "certPath", caCertPath)- return caCertPath, caKeyPath, true, nil-}--func generatedCAReadyInternal(caCertPath, caKeyPath string) bool {- if !fileExistsInternal(caCertPath) || !fileExistsInternal(caKeyPath) {- return false+ if writeErr := atomic.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), utils.FilePerm); writeErr != nil {+ return tls.Certificate{}, false, writeErr}- if err := validateGeneratedCAInternal(caCertPath, caKeyPath); err != nil {- return false- }- return true-}-func ensureClientCertificateInternal(ctx context.Context, assetsDir, envID, envName, appURL string) (string, string, bool, error) {- caCertPath, caKeyPath, _, err := ensureManagerCAInternal(ctx, assetsDir)+ keyDER, err := x509.MarshalPKCS8PrivateKey(privateKey)if err != nil {- return "", "", false, err+ return tls.Certificate{}, false, fmt.Errorf("failed to marshal CA private key: %w", err)}-- caCertPEM, err := readGeneratedAssetInternal(ctx, assetsDir, caCertPath)- if err != nil {- return "", "", false, fmt.Errorf("failed to read CA certificate: %w", err)- }- caKeyPEM, err := readCAKeyPEMInternal(caKeyPath)- if err != nil {- return "", "", false, err- }-- caCertBlock, _ := pem.Decode(caCertPEM)- if caCertBlock == nil {- return "", "", false, errors.New("failed to parse CA certificate PEM")- }- caCert, err := x509.ParseCertificate(caCertBlock.Bytes)- if err != nil {- return "", "", false, fmt.Errorf("failed to parse CA certificate: %w", err)- }-- caKey, err := parsePrivateKeyPEMInternal(caKeyPEM, "CA")- if err != nil {- return "", "", false, err- }-- safeEnvID := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envID), "_")- clientDir := filepath.Join(assetsDir, generatedClientMTLSSubdir, safeEnvID)- if mkdirAllErr := acfs.MkdirAll(ctx, assetsDir, path.Join("/", generatedClientMTLSSubdir, safeEnvID), utils.DirPerm); mkdirAllErr != nil {- return "", "", false, fmt.Errorf("failed to create client cert dir: %w", mkdirAllErr)- }- unlock, err := lockEdgeMTLSPathInternal(ctx, clientDir, ".client.lock")- if err != nil {- return "", "", false, err- }- defer unlock()-- clientCertPath := filepath.Join(clientDir, generatedMTLSClientCertName)- clientKeyPath := filepath.Join(clientDir, generatedMTLSClientKeyName)- expectedCommonName := buildGeneratedClientCommonNameInternal(envName, safeEnvID)- expectedURISAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)- if fileExistsInternal(clientCertPath) && fileExistsInternal(clientKeyPath) {- // The URI SAN is the stable edge identity. Common Name is display metadata- // for newly issued certs only, so environment renames must not rotate keys.- if validateGeneratedClientCertificateErr := validateGeneratedClientCertificateInternal(clientCertPath, clientKeyPath, "", expectedURISAN); validateGeneratedClientCertificateErr == nil {- return clientCertPath, clientKeyPath, false, nil- }- _ = acfs.Remove(ctx, clientDir, "/"+generatedMTLSClientCertName)- _ = acfs.Remove(ctx, clientDir, "/"+generatedMTLSClientKeyName)- }-- privateKey, err := generateKeyLikeInternal(caKey)- if err != nil {- return "", "", false, fmt.Errorf("failed to generate client private key: %w", err)- }- uriSAN, dnsSANs := buildGeneratedClientSANsInternal(envName, safeEnvID, appURL)- template, err := certgen.NewEdgeMTLSClientTemplate(expectedCommonName, uriSAN, dnsSANs)- if err != nil {- return "", "", false, err- }-- certDER, err := x509.CreateCertificate(rand.Reader, template, caCert, privateKey.Public(), caKey)+ ciphertext, err := caKeyEncrypt(string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})))if err != nil {- return "", "", false, fmt.Errorf("failed to create client certificate: %w", err)- }-- if writePEMFileErr := writePEMFileInternal(clientCertPath, "CERTIFICATE", certDER, utils.FilePerm); writePEMFileErr != nil {- return "", "", false, writePEMFileErr- }- // P-384 leaves keep the SEC1 framing so agents from before the ML-DSA- // migration can still parse keys issued during a rolling upgrade.- var clientKeyDER []byte- keyPEMType := "PRIVATE KEY"- if ecKey, ok := privateKey.(*ecdsa.PrivateKey); ok {- keyPEMType = "EC PRIVATE KEY"- clientKeyDER, err = x509.MarshalECPrivateKey(ecKey)- } else {- clientKeyDER, err = x509.MarshalPKCS8PrivateKey(privateKey)- }- if err != nil {- return "", "", false, fmt.Errorf("failed to marshal client private key: %w", err)- }- if writePEMFileErr2 := writePEMFileInternal(clientKeyPath, keyPEMType, clientKeyDER, 0o600); writePEMFileErr2 != nil {- return "", "", false, writePEMFileErr2- }-- return clientCertPath, clientKeyPath, true, nil-}--func buildGeneratedClientCommonNameInternal(envName, safeEnvID string) string {- safeEnvID = strings.TrimSpace(safeEnvID)- if safeEnvID == "" {- return ""- }-- safeEnvName := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envName), "-")- safeEnvName = strings.Trim(safeEnvName, "-_")- if safeEnvName == "" {- return safeEnvID- }-- const maxCommonNameLength = 64-- maxEnvNameLength := maxCommonNameLength - len(safeEnvID) - 1- if maxEnvNameLength <= 0 {- return safeEnvID- }- if len(safeEnvName) > maxEnvNameLength {- safeEnvName = strings.Trim(safeEnvName[:maxEnvNameLength], "-_")- if safeEnvName == "" {- return safeEnvID- }- }-- return fmt.Sprintf("%s-%s", safeEnvName, safeEnvID)-}--// buildGeneratedClientSANsInternal returns the URI and DNS Subject Alternative-// Names to embed in a generated edge agent client certificate. The URI SAN-// provides a stable machine-readable identity; DNS SANs improve interop with-// stricter verifiers. Returns a nil URI if safeEnvID is empty.-func buildGeneratedClientSANsInternal(envName, safeEnvID, appURL string) (*url.URL, []string) {- safeEnvID = strings.TrimSpace(safeEnvID)- if safeEnvID == "" {- return nil, nil- }-- uriSAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)- trustDomain := certgen.EdgeMTLSTrustDomain(appURL)-- dnsSANs := []string{"arcane-agent"}- safeEnvName := generatedAssetNameSanitizer.ReplaceAllString(strings.TrimSpace(envName), "-")- safeEnvName = strings.Trim(safeEnvName, "-_.")- if safeEnvName != "" && trustDomain != "" {- dnsSANs = append(dnsSANs, safeEnvName+".agent."+trustDomain)- }-- return uriSAN, dnsSANs-}--func validateGeneratedCAInternal(certPath, keyPath string) error {- cert, err := readCertificateInternal(certPath)- if err != nil {- return err- }- if !cert.IsCA {- return errors.New("generated CA certificate is not a CA")- }- if validateGeneratedKeyTypeErr := validateGeneratedKeyTypeInternal(cert.PublicKey, "generated CA certificate"); validateGeneratedKeyTypeErr != nil {- return validateGeneratedKeyTypeErr- }- keyPEM, err := readCAKeyPEMInternal(keyPath)- if err != nil {- return err- }- privateKey, err := parsePrivateKeyPEMInternal(keyPEM, "generated CA")- if err != nil {- return err- }- if validateCertificateKeyPairErr := validateCertificateKeyPairInternal(cert, privateKey, "generated CA"); validateCertificateKeyPairErr != nil {- return validateCertificateKeyPairErr- }- return nil-}--func validateGeneratedClientCertificateInternal(certPath, keyPath, expectedCommonName string, expectedURISAN *url.URL) error {- cert, err := readCertificateInternal(certPath)- if err != nil {- return err- }- now := time.Now()- if now.Before(cert.NotBefore) || !now.Before(cert.NotAfter) {- return errors.New("generated client certificate is not currently valid")- }- if strings.TrimSpace(expectedCommonName) != "" && cert.Subject.CommonName != expectedCommonName {- return fmt.Errorf("generated client certificate common name %q does not match expected %q", cert.Subject.CommonName, expectedCommonName)- }- if expectedURISAN != nil && !certificateHasURISANInternal(cert, expectedURISAN) {- return fmt.Errorf("generated client certificate URI SAN does not match expected %s", expectedURISAN.String())- }- if validateGeneratedKeyTypeErr := validateGeneratedKeyTypeInternal(cert.PublicKey, "generated client certificate"); validateGeneratedKeyTypeErr != nil {- return validateGeneratedKeyTypeErr- }- privateKey, err := readPrivateKeyInternal(keyPath)- if err != nil {- return err+ return tls.Certificate{}, false, fmt.Errorf("failed to encrypt edge mTLS CA private key: %w", err)}- if validateCertificateKeyPairErr := validateCertificateKeyPairInternal(cert, privateKey, "generated client"); validateCertificateKeyPairErr != nil {- return validateCertificateKeyPairErr- }- return nil-}--func certificateHasURISANInternal(cert *x509.Certificate, expected *url.URL) bool {- if cert == nil || expected == nil {- return false+ if ciphertext == "" {+ return tls.Certificate{}, false, errors.New("failed to encrypt edge mTLS CA private key: encrypted payload is empty")}- for _, uri := range cert.URIs {- if uri == nil {- continue- }- if strings.EqualFold(uri.Scheme, expected.Scheme) &&- strings.EqualFold(strings.TrimSuffix(uri.Host, "."), strings.TrimSuffix(expected.Host, ".")) &&- uri.Path == expected.Path {- return true- }+ if writeErr := atomic.WriteFile(keyPath, []byte(caKeyEncryptedPrefix+ciphertext), 0o600); writeErr != nil {+ return tls.Certificate{}, false, writeErr}- return false-}-func agentMTLSAssetsNeedEnrollmentInternal(certPath, keyPath string, now time.Time) (bool, string) {- if !fileExistsInternal(certPath) || !fileExistsInternal(keyPath) {- return true, "certificate or key is missing"- }- cert, err := readCertificateInternal(certPath)- if err != nil {- return true, err.Error()- }- privateKey, err := readPrivateKeyInternal(keyPath)- if err != nil {- return true, err.Error()- }- if validateCertificateKeyPairErr := validateCertificateKeyPairInternal(cert, privateKey, "edge mTLS client"); validateCertificateKeyPairErr != nil {- return true, validateCertificateKeyPairErr.Error()- }- if now.IsZero() {- now = time.Now()- }- if now.Before(cert.NotBefore) {- return true, "certificate is not valid before " + cert.NotBefore.UTC().Format(time.RFC3339)- }- if !now.Before(cert.NotAfter) {- return true, "certificate expired at " + cert.NotAfter.UTC().Format(time.RFC3339)- }- if now.Add(agentMTLSRenewBefore).After(cert.NotAfter) {- return true, "certificate expires soon at " + cert.NotAfter.UTC().Format(time.RFC3339)- }- return false, ""+ slog.InfoContext(ctx, "generated edge mTLS CA", "certPath", certPath)+ return tls.Certificate{Certificate: [][]byte{certDER}, PrivateKey: privateKey, Leaf: leaf}, true, nil}-func validateCertificateKeyPairInternal(cert *x509.Certificate, privateKey crypto.Signer, label string) error {- if cert == nil {- return fmt.Errorf("%s certificate is required", label)- }- if privateKey == nil {- return fmt.Errorf("%s private key is required", label)- }-- certPublicKeyDER, err := x509.MarshalPKIXPublicKey(cert.PublicKey)- if err != nil {- return fmt.Errorf("failed to marshal %s certificate public key: %w", label, err)+// loadClientCertificate loads a client key pair, or returns why it must be (re)issued: unreadable,+// mismatched, outside its validity window, or within agentMTLSRenewBefore of expiry.+func loadClientCertificate(certPath, keyPath string, now time.Time) (tls.Certificate, string) {+ cert, err := tls.LoadX509KeyPair(certPath, keyPath)+ switch {+ case err != nil:+ return tls.Certificate{}, err.Error()+ case now.Before(cert.Leaf.NotBefore):+ return tls.Certificate{}, "certificate is not valid before " + cert.Leaf.NotBefore.UTC().Format(time.RFC3339)+ case !now.Before(cert.Leaf.NotAfter):+ return tls.Certificate{}, "certificate expired at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)+ case now.Add(agentMTLSRenewBefore).After(cert.Leaf.NotAfter):+ return tls.Certificate{}, "certificate expires soon at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)}- privatePublicKeyDER, err := x509.MarshalPKIXPublicKey(privateKey.Public())- if err != nil {- return fmt.Errorf("failed to marshal %s private key public key: %w", label, err)- }- if !bytes.Equal(certPublicKeyDER, privatePublicKeyDER) {- return fmt.Errorf("%s certificate public key does not match private key", label)- }-- return nil+ return cert, ""}-func validateGeneratedKeyTypeInternal(publicKey any, label string) error {+// supportedGeneratedKey reports whether a generated certificate uses ECDSA P-384 or ML-DSA-87.+func supportedGeneratedKey(publicKey crypto.PublicKey) bool {switch key := publicKey.(type) {case *ecdsa.PublicKey:- if key.Curve == elliptic.P384() {- return nil- }+ return key.Curve == elliptic.P384()case *mldsa.PublicKey:- if key.Parameters() == mldsa.MLDSA87() {- return nil- }- }- return fmt.Errorf("%s is not ECDSA P-384 or ML-DSA-87", label)-}--func generateKeyLikeInternal(caKey crypto.Signer) (crypto.Signer, error) {- if _, ok := caKey.(*ecdsa.PrivateKey); ok {- return certgen.GenerateP384PrivateKey()- }- return certgen.GenerateMLDSA87PrivateKey()-}--func parsePrivateKeyPEMInternal(pemBytes []byte, label string) (crypto.Signer, error) {- block, _ := pem.Decode(pemBytes)- if block == nil {- return nil, fmt.Errorf("failed to parse %s private key PEM", label)- }- if key, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil {- signer, ok := key.(crypto.Signer)- if !ok {- return nil, fmt.Errorf("%s private key is not a signer", label)- }- return signer, nil+ return key.Parameters() == mldsa.MLDSA87()}- key, err := x509.ParseECPrivateKey(block.Bytes)- if err != nil {- return nil, fmt.Errorf("failed to parse %s private key: %w", label, err)- }- return key, nil-}--func readCertificateInternal(localPath string) (*x509.Certificate, error) {- // os.* rather than acfs: callers pass both generated-asset paths and- // user-configured absolute cert paths, so no single confinement root exists.- pemBytes, err := os.ReadFile(localPath)- if err != nil {- return nil, fmt.Errorf("failed to read certificate %s: %w", localPath, err)- }- block, _ := pem.Decode(pemBytes)- if block == nil {- return nil, fmt.Errorf("failed to parse certificate PEM %s", localPath)- }- cert, err := x509.ParseCertificate(block.Bytes)- if err != nil {- return nil, fmt.Errorf("failed to parse certificate %s: %w", localPath, err)- }- return cert, nil-}--func readPrivateKeyInternal(localPath string) (crypto.Signer, error) {- // os.* rather than acfs: callers pass both generated-asset paths and- // user-configured absolute key paths, so no single confinement root exists.- pemBytes, err := os.ReadFile(localPath)- if err != nil {- return nil, fmt.Errorf("failed to read private key %s: %w", localPath, err)- }- return parsePrivateKeyPEMInternal(pemBytes, localPath)+ return false}-func lockEdgeMTLSPathInternal(ctx context.Context, dir, lockName string) (func(), error) {+// lockEdgeMTLSPath serializes asset generation in this process and, through an O_EXCL lock+// file (which acfs cannot create), across processes sharing the directory.+func lockEdgeMTLSPath(ctx context.Context, dir, lockName string) (func(), error) {absDir, err := filepath.Abs(dir)if err != nil {return nil, fmt.Errorf("failed to resolve edge mTLS lock dir: %w", err)}- lockName = cmp.Or(strings.TrimSpace(lockName), ".lock")-lockPath := filepath.Join(absDir, lockName)deadline := time.Now().Add(managerCALockTimeout)for {- unlock, held := managerCALocks.TryLock(lockPath)- if !held {- if waitForEdgeMTLSLockPollErr := waitForEdgeMTLSLockPollInternal(ctx); waitForEdgeMTLSLockPollErr != nil {- return nil, waitForEdgeMTLSLockPollErr+ if unlock, held := managerCALocks.TryLock(lockPath); held {+ file, openErr := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)+ if openErr == nil {+ _, _ = fmt.Fprintf(file, "%d %s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano))+ _ = file.Close()+ return func() {+ _ = os.Remove(lockPath)+ unlock()+ }, nil}- continue- }- // os.* rather than acfs, here and in the stale-lock helpers below: acfs- // has no exclusive-create (O_EXCL) lockfile API.- file, openFileErr := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)- if openFileErr == nil {- _, _ = fmt.Fprintf(file, "%d %s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano))- _ = file.Close()- return func() {- _ = os.Remove(lockPath)- unlock()- }, nil- }- if !os.IsExist(openFileErr) {+ expired := time.Now().After(deadline)+ removedStale := os.IsExist(openErr) && expired && removeStaleLock(lockPath)unlock()- return nil, fmt.Errorf("failed to acquire edge mTLS CA lock: %w", openFileErr)- }- if time.Now().After(deadline) {- if removeStaleEdgeMTLSLockInternal(lockPath) {- // Retake the lock from the top; continuing while still holding it- // makes the next TryLock fail against ourselves and spin until the- // context is cancelled.- unlock()+ switch {+ case !os.IsExist(openErr):+ return nil, fmt.Errorf("failed to acquire edge mTLS CA lock: %w", openErr)+ case removedStale:+ // Retry at once with a fresh deadline now that the stale lock is gone.deadline = time.Now().Add(managerCALockTimeout)continue+ case expired:+ return nil, fmt.Errorf("timed out waiting for edge mTLS CA lock %s", lockPath)}- unlock()- return nil, fmt.Errorf("timed out waiting for edge mTLS CA lock %s", lockPath)}- unlock()- if waitForEdgeMTLSLockPollErr2 := waitForEdgeMTLSLockPollInternal(ctx); waitForEdgeMTLSLockPollErr2 != nil {- return nil, waitForEdgeMTLSLockPollErr2- }- }-}--func waitForEdgeMTLSLockPollInternal(ctx context.Context) error {- timer := time.NewTimer(managerCALockPollInterval)- defer timer.Stop()-- select {- case <-ctx.Done():- return fmt.Errorf("cancelled waiting for edge mTLS CA lock: %w", ctx.Err())- case <-timer.C:- return nil- }-}-func removeStaleEdgeMTLSLockInternal(lockPath string) bool {- info, err := readEdgeMTLSLockInfoInternal(lockPath)- if err != nil {- return false- }- if !info.createdAt.IsZero() && time.Since(info.createdAt) > 2*managerCALockTimeout {- return removeEdgeMTLSLockFileInternal(lockPath)- }- if edgeMTLSLockPIDAliveInternal(info.pid) {- return false+ select {+ case <-ctx.Done():+ return nil, fmt.Errorf("cancelled waiting for edge mTLS CA lock: %w", ctx.Err())+ case <-time.After(managerCALockPollInterval):+ }}- return removeEdgeMTLSLockFileInternal(lockPath)}-func readEdgeMTLSLockInfoInternal(lockPath string) (*edgeMTLSLockInfo, error) {+// removeStaleLock removes a lock file whose owning process is gone or that is far older than any holder.+func removeStaleLock(lockPath string) bool {content, err := os.ReadFile(lockPath)if err != nil {- return nil, err+ return false}fields := strings.Fields(string(content))if len(fields) == 0 {- return nil, errors.New("edge mTLS lock does not contain a PID")- }- pid, err := strconv.Atoi(fields[0])- if err != nil {- return nil, fmt.Errorf("parse edge mTLS lock PID: %w", err)- }- if pid <= 0 {- return nil, errors.New("edge mTLS lock PID must be positive")- }- info := &edgeMTLSLockInfo{pid: pid}- if len(fields) > 1 {- createdAt, parseErr := time.Parse(time.RFC3339Nano, fields[1])- if parseErr == nil {- info.createdAt = createdAt- }- }- return info, nil-}--func removeEdgeMTLSLockFileInternal(lockPath string) bool {- if err := os.Remove(lockPath); err != nil && !errors.Is(err, os.ErrNotExist) {return false}- return true-}--func edgeMTLSLockPIDAliveInternal(pid int) bool {- process, err := os.FindProcess(pid)- if err != nil {+ pid, err := strconv.Atoi(fields[0])+ if err != nil || pid <= 0 {return false}- err = process.Signal(syscall.Signal(0))- return err == nil || errors.Is(err, syscall.EPERM)-}--func writePEMFileInternal(localPath, blockType string, derBytes []byte, perm os.FileMode) error {- pemBytes := pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: derBytes})- if pemBytes == nil {- return fmt.Errorf("failed to encode PEM file %s", localPath)- }- return atomic.WriteFile(localPath, pemBytes, perm)-}-var caKeyEncryptInternal = libcrypto.Encrypt--// writeCAKeyFileInternal writes the edge CA private key to disk using envelope-// encryption via libcrypto.-func writeCAKeyFileInternal(localPath string, derBytes []byte) error {- pemBytes := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: derBytes})- if pemBytes == nil {- return errors.New("failed to encode CA private key to PEM")- }-- ciphertext, err := caKeyEncryptInternal(string(pemBytes))- if err != nil {- return fmt.Errorf("failed to encrypt edge mTLS CA private key: %w", err)+ expired := false+ if len(fields) > 1 {+ if createdAt, parseErr := time.Parse(time.RFC3339Nano, fields[1]); parseErr == nil {+ expired = time.Since(createdAt) > 2*managerCALockTimeout+ }}- if ciphertext == "" {- return errors.New("failed to encrypt edge mTLS CA private key: encrypted payload is empty")+ if !expired {+ // Signal 0 only probes the process; EPERM means it exists under another user.+ if process, findErr := os.FindProcess(pid); findErr == nil {+ if signalErr := process.Signal(syscall.Signal(0)); signalErr == nil || errors.Is(signalErr, syscall.EPERM) {+ return false+ }+ }}- return atomic.WriteFile(localPath, []byte(caKeyEncryptedPrefix+ciphertext), 0o600)+ removeErr := os.Remove(lockPath)+ return removeErr == nil || errors.Is(removeErr, os.ErrNotExist)}-// readCAKeyPEMInternal returns the plain PEM bytes of the edge CA private key,-// reading a libcrypto-envelope-encrypted file written by writeCAKeyFileInternal.-// os.* rather than acfs: callers pass paths derived from a possibly-// user-configured assets dir, so no confinement root handle is in scope.-func readCAKeyPEMInternal(localPath string) ([]byte, error) {- raw, err := os.ReadFile(localPath)- if err != nil {- return nil, fmt.Errorf("failed to read CA private key %s: %w", localPath, err)- }- trimmed := strings.TrimSpace(string(raw))- if !strings.HasPrefix(trimmed, caKeyEncryptedPrefix) {- return nil, fmt.Errorf("CA private key %s is not in the expected encrypted envelope format", localPath)- }- ciphertext := strings.TrimPrefix(trimmed, caKeyEncryptedPrefix)- plaintext, err := libcrypto.Decrypt(ciphertext)- if err != nil {- return nil, fmt.Errorf("failed to decrypt CA private key %s: %w", localPath, err)- }- return []byte(plaintext), nil-}--func fileExistsInternal(localPath string) bool {- if strings.TrimSpace(localPath) == "" {- return false- }- // os.* rather than acfs: callers probe both generated-asset paths and- // user-configured absolute paths, so no single confinement root exists.+// fileExists reports whether localPath is a regular file. Paths may be configured anywhere on the host.+func fileExists(localPath string) bool {info, err := os.Stat(localPath)return err == nil && !info.IsDir()}diff --git a/backend/pkg/libarcane/edge/tls_test.go b/backend/pkg/libarcane/edge/tls_test.goindex b2de181f4..e7e6f8861 100644--- a/backend/pkg/libarcane/edge/tls_test.go+++ b/backend/pkg/libarcane/edge/tls_test.go@@ -25,16 +25,10 @@ import ("github.com/stretchr/testify/require"kit "go.getarcane.app/kit/pkg"libcrypto "go.getarcane.app/sys/crypto"- "google.golang.org/grpc/credentials"- "google.golang.org/grpc/peer""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")-type testAuthInfo struct{}--func (testAuthInfo) AuthType() string { return "test" }-func TestMain(m *testing.M) {libcrypto.InitEncryption(&libcrypto.Config{EncryptionKey: "test-encryption-key-for-edge-mtls-32bytes-min",@@ -61,6 +55,14 @@ func TestPrepareManagerMTLSAssetsWithContext(t *testing.T) {require.NoError(t, PrepareManagerMTLSAssetsWithContext(t.Context(), cfg))require.NotEmpty(t, cfg.EdgeMTLSCAFile)require.FileExists(t, cfg.EdgeMTLSCAFile)++ configured := &Config{+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSAssetsDir: t.TempDir(),+ EdgeMTLSCAFile: filepath.Join(t.TempDir(), "ca.crt"),+ }+ require.NoError(t, PrepareManagerMTLSAssetsWithContext(t.Context(), configured))+ require.NoFileExists(t, filepath.Join(configured.EdgeMTLSAssetsDir, generatedMTLSCACertFileName))}func TestGenerateManagerClientMTLSAssetsWithContext(t *testing.T) {@@ -136,10 +138,10 @@ func TestGeneratedClientCertificate_IncludesSANs(t *testing.T) {uri := cert.URIs[0]require.Equal(t, "spiffe", uri.Scheme)require.Equal(t, "manager.example.com", uri.Host)- require.Contains(t, uri.Path, "env-abc")+ require.Equal(t, "/edge/env-abc", uri.Path)- require.NotEmpty(t, cert.DNSNames, "agent cert must include at least one DNS SAN")require.Contains(t, cert.DNSNames, "arcane-agent")+ require.Contains(t, cert.DNSNames, "Lab-Server.agent.manager.example.com")require.True(t, cert.NotBefore.Before(cert.NotAfter))skew := cert.NotAfter.Sub(cert.NotBefore)@@ -157,6 +159,10 @@ func TestEnsureAgentMTLSAssets_RejectsPlainHTTPEnrollment(t *testing.T) {err := EnsureAgentMTLSAssets(t.Context(), cfg)require.Error(t, err)require.Contains(t, err.Error(), "MANAGER_API_URL to use https for certificate enrollment")++ cfg.EdgeMTLSCertFile = filepath.Join(t.TempDir(), "agent.crt")+ cfg.EdgeMTLSKeyFile = filepath.Join(t.TempDir(), "agent.key")+ require.NoError(t, EnsureAgentMTLSAssets(t.Context(), cfg), "configured client files skip enrollment")}func TestEnsureAgentMTLSAssets_LimitsEnrollmentErrorBody(t *testing.T) {@@ -217,31 +223,31 @@ func TestEnsureAgentMTLSAssets_UsesDownloadedCAPathWhenPresent(t *testing.T) {require.Equal(t, caPath, cfg.EdgeMTLSCAFile)}-func TestRemoveStaleEdgeMTLSLockInternal_PreservesLivePID(t *testing.T) {+func TestRemoveStaleLock_PreservesLivePID(t *testing.T) {lockPath := filepath.Join(t.TempDir(), ".ca.lock")require.NoError(t, os.WriteFile(lockPath, fmt.Appendf(nil, "%d %s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano)), 0o600))- require.False(t, removeStaleEdgeMTLSLockInternal(lockPath))+ require.False(t, removeStaleLock(lockPath))require.FileExists(t, lockPath)}-func TestRemoveStaleEdgeMTLSLockInternal_RemovesOldLockDespiteLivePID(t *testing.T) {+func TestRemoveStaleLock_RemovesOldLockDespiteLivePID(t *testing.T) {lockPath := filepath.Join(t.TempDir(), ".ca.lock")oldTimestamp := time.Now().Add(-(2*managerCALockTimeout + time.Second)).UTC().Format(time.RFC3339Nano)require.NoError(t, os.WriteFile(lockPath, fmt.Appendf(nil, "%d %s\n", os.Getpid(), oldTimestamp), 0o600))- require.True(t, removeStaleEdgeMTLSLockInternal(lockPath))+ require.True(t, removeStaleLock(lockPath))require.NoFileExists(t, lockPath)}-func TestBuildManagerClientTLSConfigInternal_OptionalIgnoresBrokenClientCertificate(t *testing.T) {+func TestBuildManagerClientTLSConfig_OptionalIgnoresBrokenClientCertificate(t *testing.T) {assetsDir := t.TempDir()certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)require.NoError(t, os.WriteFile(certPath, []byte("not a cert"), 0o644))require.NoError(t, os.WriteFile(keyPath, []byte("not a key"), 0o600))- tlsConfig, err := buildManagerClientTLSConfigInternal(&Config{+ tlsConfig, err := buildManagerClientTLSConfig(&Config{ManagerApiUrl: "https://manager.example.com",EdgeMTLSMode: EdgeMTLSModeOptional,EdgeMTLSCertFile: certPath,@@ -252,21 +258,7 @@ func TestBuildManagerClientTLSConfigInternal_OptionalIgnoresBrokenClientCertificrequire.Empty(t, tlsConfig.Certificates)}-func TestVerifiedPeerCertificateEnvironmentIDMatchesInternal(t *testing.T) {- uriSAN, err := url.Parse("spiffe://manager.example.com/edge/env-123")- require.NoError(t, err)- cert := &x509.Certificate{URIs: []*url.URL{uriSAN}}- state := &tls.ConnectionState{- PeerCertificates: []*x509.Certificate{cert},- VerifiedChains: [][]*x509.Certificate{{cert}},- }-- require.NoError(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(state, "env-123", "manager.example.com"))- require.Error(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(state, "env-456", "manager.example.com"))- require.Error(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(state, "env-123", "other.example.com"))-}--func TestTunnelServerRequireCertificateIdentityInternal_RejectsWrongEnvironmentURI(t *testing.T) {+func TestTunnelServerRequireCertificateIdentity_RejectsWrongEnvironmentURI(t *testing.T) {uriSAN, err := url.Parse("spiffe://manager.example.com/edge/env-a")require.NoError(t, err)cert := &x509.Certificate{URIs: []*url.URL{uriSAN}}@@ -275,13 +267,16 @@ func TestTunnelServerRequireCertificateIdentityInternal_RejectsWrongEnvironmentUVerifiedChains: [][]*x509.Certificate{{cert}},}server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)- server.SetConfig(&Config{+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired,AppURL: "https://manager.example.com",- })+ }++ require.NoError(t, server.requireCertificateIdentity(state, "env-a"))+ require.Error(t, server.requireCertificateIdentity(state, "env-b"))- require.NoError(t, server.requireCertificateIdentityInternal(state, "env-a"))- require.Error(t, server.requireCertificateIdentityInternal(state, "env-b"))+ server.Config.AppURL = "https://other.example.com"+ require.Error(t, server.requireCertificateIdentity(state, "env-a"), "a different trust domain must not match")}func TestValidateManagerMTLSConfig_DoesNotRequireArcaneTLSTermination(t *testing.T) {@@ -290,8 +285,9 @@ func TestValidateManagerMTLSConfig_DoesNotRequireArcaneTLSTermination(t *testing}))assetsDir := t.TempDir()- caPath, _, _, err := ensureManagerCAInternal(t.Context(), assetsDir)+ _, _, err := ensureManagerCA(t.Context(), assetsDir)require.NoError(t, err)+ caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)err = ValidateManagerMTLSConfig(&Config{EdgeMTLSMode: EdgeMTLSModeRequired,@@ -325,33 +321,33 @@ func TestValidateManagerMTLSConfig_RejectsMissingOrMalformedCAFile(t *testing.T)func TestTunnelServerRequiredMTLS_AllowsHTTPRequestsWithoutVisibleTLSState(t *testing.T) {server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)- server.SetConfig(&Config{+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired,AppURL: "https://manager.example.com",- })+ }req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)- require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))+ require.NoError(t, server.requireCertificateIdentity(req.TLS, "env-a"))}func TestTunnelServerRequiredMTLS_RejectsDirectTLSWithoutVerifiedClientCertificate(t *testing.T) {server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)- server.SetConfig(&Config{EdgeMTLSMode: EdgeMTLSModeRequired})+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired}req := httptest.NewRequest(http.MethodGet, "https://manager.example.com/api/tunnel/connect", http.NoBody)req.TLS = &tls.ConnectionState{}- require.ErrorContains(t, server.requireRequestCertificateIdentityInternal(req, "env-a"), "verified edge mTLS client certificate is required")+ require.ErrorContains(t, server.requireCertificateIdentity(req.TLS, "env-a"), "verified edge mTLS client certificate is required")- server.SetConfig(&Config{EdgeMTLSMode: EdgeMTLSModeOptional})- require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeOptional}+ require.NoError(t, server.requireCertificateIdentity(req.TLS, "env-a"))}func TestTunnelServerRequiredMTLS_DetectsOnlyDirectTLSForRequestSecurityMode(t *testing.T) {server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)- server.SetConfig(&Config{+ server.Config = &Config{EdgeMTLSMode: EdgeMTLSModeRequired,AppURL: "https://manager.example.com",- })+ }cert := &x509.Certificate{}req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)@@ -359,73 +355,15 @@ func TestTunnelServerRequiredMTLS_DetectsOnlyDirectTLSForRequestSecurityMode(t *PeerCertificates: []*x509.Certificate{cert},VerifiedChains: [][]*x509.Certificate{{cert}},}- require.Equal(t, "mtls", requestSecurityModeInternal(req))+ require.True(t, hasVerifiedPeerCertificate(req.TLS))req = httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)req.Header.Set("X-SSL-Client-Verify", "SUCCESS")- require.Equal(t, "token", requestSecurityModeInternal(req))- require.NoError(t, server.requireRequestCertificateIdentityInternal(req, "env-a"))-}--func TestTunnelServerRequiredMTLS_IgnoresProxyVerificationHeaders(t *testing.T) {- tests := []struct {- name string- value string- }{- {name: "empty", value: ""},- {name: "success", value: "SUCCESS"},- {name: "true", value: "true"},- {name: "none", value: "NONE"},- {name: "failed", value: "FAILED"},- {name: "unknown", value: "probably"},- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- req := httptest.NewRequest(http.MethodGet, "/api/tunnel/connect", http.NoBody)- req.Header.Set("X-SSL-Client-Verify", tt.value)-- require.Equal(t, "token", requestSecurityModeInternal(req))- })- }+ require.False(t, hasVerifiedPeerCertificate(req.TLS))+ require.NoError(t, server.requireCertificateIdentity(req.TLS, "env-a"))}-func TestTunnelServerRequiredMTLS_AllowsGRPCContextsWithoutVisibleTLSState(t *testing.T) {- server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)- server.SetConfig(&Config{EdgeMTLSMode: EdgeMTLSModeRequired})-- t.Run("missing peer", func(t *testing.T) {- require.NoError(t, server.requireCertificateIdentityFromContextInternal(t.Context(), "env-a"))- })-- t.Run("non tls peer", func(t *testing.T) {- ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: testAuthInfo{}})- require.NoError(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"))- })-- t.Run("direct tls without verified client certificate", func(t *testing.T) {- ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{}})- require.ErrorContains(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"), "verified edge mTLS client certificate is required")- })-- t.Run("verified tls peer checks identity", func(t *testing.T) {- uriSAN, err := url.Parse("spiffe://manager.example.com/edge/env-a")- require.NoError(t, err)- cert := &x509.Certificate{URIs: []*url.URL{uriSAN}}- ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{- PeerCertificates: []*x509.Certificate{cert},- VerifiedChains: [][]*x509.Certificate{{cert}},- }}})- server.SetConfig(&Config{- EdgeMTLSMode: EdgeMTLSModeRequired,- AppURL: "https://manager.example.com",- })- require.NoError(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-a"))- require.Error(t, server.requireCertificateIdentityFromContextInternal(ctx, "env-b"))- })-}--func TestAgentMTLSAssetsNeedEnrollmentInternal_RenewsExpiredCertificate(t *testing.T) {+func TestLoadClientCertificate_RenewsExpiredCertificate(t *testing.T) {assetsDir := t.TempDir()assets, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{EdgeMTLSMode: EdgeMTLSModeRequired,@@ -440,58 +378,54 @@ func TestAgentMTLSAssetsNeedEnrollmentInternal_RenewsExpiredCertificate(t *testirequire.NoError(t, os.WriteFile(targetPath, []byte(file.Content), perm))}- needsEnrollment, reason := agentMTLSAssetsNeedEnrollmentInternal(- filepath.Join(assetsDir, generatedMTLSClientCertName),- filepath.Join(assetsDir, generatedMTLSClientKeyName),- time.Now().Add(generatedMTLSCertValidity+24*time.Hour),- )- require.True(t, needsEnrollment)+ certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)+ keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now())+ require.Empty(t, reason)++ _, reason = loadClientCertificate(certPath, keyPath, cert.Leaf.NotAfter.Add(24*time.Hour))require.Contains(t, reason, "expired")}-func TestValidateGeneratedClientCertificateInternal_RejectsMismatchedKeyPair(t *testing.T) {- assetsDir := t.TempDir()-- clientCertPath, _, _, err := ensureClientCertificateInternal(t.Context(), assetsDir, "env-123", "Lab Server", "https://manager.example.com")+func TestGenerateManagerClientMTLSAssets_ReissuesMismatchedKeyPair(t *testing.T) {+ cfg := &Config{+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSAssetsDir: t.TempDir(),+ AppURL: "https://manager.example.com",+ }+ _, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "Lab Server")require.NoError(t, err)replacementKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)require.NoError(t, err)-replacementKeyDER, err := x509.MarshalECPrivateKey(replacementKey)require.NoError(t, err)+ clientKeyPath := filepath.Join(cfg.EdgeMTLSAssetsDir, generatedClientMTLSSubdir, "env-123", generatedMTLSClientKeyName)+ require.NoError(t, os.WriteFile(clientKeyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: replacementKeyDER}), 0o600))- clientKeyPath := filepath.Join(assetsDir, generatedClientMTLSSubdir, "env-123", generatedMTLSClientKeyName)- err = writePEMFileInternal(clientKeyPath, "EC PRIVATE KEY", replacementKeyDER, 0o600)+ assets, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "Lab Server")require.NoError(t, err)-- expectedURI, err := url.Parse("spiffe://manager.example.com/edge/env-123")+ require.True(t, assets.CertIssued, "a key that no longer matches the certificate must be reissued")+ _, err = tls.X509KeyPair([]byte(assets.Files[1].Content), []byte(assets.Files[2].Content))require.NoError(t, err)-- err = validateGeneratedClientCertificateInternal(clientCertPath, clientKeyPath, "Lab-Server-env-123", expectedURI)- require.Error(t, err)- require.Contains(t, err.Error(), "does not match private key")}-func TestEnsureClientCertificateInternal_PreservesCertificateWhenEnvironmentNameChanges(t *testing.T) {- assetsDir := t.TempDir()-- clientCertPath, _, _, err := ensureClientCertificateInternal(t.Context(), assetsDir, "env-123", "", "https://manager.example.com")- require.NoError(t, err)-- originalPEM, err := os.ReadFile(clientCertPath)- require.NoError(t, err)-- clientCertPath, _, _, err = ensureClientCertificateInternal(t.Context(), assetsDir, "env-123", "Lab Server", "https://manager.example.com")+func TestGenerateManagerClientMTLSAssets_PreservesCertificateWhenEnvironmentNameChanges(t *testing.T) {+ cfg := &Config{+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSAssetsDir: t.TempDir(),+ AppURL: "https://manager.example.com",+ }+ original, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "")require.NoError(t, err)- updatedPEM, err := os.ReadFile(clientCertPath)+ renamed, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), cfg, "env-123", "Lab Server")require.NoError(t, err)- require.Equal(t, string(originalPEM), string(updatedPEM))+ require.False(t, renamed.CertIssued)+ require.Equal(t, original.Files[1].Content, renamed.Files[1].Content)- certBlock, _ := pem.Decode(updatedPEM)+ certBlock, _ := pem.Decode([]byte(renamed.Files[1].Content))require.NotNil(t, certBlock)-cert, err := x509.ParseCertificate(certBlock.Bytes)require.NoError(t, err)require.Equal(t, "env-123", cert.Subject.CommonName)@@ -501,114 +435,73 @@ func TestCAKey_EncryptedOnDiskWhenCryptoInitialized(t *testing.T) {initEdgeTestCrypto(t)assetsDir := t.TempDir()- caCertPath, caKeyPath, _, err := ensureManagerCAInternal(t.Context(), assetsDir)+ ca, generated, err := ensureManagerCA(t.Context(), assetsDir)require.NoError(t, err)- require.FileExists(t, caCertPath)- require.FileExists(t, caKeyPath)+ require.True(t, generated)- raw, err := os.ReadFile(caKeyPath)+ raw, err := os.ReadFile(filepath.Join(assetsDir, generatedMTLSCAKeyFileName))require.NoError(t, err)- require.NotContains(t, string(raw), "BEGIN PRIVATE KEY",- "encrypted CA key file must not contain plain PEM markers")+ require.True(t, strings.HasPrefix(string(raw), caKeyEncryptedPrefix))+ require.NotContains(t, string(raw), "BEGIN PRIVATE KEY", "encrypted CA key file must not contain plain PEM markers")- pemBytes, err := readCAKeyPEMInternal(caKeyPath)- require.NoError(t, err)- block, _ := pem.Decode(pemBytes)- require.NotNil(t, block)- _, err = x509.ParsePKCS8PrivateKey(block.Bytes)+ reused, generated, err := ensureManagerCA(t.Context(), assetsDir)require.NoError(t, err)+ require.False(t, generated, "the encrypted CA key must decrypt and be reused")+ require.Equal(t, ca.Certificate, reused.Certificate)- clientCertPath, clientKeyPath, _, err := ensureClientCertificateInternal(t.Context(), assetsDir, "env-round", "Round Trip", "https://manager.example.com")+ _, err = GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSAssetsDir: assetsDir,+ AppURL: "https://manager.example.com",+ }, "env-round", "Round Trip")require.NoError(t, err)- require.FileExists(t, clientCertPath)- require.FileExists(t, clientKeyPath)}func TestCAKey_EncryptionFailureReturnsError(t *testing.T) {- originalEncrypt := caKeyEncryptInternal+ originalEncrypt := caKeyEncryptt.Cleanup(func() {- caKeyEncryptInternal = originalEncrypt+ caKeyEncrypt = originalEncrypt})-- caKeyPath := filepath.Join(t.TempDir(), "ca.key")- caKeyEncryptInternal = func(string) (string, error) {+ caKeyEncrypt = func(string) (string, error) {return "", errors.New("encrypt failed")}- err := writeCAKeyFileInternal(caKeyPath, []byte{1, 2, 3})+ assetsDir := t.TempDir()+ _, _, err := ensureManagerCA(t.Context(), assetsDir)require.Error(t, err)require.Contains(t, err.Error(), "failed to encrypt edge mTLS CA private key")- require.NoFileExists(t, caKeyPath)+ require.NoFileExists(t, filepath.Join(assetsDir, generatedMTLSCAKeyFileName))}-func TestLockEdgeMTLSPathInternal_ReturnsOnContextCancellation(t *testing.T) {+func TestLockEdgeMTLSPath_ReturnsOnContextCancellation(t *testing.T) {assetsDir := t.TempDir()- unlock, err := lockEdgeMTLSPathInternal(t.Context(), assetsDir, ".ca.lock")+ unlock, err := lockEdgeMTLSPath(t.Context(), assetsDir, ".ca.lock")require.NoError(t, err)defer unlock()ctx, cancel := context.WithCancel(t.Context())cancel()- _, err = lockEdgeMTLSPathInternal(ctx, assetsDir, ".ca.lock")+ _, err = lockEdgeMTLSPath(ctx, assetsDir, ".ca.lock")require.Error(t, err)require.Contains(t, err.Error(), "cancelled waiting for edge mTLS CA lock")}func TestCAKey_EmptyEncryptionPayloadReturnsError(t *testing.T) {- originalEncrypt := caKeyEncryptInternal+ originalEncrypt := caKeyEncryptt.Cleanup(func() {- caKeyEncryptInternal = originalEncrypt+ caKeyEncrypt = originalEncrypt})-- caKeyPath := filepath.Join(t.TempDir(), "ca.key")- caKeyEncryptInternal = func(string) (string, error) {+ caKeyEncrypt = func(string) (string, error) {return "", nil}- err := writeCAKeyFileInternal(caKeyPath, []byte{1, 2, 3})+ assetsDir := t.TempDir()+ _, _, err := ensureManagerCA(t.Context(), assetsDir)require.Error(t, err)require.Contains(t, err.Error(), "encrypted payload is empty")- require.NoFileExists(t, caKeyPath)-}--func TestBuildGeneratedClientSANsInternal(t *testing.T) {- uri, dns := buildGeneratedClientSANsInternal("Lab Server", "env-123", "https://manager.example.com")- require.NotNil(t, uri)- require.Equal(t, "spiffe", uri.Scheme)- require.Equal(t, "manager.example.com", uri.Host)- require.Equal(t, "/edge/env-123", uri.Path)- require.Contains(t, dns, "arcane-agent")- require.Contains(t, dns, "Lab-Server.agent.manager.example.com")-- uriEmpty, dnsEmpty := buildGeneratedClientSANsInternal("", "", "https://manager.example.com")- require.Nil(t, uriEmpty)- require.Nil(t, dnsEmpty)-}--func TestShouldAutoGenerateManagerCAInternal(t *testing.T) {- cfg := &Config{- EdgeMTLSMode: EdgeMTLSModeRequired,- EdgeMTLSAssetsDir: t.TempDir(),- }-- require.True(t, shouldAutoGenerateManagerCAInternal(cfg))-- cfg.EdgeMTLSCAFile = filepath.Join(t.TempDir(), "ca.crt")- require.False(t, shouldAutoGenerateManagerCAInternal(cfg))-}--func TestShouldAutoEnrollAgentMTLSInternal(t *testing.T) {- cfg := &Config{- EdgeMTLSMode: EdgeMTLSModeRequired,- }-- require.True(t, shouldAutoEnrollAgentMTLSInternal(cfg))-- cfg.EdgeMTLSCertFile = filepath.Join(t.TempDir(), "agent.crt")- cfg.EdgeMTLSKeyFile = filepath.Join(t.TempDir(), "agent.key")- require.False(t, shouldAutoEnrollAgentMTLSInternal(cfg))+ require.NoFileExists(t, filepath.Join(assetsDir, generatedMTLSCAKeyFileName))}func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.T) {@@ -622,22 +515,15 @@ func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.require.NoError(t, err)caCertPath := filepath.Join(assetsDir, "ca.crt")- caCertPEM, err := os.ReadFile(caCertPath)- require.NoError(t, err)- caBlock, _ := pem.Decode(caCertPEM)- require.NotNil(t, caBlock)- caCert, err := x509.ParseCertificate(caBlock.Bytes)- require.NoError(t, err)- caKeyPEM, err := readCAKeyPEMInternal(filepath.Join(assetsDir, "ca.key"))- require.NoError(t, err)- caKey, err := parsePrivateKeyPEMInternal(caKeyPEM, "CA")+ ca, caGenerated, err := ensureManagerCA(t.Context(), assetsDir)require.NoError(t, err)+ require.False(t, caGenerated)serverKey, err := certgen.GenerateMLDSA87PrivateKey()require.NoError(t, err)serverTemplate, err := certgen.NewServerTLSTemplate("localhost", []string{"localhost", "127.0.0.1"})require.NoError(t, err)- serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caCert, serverKey.PublicKey(), caKey)+ serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, ca.Leaf, serverKey.PublicKey(), ca.PrivateKey)require.NoError(t, err)serverTLS, err := BuildManagerServerTLSConfig(&Config{@@ -650,7 +536,7 @@ func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.clientCertPath, err := GeneratedManagerClientMTLSCertPath(cfg, "env-hs")require.NoError(t, err)- clientTLS, err := buildManagerClientTLSConfigInternal(&Config{+ clientTLS, err := buildManagerClientTLSConfig(&Config{EdgeMTLSMode: EdgeMTLSModeRequired,EdgeMTLSCAFile: caCertPath,EdgeMTLSCertFile: clientCertPath,@@ -694,6 +580,8 @@ func TestGeneratedMLDSAAssets_HandshakeWithVerifiedClientCertificate(t *testing.result := <-resultChrequire.NoError(t, result.err)require.Equal(t, uint16(tls.VersionTLS13), result.state.Version)- require.True(t, hasVerifiedPeerCertificateInternal(&result.state))- require.NoError(t, verifiedPeerCertificateEnvironmentIDMatchesInternal(&result.state, "env-hs", "manager.example.com"))+ require.True(t, hasVerifiedPeerCertificate(&result.state))+ server := NewTunnelServerWithRegistry(GetRegistry(), nil, nil)+ server.Config = cfg+ require.NoError(t, server.requireCertificateIdentity(&result.state, "env-hs"))}diff --git a/backend/pkg/libarcane/edge/tunnel.go b/backend/pkg/libarcane/edge/tunnel.goindex 2a61d89d1..232b5abb2 100644--- a/backend/pkg/libarcane/edge/tunnel.go+++ b/backend/pkg/libarcane/edge/tunnel.go@@ -7,6 +7,7 @@ import ("io""log/slog""net"+ "sync/atomic""github.com/coder/websocket""google.golang.org/grpc/codes"@@ -26,9 +27,6 @@ const (// gRPC limit, since JSON base64-encodes binary bodies a gRPC peer would accept.maxWebSocketTunnelMessageSize = 2 * maxGRPCTunnelMessageSize- // ErrTunnelConnectionClosed is returned by Send and Receive on every transport- // once the tunnel connection is closed.-// MessageTypeRequest is sent from manager to agent to initiate a request.MessageTypeRequest TunnelMessageType = "request"// MessageTypeResponse is sent from agent to manager with the response.@@ -69,15 +67,17 @@ const (MessageTypeStreamClose TunnelMessageType = "stream_close"// MessageTypeCancelRequest requests cancellation of an in-flight command.MessageTypeCancelRequest TunnelMessageType = "cancel_request"+ // MessageTypeCommandCredit returns flow-control credit for consumed command output.+ MessageTypeCommandCredit TunnelMessageType = "command_credit")-var ErrTunnelConnectionClosed = errors.New("edge tunnel connection is closed")--// tunnelReadWait bounds how long a websocket tunnel read may sit idle. Both-// peers see traffic at least every DefaultHeartbeatInterval (agent heartbeat,-// manager heartbeat_ack), so 3x tolerates transient stalls while still-// detecting a silently dead peer. Variable so tests can shorten it.-var tunnelReadWait = 3 * DefaultHeartbeatInterval+var (+ // ErrTunnelConnectionClosed is returned by Send and Receive on every transport once the connection is closed.+ ErrTunnelConnectionClosed = errors.New("edge tunnel connection is closed")+ // tunnelReadWait bounds an idle websocket read; heartbeats arrive every DefaultHeartbeatInterval.+ // It is a variable so tests can shorten it.+ tunnelReadWait = 3 * DefaultHeartbeatInterval+)// TunnelConnection is the transport contract shared by WebSocket and gRPC wrappers.type TunnelConnection interface {@@ -143,18 +143,7 @@ func (t *TunnelConn) Receive() (*TunnelMessage, error) {// IsExpectedReceiveError returns true for normal WebSocket close/teardown errors.func (t *TunnelConn) IsExpectedReceiveError(err error) bool {- if err == nil {- return false- }-- if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {- return true- }- if errors.Is(err, net.ErrClosed) || errors.Is(err, ErrTunnelConnectionClosed) {- return true- }-- return wshub.IsExpectedClose(err)+ return isExpectedReceiveError(err)}// Close closes the WebSocket tunnel connection, sending a close frame on the@@ -208,6 +197,7 @@ func NewGRPCManagerTunnelConn(stream grpcManagerStream) *GRPCManagerTunnelConn {stream: &cancelableGRPCManagerStream{stream: stream,ctx: recvCtx,+ recvCh: make(chan grpcRecvResult),},cancel: cancel,}@@ -222,61 +212,34 @@ func (t *GRPCManagerTunnelConn) Send(msg *TunnelMessage) error {return ErrTunnelConnectionClosed}- protoMsg, err := tunnelMessageToManagerProto(msg, t.parityEncoding.Load())+ protoMsg, err := tunnelMessageToManagerProto(msg, t.parity)if err != nil {return err}if sendErr := t.stream.Send(protoMsg); sendErr != nil {- t.markClosed()+ t.closed.Store(true)return sendErr}return nil}-// SetParityEncoding enables native proto encodings for message types that are-// otherwise re-encoded for legacy agents. Set when the agent advertises the-// proto-parity-v1 capability during registration.-func (t *GRPCManagerTunnelConn) SetParityEncoding(enabled bool) {- t.parityEncoding.Store(enabled)-}-// Receive receives an agent->manager tunnel message from gRPC.func (t *GRPCManagerTunnelConn) Receive() (*TunnelMessage, error) {if t.stream == nil {return nil, ErrTunnelConnectionClosed}-- for {- protoMsg, err := t.stream.Recv()- if err != nil {- // A gRPC stream is dead after any Recv error, not just io.EOF.- t.markClosed()- return nil, err- }-- msg, err := agentProtoToTunnelMessage(protoMsg)- if err != nil {- if errors.Is(err, errUnknownTunnelPayload) {- // A newer peer sent a payload this build cannot decode yet;- // skip it like unknown websocket message types are skipped.- slog.DebugContext(t.stream.Context(), "Ignoring unknown edge tunnel payload", "error", err)- continue- }- return nil, err- }- return msg, nil- }+ return receiveGRPC(t.stream.Context(), t.stream.Recv, agentProtoToTunnelMessage, &t.closed)}// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.func (t *GRPCManagerTunnelConn) IsExpectedReceiveError(err error) bool {- return isExpectedGRPCReceiveErrorInternal(err)+ return isExpectedReceiveError(err)}// Close marks the stream closed on manager side.func (t *GRPCManagerTunnelConn) Close() error {- t.markClosed()+ t.closed.Store(true)if t.cancel != nil {t.cancel()}@@ -288,10 +251,6 @@ func (t *GRPCManagerTunnelConn) IsClosed() bool {return t.closed.Load()}-func (t *GRPCManagerTunnelConn) markClosed() {- t.closed.Store(true)-}-// Transport identifies the underlying tunnel transport.func (t *GRPCManagerTunnelConn) Transport() string {return EdgeTransportGRPC@@ -322,7 +281,7 @@ func (t *GRPCAgentTunnelConn) Send(msg *TunnelMessage) error {}if sendErr := t.stream.Send(protoMsg); sendErr != nil {- t.markClosed()+ t.closed.Store(true)return sendErr}return nil@@ -333,37 +292,16 @@ func (t *GRPCAgentTunnelConn) Receive() (*TunnelMessage, error) {if t.stream == nil {return nil, ErrTunnelConnectionClosed}-- for {- protoMsg, err := t.stream.Recv()- if err != nil {- // A gRPC stream is dead after any Recv error, not just io.EOF.- t.markClosed()- return nil, err- }-- msg, err := managerProtoToTunnelMessage(protoMsg)- if err != nil {- if errors.Is(err, errUnknownTunnelPayload) {- // A newer peer sent a payload this build cannot decode yet;- // skip it like unknown websocket message types are skipped.- slog.DebugContext(t.stream.Context(), "Ignoring unknown edge tunnel payload", "error", err)- continue- }- return nil, err- }- return msg, nil- }+ return receiveGRPC(t.stream.Context(), t.stream.Recv, managerProtoToTunnelMessage, &t.closed)}// IsExpectedReceiveError returns true for expected gRPC stream shutdown errors.func (t *GRPCAgentTunnelConn) IsExpectedReceiveError(err error) bool {- return isExpectedGRPCReceiveErrorInternal(err)+ return isExpectedReceiveError(err)}-// Close closes the client send stream. The half-close happens before cancel so-// the manager observes a clean io.EOF (the gRPC analog of a websocket close-// frame) instead of codes.Canceled.+// Close half-closes the send stream before cancelling, so the manager sees a+// clean io.EOF instead of codes.Canceled.func (t *GRPCAgentTunnelConn) Close() error {if t.closed.Swap(true) {return nil@@ -394,10 +332,6 @@ func (t *GRPCAgentTunnelConn) IsClosed() bool {return t.closed.Load()}-func (t *GRPCAgentTunnelConn) markClosed() {- t.closed.Store(true)-}-// Transport identifies the underlying tunnel transport.func (t *GRPCAgentTunnelConn) Transport() string {return EdgeTransportGRPC@@ -407,22 +341,29 @@ func (s *cancelableGRPCManagerStream) Send(msg *tunnelpb.ManagerMessage) error {return s.stream.Send(msg)}+// Recv returns the next message, or ctx's error once ctx is done. gRPC Recv ignores ctx,+// so a single reader goroutine waits on the stream instead.func (s *cancelableGRPCManagerStream) Recv() (*tunnelpb.AgentMessage, error) {- type recvResult struct {- msg *tunnelpb.AgentMessage- err error- }-- recvCh := make(chan recvResult, 1)- go func() {- msg, err := s.stream.Recv()- recvCh <- recvResult{msg: msg, err: err}- }()+ s.recvOnce.Do(func() {+ go func() {+ for {+ msg, err := s.stream.Recv()+ select {+ case s.recvCh <- grpcRecvResult{msg: msg, err: err}:+ case <-s.ctx.Done():+ return+ }+ if err != nil {+ return+ }+ }+ }()+ })select {case <-s.ctx.Done():return nil, s.ctx.Err()- case result := <-recvCh:+ case result := <-s.recvCh:return result.msg, result.err}}@@ -431,18 +372,35 @@ func (s *cancelableGRPCManagerStream) Context() context.Context {return s.ctx}-func isExpectedGRPCReceiveErrorInternal(err error) bool {- if err == nil {- return false+// receiveGRPC decodes the next stream message, skipping payloads from newer peers+// that this build cannot decode, as unknown websocket message types are skipped.+func receiveGRPC[P any](ctx context.Context, recv func() (P, error), decode func(P) (*TunnelMessage, error), closed *atomic.Bool) (*TunnelMessage, error) {+ for {+ protoMsg, err := recv()+ if err != nil {+ // A gRPC stream is dead after any Recv error, not just io.EOF.+ closed.Store(true)+ return nil, err+ }+ msg, err := decode(protoMsg)+ if errors.Is(err, errUnknownTunnelPayload) {+ slog.DebugContext(ctx, "Ignoring unknown edge tunnel payload", "error", err)+ continue+ }+ return msg, err}+}- if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {- return true+// isExpectedReceiveError reports normal teardown on either transport: EOF, cancellation,+// a closed connection, a clean websocket close, or a cancelled gRPC stream.+func isExpectedReceiveError(err error) bool {+ if err == nil {+ return false}- if errors.Is(err, ErrTunnelConnectionClosed) {+ if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) ||+ errors.Is(err, net.ErrClosed) || errors.Is(err, ErrTunnelConnectionClosed) || wshub.IsExpectedClose(err) {return true}-code := status.Code(err)return code == codes.Canceled || code == codes.DeadlineExceeded}diff --git a/backend/pkg/libarcane/edge/tunnel_proto_adapter.go b/backend/pkg/libarcane/edge/tunnel_proto_adapter.goindex e2ba97d1a..ea989394a 100644--- a/backend/pkg/libarcane/edge/tunnel_proto_adapter.go+++ b/backend/pkg/libarcane/edge/tunnel_proto_adapter.go@@ -1,27 +1,30 @@package edgeimport (+ "bytes""errors""fmt""maps""math"+ "slices"tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1")-// errUnknownTunnelPayload marks a proto payload this peer cannot decode, e.g.-// a oneof case added by a newer peer. Receivers skip such messages instead of-// killing the stream, mirroring how unknown JSON message types are ignored on-// the websocket transport.+// errUnknownTunnelPayload marks a proto payload this peer cannot decode, such as a oneof case+// added by a newer peer. Receivers skip it, as unknown websocket message types are skipped.var errUnknownTunnelPayload = errors.New("unknown tunnel payload type")-// tunnelMessageToManagerProto encodes a manager->agent message. parity is true-// once the agent advertised tunnelCapabilityProtoParity; without it, legacy-// agents receive stream_data re-encoded as ws_data and cannot receive stream_end.+// tunnelMessageToManagerProto encodes a manager->agent message. Without parity, legacy agents+// receive stream_data re-encoded as ws_data and cannot receive stream_end.func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManagerMessage, error) {if msg == nil {return nil, errors.New("message is nil")}+ messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")+ if err != nil {+ return nil, err+ }switch msg.Type {case MessageTypeRequest:@@ -30,7 +33,7 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManMethod: msg.Method,Path: msg.Path,Query: msg.Query,- Headers: cloneHeaderMap(msg.Headers),+ Headers: maps.Clone(msg.Headers),Body: msg.Body,}}}, nilcase MessageTypeHeartbeatAck:@@ -40,24 +43,10 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManStreamId: msg.ID,Path: msg.Path,Query: msg.Query,- Headers: cloneHeaderMap(msg.Headers),+ Headers: maps.Clone(msg.Headers),}}}, nil- case MessageTypeWebSocketData:- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")- if err != nil {- return nil, err- }- return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_WsData{WsData: &tunnelpb.WebSocketData{- StreamId: msg.ID,- Data: msg.Body,- MessageType: messageType,- }}}, nil- case MessageTypeStreamData:- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")- if err != nil {- return nil, err- }- if parity {+ case MessageTypeWebSocketData, MessageTypeStreamData:+ if msg.Type == MessageTypeStreamData && parity {return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamData{StreamData: &tunnelpb.StreamData{RequestId: msg.ID,Data: msg.Body,@@ -73,7 +62,6 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Manif parity {return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamEnd{StreamEnd: &tunnelpb.StreamEnd{RequestId: msg.ID}}}, nil}- return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)case MessageTypeWebSocketClose:return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_WsClose{WsClose: &tunnelpb.WebSocketClose{StreamId: msg.ID}}}, nilcase MessageTypeRegisterResponse:@@ -84,7 +72,7 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManError: msg.Error,SessionId: msg.SessionID,SecurityMode: msg.SecurityMode,- Capabilities: append([]string(nil), msg.Capabilities...),+ Capabilities: slices.Clone(msg.Capabilities),DrainPrevious: msg.DrainPrevious,}}}, nilcase MessageTypeCommandRequest:@@ -94,12 +82,12 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManMethod: msg.Method,Path: msg.Path,Query: msg.Query,- Headers: cloneHeaderMap(msg.Headers),+ Headers: maps.Clone(msg.Headers),Body: msg.Body,TimeoutMillis: msg.TimeoutMillis,SessionId: msg.SessionID,AgentInstanceId: msg.AgentInstance,- Metadata: cloneHeaderMap(msg.Metadata),+ Metadata: maps.Clone(msg.Metadata),}}}, nilcase MessageTypeStreamOpen:return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_StreamOpen{StreamOpen: &tunnelpb.StreamOpen{@@ -107,7 +95,7 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.ManCommandName: msg.Command,Path: msg.Path,Query: msg.Query,- Headers: cloneHeaderMap(msg.Headers),+ Headers: maps.Clone(msg.Headers),SessionId: msg.SessionID,}}}, nilcase MessageTypeStreamClose:@@ -119,25 +107,21 @@ func tunnelMessageToManagerProto(msg *TunnelMessage, parity bool) (*tunnelpb.Manreturn &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_CancelRequest{CancelRequest: &tunnelpb.CancelRequest{CommandId: msg.ID,}}}, nil+ case MessageTypeCommandCredit:+ return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_CommandCredit{CommandCredit: &tunnelpb.CommandCredit{+ CommandId: msg.ID,+ Bytes: msg.Credit,+ }}}, nilcase MessageTypeFileChunk:return &tunnelpb.ManagerMessage{Payload: &tunnelpb.ManagerMessage_FileChunk{FileChunk: &tunnelpb.FileChunk{TransferId: msg.ID,Data: msg.Body,Sequence: msg.Sequence,Eof: msg.EOF,- Metadata: cloneHeaderMap(msg.Metadata),+ Metadata: maps.Clone(msg.Metadata),}}}, nil- case MessageTypeResponse,- MessageTypeHeartbeat,- MessageTypeRegister,- MessageTypeEvent,- MessageTypeCommandAck,- MessageTypeCommandOutput,- MessageTypeCommandComplete:- return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)- default:- return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)}+ return nil, fmt.Errorf("unsupported manager message type: %s", msg.Type)}func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage, error) {@@ -153,7 +137,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,Method: payload.HttpRequest.GetMethod(),Path: payload.HttpRequest.GetPath(),Query: payload.HttpRequest.GetQuery(),- Headers: cloneHeaderMap(payload.HttpRequest.GetHeaders()),+ Headers: maps.Clone(payload.HttpRequest.GetHeaders()),Body: payload.HttpRequest.GetBody(),}, nilcase *tunnelpb.ManagerMessage_HeartbeatPong:@@ -164,15 +148,14 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,Type: MessageTypeWebSocketStart,Path: payload.WsStart.GetPath(),Query: payload.WsStart.GetQuery(),- Headers: cloneHeaderMap(payload.WsStart.GetHeaders()),+ Headers: maps.Clone(payload.WsStart.GetHeaders()),}, nilcase *tunnelpb.ManagerMessage_WsData:return &TunnelMessage{ID: payload.WsData.GetStreamId(),Type: MessageTypeWebSocketData,Body: payload.WsData.GetData(),- // WSMessageType carries RFC 6455 opcodes (text=1, binary=2) on the- // wire; a mixed-version fleet depends on these exact values.+ // RFC 6455 opcodes (text=1, binary=2) on the wire; mixed-version fleets depend on them.WSMessageType: int(payload.WsData.GetMessageType()),}, nilcase *tunnelpb.ManagerMessage_WsClose:@@ -186,7 +169,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,Error: payload.RegisterResponse.GetError(),SessionID: payload.RegisterResponse.GetSessionId(),SecurityMode: payload.RegisterResponse.GetSecurityMode(),- Capabilities: append([]string(nil), payload.RegisterResponse.GetCapabilities()...),+ Capabilities: slices.Clone(payload.RegisterResponse.GetCapabilities()),DrainPrevious: payload.RegisterResponse.GetDrainPrevious(),}, nilcase *tunnelpb.ManagerMessage_CommandRequest:@@ -197,12 +180,12 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,Method: payload.CommandRequest.GetMethod(),Path: payload.CommandRequest.GetPath(),Query: payload.CommandRequest.GetQuery(),- Headers: cloneHeaderMap(payload.CommandRequest.GetHeaders()),+ Headers: maps.Clone(payload.CommandRequest.GetHeaders()),Body: payload.CommandRequest.GetBody(),TimeoutMillis: payload.CommandRequest.GetTimeoutMillis(),SessionID: payload.CommandRequest.GetSessionId(),AgentInstance: payload.CommandRequest.GetAgentInstanceId(),- Metadata: cloneHeaderMap(payload.CommandRequest.GetMetadata()),+ Metadata: maps.Clone(payload.CommandRequest.GetMetadata()),}, nilcase *tunnelpb.ManagerMessage_StreamOpen:return &TunnelMessage{@@ -211,7 +194,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,Command: payload.StreamOpen.GetCommandName(),Path: payload.StreamOpen.GetPath(),Query: payload.StreamOpen.GetQuery(),- Headers: cloneHeaderMap(payload.StreamOpen.GetHeaders()),+ Headers: maps.Clone(payload.StreamOpen.GetHeaders()),SessionID: payload.StreamOpen.GetSessionId(),}, nilcase *tunnelpb.ManagerMessage_StreamClose:@@ -232,7 +215,7 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,Body: payload.FileChunk.GetData(),Sequence: payload.FileChunk.GetSequence(),EOF: payload.FileChunk.GetEof(),- Metadata: cloneHeaderMap(payload.FileChunk.GetMetadata()),+ Metadata: maps.Clone(payload.FileChunk.GetMetadata()),}, nilcase *tunnelpb.ManagerMessage_StreamData:return &TunnelMessage{@@ -243,6 +226,8 @@ func managerProtoToTunnelMessage(msg *tunnelpb.ManagerMessage) (*TunnelMessage,}, nilcase *tunnelpb.ManagerMessage_StreamEnd:return &TunnelMessage{ID: payload.StreamEnd.GetRequestId(), Type: MessageTypeStreamEnd}, nil+ case *tunnelpb.ManagerMessage_CommandCredit:+ return &TunnelMessage{ID: payload.CommandCredit.GetCommandId(), Type: MessageTypeCommandCredit, Credit: payload.CommandCredit.GetBytes()}, nildefault:return nil, fmt.Errorf("manager payload %T: %w", payload, errUnknownTunnelPayload)}@@ -252,26 +237,26 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erroif msg == nil {return nil, errors.New("message is nil")}+ status, err := intToInt32(msg.Status, "status")+ if err != nil {+ return nil, err+ }+ messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")+ if err != nil {+ return nil, err+ }switch msg.Type {case MessageTypeResponse:- status, err := intToInt32(msg.Status, "status")- if err != nil {- return nil, err- }return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_HttpResponse{HttpResponse: &tunnelpb.HttpResponse{RequestId: msg.ID,Status: status,- Headers: cloneHeaderMap(msg.Headers),+ Headers: maps.Clone(msg.Headers),Body: msg.Body,}}}, nilcase MessageTypeHeartbeat:return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_HeartbeatPing{HeartbeatPing: &tunnelpb.HeartbeatPing{Id: msg.ID}}}, nilcase MessageTypeWebSocketData:- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")- if err != nil {- return nil, err- }return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_WsData{WsData: &tunnelpb.WebSocketData{StreamId: msg.ID,Data: msg.Body,@@ -280,10 +265,6 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, errocase MessageTypeWebSocketClose:return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_WsClose{WsClose: &tunnelpb.WebSocketClose{StreamId: msg.ID}}}, nilcase MessageTypeStreamData:- messageType, err := intToInt32(msg.WSMessageType, "ws_message_type")- if err != nil {- return nil, err- }return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_StreamData{StreamData: &tunnelpb.StreamData{RequestId: msg.ID,Data: msg.Body,@@ -295,7 +276,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erroreturn &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_Register{Register: &tunnelpb.RegisterRequest{AgentToken: msg.AgentToken,AgentInstanceId: msg.AgentInstance,- Capabilities: append([]string(nil), msg.Capabilities...),+ Capabilities: slices.Clone(msg.Capabilities),ResumeSessionId: msg.ResumeSession,}}}, nilcase MessageTypeEvent:@@ -313,7 +294,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erroResourceName: msg.Event.ResourceName,UserId: msg.Event.UserID,Username: msg.Event.Username,- MetadataJson: append([]byte(nil), msg.Event.MetadataJSON...),+ MetadataJson: bytes.Clone(msg.Event.MetadataJSON),}}}, nilcase MessageTypeCommandAck:return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_CommandAck{CommandAck: &tunnelpb.CommandAck{@@ -324,16 +305,14 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erroCommandId: msg.ID,Data: msg.Body,Sequence: msg.Sequence,+ Status: status,+ Headers: maps.Clone(msg.Headers),}}}, nilcase MessageTypeCommandComplete:- status, err := intToInt32(msg.Status, "status")- if err != nil {- return nil, err- }return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_CommandComplete{CommandComplete: &tunnelpb.CommandComplete{CommandId: msg.ID,Status: status,- Headers: cloneHeaderMap(msg.Headers),+ Headers: maps.Clone(msg.Headers),Body: msg.Body,Error: msg.Error,Streaming: msg.Streaming,@@ -344,7 +323,7 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erroData: msg.Body,Sequence: msg.Sequence,Eof: msg.EOF,- Metadata: cloneHeaderMap(msg.Metadata),+ Metadata: maps.Clone(msg.Metadata),}}}, nilcase MessageTypeStreamClose:return &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_StreamClose{StreamClose: &tunnelpb.StreamClose{@@ -357,16 +336,8 @@ func tunnelMessageToAgentProto(msg *TunnelMessage) (*tunnelpb.AgentMessage, erroreturn &tunnelpb.AgentMessage{Payload: &tunnelpb.AgentMessage_CancelRequest{CancelRequest: &tunnelpb.CancelRequest{CommandId: msg.ID,}}}, nil- case MessageTypeRequest,- MessageTypeHeartbeatAck,- MessageTypeWebSocketStart,- MessageTypeRegisterResponse,- MessageTypeCommandRequest,- MessageTypeStreamOpen:- return nil, fmt.Errorf("unsupported agent message type: %s", msg.Type)- default:- return nil, fmt.Errorf("unsupported agent message type: %s", msg.Type)}+ return nil, fmt.Errorf("unsupported agent message type: %s", msg.Type)}func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, error) {@@ -380,7 +351,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erroID: payload.HttpResponse.GetRequestId(),Type: MessageTypeResponse,Status: int(payload.HttpResponse.GetStatus()),- Headers: cloneHeaderMap(payload.HttpResponse.GetHeaders()),+ Headers: maps.Clone(payload.HttpResponse.GetHeaders()),Body: payload.HttpResponse.GetBody(),}, nilcase *tunnelpb.AgentMessage_HeartbeatPing:@@ -408,7 +379,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erroType: MessageTypeRegister,AgentToken: payload.Register.GetAgentToken(),AgentInstance: payload.Register.GetAgentInstanceId(),- Capabilities: append([]string(nil), payload.Register.GetCapabilities()...),+ Capabilities: slices.Clone(payload.Register.GetCapabilities()),ResumeSession: payload.Register.GetResumeSessionId(),}, nilcase *tunnelpb.AgentMessage_Event:@@ -425,7 +396,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erroResourceName: payload.Event.GetResourceName(),UserID: payload.Event.GetUserId(),Username: payload.Event.GetUsername(),- MetadataJSON: append([]byte(nil), payload.Event.GetMetadataJson()...),+ MetadataJSON: bytes.Clone(payload.Event.GetMetadataJson()),},}, nilcase *tunnelpb.AgentMessage_CommandAck:@@ -436,13 +407,15 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erroType: MessageTypeCommandOutput,Body: payload.CommandOutput.GetData(),Sequence: payload.CommandOutput.GetSequence(),+ Status: int(payload.CommandOutput.GetStatus()),+ Headers: maps.Clone(payload.CommandOutput.GetHeaders()),}, nilcase *tunnelpb.AgentMessage_CommandComplete:return &TunnelMessage{ID: payload.CommandComplete.GetCommandId(),Type: MessageTypeCommandComplete,Status: int(payload.CommandComplete.GetStatus()),- Headers: cloneHeaderMap(payload.CommandComplete.GetHeaders()),+ Headers: maps.Clone(payload.CommandComplete.GetHeaders()),Body: payload.CommandComplete.GetBody(),Error: payload.CommandComplete.GetError(),Streaming: payload.CommandComplete.GetStreaming(),@@ -454,7 +427,7 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erroBody: payload.FileChunk.GetData(),Sequence: payload.FileChunk.GetSequence(),EOF: payload.FileChunk.GetEof(),- Metadata: cloneHeaderMap(payload.FileChunk.GetMetadata()),+ Metadata: maps.Clone(payload.FileChunk.GetMetadata()),}, nilcase *tunnelpb.AgentMessage_StreamClose:return &TunnelMessage{@@ -472,15 +445,6 @@ func agentProtoToTunnelMessage(msg *tunnelpb.AgentMessage) (*TunnelMessage, erro}}-func cloneHeaderMap(in map[string]string) map[string]string {- if len(in) == 0 {- return nil- }- out := make(map[string]string, len(in))- maps.Copy(out, in)- return out-}-func intToInt32(value int, field string) (int32, error) {if value < math.MinInt32 || value > math.MaxInt32 {return 0, fmt.Errorf("%s value %d is out of int32 range", field, value)diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.goindex cccbb2298..8a177c970 100644--- a/backend/pkg/libarcane/edge/types.go+++ b/backend/pkg/libarcane/edge/types.go@@ -12,6 +12,7 @@ import ("google.golang.org/grpc""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"+ tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1")// activeWSStream tracks an active WebSocket stream on the agent side.@@ -54,6 +55,7 @@ type TunnelClient struct {requestTimeout time.DurationactiveStreams sync.Map // map[string]*activeWSStreamrequestTransfers sync.Map // map[string]*commandRequestTransfer+ commandRecorders sync.Map // map[string]*commandResponseRecorder awaiting credittransportPreferenceMu sync.RWMutexpreferWebSocketUntil time.TimegrpcFailureStreak int@@ -63,6 +65,8 @@ type TunnelClient struct {type clientRegistrationInternal struct {sessionID string+ // commandCredit is set when the manager grants command output flow-control credit.+ commandCredit bool}// connBox wraps the active TunnelConnection so it can be swapped atomically on@@ -93,6 +97,13 @@ type commandResponseRecorder struct {buffer bytes.BufferstatusCode intsequence int64+ // flushErr keeps the first send failure from Flush, which cannot return it.+ flushErr error+ // With manager flow control, sends wait while inFlight would exceed window; credits signal room.+ ctx context.Context+ window int64+ inFlight atomic.Int64+ credited chan struct{}mu sync.MutexwroteHeader boolstreaming bool@@ -124,6 +135,8 @@ type CommandRequest struct {Headers map[string]stringBody []byteTimeoutMillis int64+ // Output, when set, receives the response as it streams in; CommandResult.Body is then nil.+ Output http.ResponseWriter}type CommandResult struct {@@ -191,10 +204,14 @@ type PollRuntimeRegistry struct {}type grpcResponseState struct {+ out http.ResponseWriter+ outErr error+ credit func(consumed int)status intrespHeaders map[string]stringrespBody bytes.BuffergotResponse bool+ committed bool}// AgentTunnel represents an active tunnel connection from an edge agent@@ -214,6 +231,12 @@ type AgentTunnel struct {mu sync.RWMutexdone chan struct{}closeOnce sync.Once++ // Command credits queue here for a per-tunnel sender, so readers never block on the connection.+ creditOnce sync.Once+ creditMu sync.Mutex+ credits map[string]int64+ creditSignal chan struct{}}// TunnelRegistry manages active edge agent tunnel connections@@ -226,19 +249,30 @@ type internalTunnelRequestContextKey struct{}// TunnelServer handles incoming edge agent connections on the manager side.type TunnelServer struct {- registry *TunnelRegistry- resolver EnvironmentResolver- nameResolver EnvironmentNameResolver- statusCallback StatusUpdateCallback- eventCallback EventCallback- enrollmentCallback EnrollmentCallback- cleanupDone chan struct{}- cleanupDoneOnce sync.Once- cfg *Config- statusMu sync.Mutex+ // Config holds the edge runtime settings; set it before serving.+ Config *Config+ // NameResolver, when set, names environments in generated mTLS assets.+ NameResolver EnvironmentNameResolver+ // EventCallback, when set, receives agent events.+ EventCallback EventCallback+ // EnrollmentCallback, when set, runs after a successful edge mTLS enrollment.+ EnrollmentCallback EnrollmentCallback++ registry *TunnelRegistry+ resolver EnvironmentResolver+ statusCallback StatusUpdateCallback+ cleanupDone chan struct{}+ cleanupDoneOnce sync.Once+ statusMu sync.Mutex}-type resolvedEnvironmentIDKey struct{}+type streamIdentityKey struct{}++// streamIdentity is what the gRPC auth interceptor verified for a tunnel stream.+type streamIdentity struct {+ environmentID string+ securityMode string+}type contextualServerStream struct {grpc.ServerStream@@ -267,11 +301,6 @@ type enrollMTLSResponse struct {Files []GeneratedMTLSFile `json:"files"`}-type edgeMTLSLockInfo struct {- pid int- createdAt time.Time-}-// Config contains the public edge-tunnel runtime settings needed by pkg/libarcane/edge.type Config struct {EdgeAgent bool@@ -326,6 +355,7 @@ type TunnelMessage struct {Status int `json:"status,omitempty"` // HTTP status for responsesTimeoutMillis int64 `json:"timeout_millis,omitempty"` // Command timeoutSequence int64 `json:"sequence,omitempty"` // Chunk sequence number+ Credit int64 `json:"credit,omitempty"` // Command output bytes the manager consumedAccepted bool `json:"accepted,omitempty"` // Registration acceptedDrainPrevious bool `json:"drain_previous,omitempty"` // Replace previous sessionStreaming bool `json:"streaming,omitempty"` // Response used chunked output@@ -350,6 +380,13 @@ type TunnelEvent struct {type PendingRequest struct {ResponseCh chan *TunnelMessagefailureCh chan error++ // mu guards the ordered backlog that absorbs bursts while ResponseCh is full.+ mu sync.Mutex+ backlog []*TunnelMessage+ backlogBytes int+ draining bool+ failed bool}// TunnelConn wraps a WebSocket connection with send/receive helpers.@@ -365,10 +402,9 @@ type GRPCManagerTunnelConn struct {cancel context.CancelFuncmu sync.Mutexclosed atomic.Bool- // parityEncoding is set once the agent advertises the proto-parity-v1- // capability, enabling native stream_data/stream_end encodings instead of- // the legacy ws_data re-encode.- parityEncoding atomic.Bool+ // parity is set at registration when the agent advertises proto-parity-v1,+ // enabling native stream_data/stream_end instead of the legacy ws_data re-encode.+ parity bool}// GRPCAgentTunnelConn wraps the agent-side gRPC tunnel stream.@@ -380,6 +416,13 @@ type GRPCAgentTunnelConn struct {}type cancelableGRPCManagerStream struct {- stream grpcManagerStream- ctx context.Context+ stream grpcManagerStream+ ctx context.Context+ recvCh chan grpcRecvResult+ recvOnce sync.Once+}++type grpcRecvResult struct {+ msg *tunnelpb.AgentMessage+ err error}diff --git a/backend/pkg/libarcane/startup/startup.go b/backend/pkg/libarcane/startup/startup.goindex 566485b33..4acd8634f 100644--- a/backend/pkg/libarcane/startup/startup.go+++ b/backend/pkg/libarcane/startup/startup.go@@ -3,6 +3,10 @@ package startupimport ("context""log/slog"+ "os"+ "runtime/debug"++ "go.getarcane.app/sys/cgroup""github.com/getarcaneapp/arcane/backend/v2/buildables")@@ -16,6 +20,22 @@ type RuntimeConfig struct {AdminStaticAPIKey string}+// ApplyMemoryLimit sets the Go soft memory limit to 90% of the cgroup memory+// limit so the GC tightens before the container is OOM-killed. GOMEMLIMIT wins.+func ApplyMemoryLimit(ctx context.Context) {+ if os.Getenv("GOMEMLIMIT") != "" {+ return+ }+ limits, err := cgroup.DetectLimits()+ // cgroup v1 reports "no limit" as a page-aligned value near MaxInt64.+ if err != nil || limits.MemoryLimit <= 0 || limits.MemoryLimit >= 1<<62 {+ return+ }+ limit := limits.MemoryLimit / 10 * 9+ debug.SetMemoryLimit(limit)+ slog.InfoContext(ctx, "Set Go memory limit from cgroup", "cgroupLimitBytes", limits.MemoryLimit, "goMemoryLimitBytes", limit)+}+func LoadAgentToken(ctx context.Context, cfg *RuntimeConfig, getSettingFunc func(context.Context, string, string) string) {if cfg.AgentMode && cfg.AgentToken == "" {if tok := getSettingFunc(ctx, "agentToken", ""); tok != "" {diff --git a/backend/pkg/libarcane/ws/proxy.go b/backend/pkg/libarcane/ws/proxy.goindex 04a84260a..bc14da766 100644--- a/backend/pkg/libarcane/ws/proxy.go+++ b/backend/pkg/libarcane/ws/proxy.go@@ -9,8 +9,14 @@ import ("time""github.com/coder/websocket"+ httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx")+// dialClient is shared by every upstream dial instead of building a transport per connection.+var dialClient = httpx.NewHTTPClient(httpxtypes.ClientOptions{TLSHandshakeTimeout: 10 * time.Second})+// ProxyHTTP upgrades the incoming client connection and bridges it to remoteWS.//// checkOrigin must be the same Origin validator the local WebSocket endpoints@@ -35,7 +41,7 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header hdialCtx, dialCancel := context.WithTimeout(r.Context(), 45*time.Second)remoteConn, resp, err := websocket.Dial(dialCtx, remoteWS, &websocket.DialOptions{HTTPHeader: header,- HTTPClient: &http.Client{Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}},+ HTTPClient: dialClient,})dialCancel()// Ensure the response body is drained & closed to avoid leaking resources.@@ -71,7 +77,7 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header hfor {mt, msg, readErr := clientConn.Read(pumpCtx)if readErr != nil {- relayCloseInternal(r.Context(), remoteConn, readErr)+ closeRelay(r.Context(), remoteConn, readErr)return}if writeErr := remoteConn.Write(pumpCtx, mt, msg); writeErr != nil {@@ -86,7 +92,7 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header hfor {mt, msg, readErr := remoteConn.Read(pumpCtx)if readErr != nil {- relayCloseInternal(r.Context(), clientConn, readErr)+ closeRelay(r.Context(), clientConn, readErr)return}if writeErr := clientConn.Write(pumpCtx, mt, msg); writeErr != nil {@@ -99,9 +105,9 @@ func ProxyHTTP(w http.ResponseWriter, r *http.Request, remoteWS string, header hreturn nil}-// relayCloseInternal forwards a peer's close status to the other side of the+// closeRelay forwards a peer's close status to the other side of the// bridge so the terminating reason survives the proxy hop.-func relayCloseInternal(ctx context.Context, conn *websocket.Conn, err error) {+func closeRelay(ctx context.Context, conn *websocket.Conn, err error) {var ce websocket.CloseErrorif !errors.As(err, &ce) {returndiff --git a/backend/pkg/projects/cache.go b/backend/pkg/projects/cache.goindex 3e417a201..812a7cf0f 100644--- a/backend/pkg/projects/cache.go+++ b/backend/pkg/projects/cache.go@@ -150,18 +150,35 @@ func LoadCachedComposeProject(ctx context.Context,if cache == nil {return model, nil}+ composeFiles, envFiles, ok := ComposeCacheDependencies(ctx, model, dependencies, composePath, projectsDirectory, autoInject)+ if !ok {+ // Metadata discovery failure disables caching; the executable model is valid.+ return model, nil+ }+ if setErr := cache.Set(projectID, fingerprint, projectPath, projectsDirectory, composePath, composeFiles, envFiles, model); setErr != nil {+ return nil, setErr+ }+ return model, nil+}++// ComposeCacheDependencies returns the compose and env files a model loaded+// from composePath depends on. ok is false when they cannot be discovered.+func ComposeCacheDependencies(+ ctx context.Context,+ model *composetypes.Project,+ dependencies projecttypes.ComposeDependencies,+ composePath, projectsDirectory string,+ autoInject bool,+) ([]string, []string, bool) {+ envFiles := dependencies.EnvFilesif envOpts, parseErr := ParseComposeEnvOptions(model.WorkingDir, EnvMap(model.Environment)); parseErr == nil {- dependencies.EnvFiles = append(dependencies.EnvFiles, envOpts.EnvFiles...)+ envFiles = append(envFiles, envOpts.EnvFiles...)}// Metadata discovery already follows recursive includes and their interpolation env files.meta, err := ParseArcaneComposeMetadata(ctx, composePath, projectsDirectory, autoInject)if err != nil {- return model, nil //nolint:nilerr // Metadata discovery failure disables caching; the executable model is valid.+ return nil, nil, false}composeFiles := append(append([]string{}, model.ComposeFiles...), meta.ComposeFiles...)- dependencies.EnvFiles = append(dependencies.EnvFiles, meta.EnvFiles...)- if setErr := cache.Set(projectID, fingerprint, projectPath, projectsDirectory, composePath, composeFiles, dependencies.EnvFiles, model); setErr != nil {- return nil, setErr- }- return model, nil+ return composeFiles, append(envFiles, meta.EnvFiles...), true}diff --git a/backend/pkg/utils/httpx/request.go b/backend/pkg/utils/httpx/request.goindex 480686cd2..e0bceab76 100644--- a/backend/pkg/utils/httpx/request.go+++ b/backend/pkg/utils/httpx/request.go@@ -147,6 +147,7 @@ func NewHTTPClient(options httpxtypes.ClientOptions) *http.Client {transport := &http.Transport{Proxy: http.ProxyFromEnvironment,MaxIdleConns: 100,+ MaxIdleConnsPerHost: 32,IdleConnTimeout: 90 * time.Second,TLSHandshakeTimeout: options.TLSHandshakeTimeout,ExpectContinueTimeout: 1 * time.Second,diff --git a/backend/proto/tunnel/v1/tunnel.pb.go b/backend/proto/tunnel/v1/tunnel.pb.goindex 0e2a2d958..19c1b5a7e 100644--- a/backend/proto/tunnel/v1/tunnel.pb.go+++ b/backend/proto/tunnel/v1/tunnel.pb.go@@ -1,6 +1,6 @@// Code generated by protoc-gen-go. DO NOT EDIT.// versions:-// protoc-gen-go v1.36.11+// protoc-gen-go v1.36.12// protoc (unknown)// source: tunnel/v1/tunnel.proto@@ -317,6 +317,7 @@ type ManagerMessage struct {// *ManagerMessage_FileChunk// *ManagerMessage_StreamData// *ManagerMessage_StreamEnd+ // *ManagerMessage_CommandCreditPayload isManagerMessage_Payload `protobuf_oneof:"payload"`unknownFields protoimpl.UnknownFieldssizeCache protoimpl.SizeCache@@ -476,6 +477,15 @@ func (x *ManagerMessage) GetStreamEnd() *StreamEnd {return nil}+func (x *ManagerMessage) GetCommandCredit() *CommandCredit {+ if x != nil {+ if x, ok := x.Payload.(*ManagerMessage_CommandCredit); ok {+ return x.CommandCredit+ }+ }+ return nil+}+type isManagerMessage_Payload interface {isManagerMessage_Payload()}@@ -534,6 +544,11 @@ type ManagerMessage_StreamEnd struct {StreamEnd *StreamEnd `protobuf:"bytes,13,opt,name=stream_end,json=streamEnd,proto3,oneof"`}+type ManagerMessage_CommandCredit struct {+ // Only sent to agents that advertised command-credit-v1.+ CommandCredit *CommandCredit `protobuf:"bytes,14,opt,name=command_credit,json=commandCredit,proto3,oneof"`+}+func (*ManagerMessage_HttpRequest) isManagerMessage_Payload() {}func (*ManagerMessage_HeartbeatPong) isManagerMessage_Payload() {}@@ -560,6 +575,8 @@ func (*ManagerMessage_StreamData) isManagerMessage_Payload() {}func (*ManagerMessage_StreamEnd) isManagerMessage_Payload() {}+func (*ManagerMessage_CommandCredit) isManagerMessage_Payload() {}+type RegisterRequest struct {state protoimpl.MessageState `protogen:"open.v1"`AgentToken string `protobuf:"bytes,1,opt,name=agent_token,json=agentToken,proto3" json:"agent_token,omitempty"`@@ -897,10 +914,14 @@ func (x *CommandAck) GetCommandId() string {}type CommandOutput struct {- state protoimpl.MessageState `protogen:"open.v1"`- CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`- Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`- Sequence int64 `protobuf:"varint,3,opt,name=sequence,proto3" json:"sequence,omitempty"`+ state protoimpl.MessageState `protogen:"open.v1"`+ CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`+ Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`+ Sequence int64 `protobuf:"varint,3,opt,name=sequence,proto3" json:"sequence,omitempty"`+ // Set on the first chunk so the manager can stream the response before+ // command_complete arrives. Older managers ignore these fields.+ Status int32 `protobuf:"varint,4,opt,name=status,proto3" json:"status,omitempty"`+ Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`unknownFields protoimpl.UnknownFieldssizeCache protoimpl.SizeCache}@@ -956,6 +977,20 @@ func (x *CommandOutput) GetSequence() int64 {return 0}+func (x *CommandOutput) GetStatus() int32 {+ if x != nil {+ return x.Status+ }+ return 0+}++func (x *CommandOutput) GetHeaders() map[string]string {+ if x != nil {+ return x.Headers+ }+ return nil+}+type CommandComplete struct {state protoimpl.MessageState `protogen:"open.v1"`CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`@@ -1296,6 +1331,59 @@ func (x *CancelRequest) GetCommandId() string {return ""}+// CommandCredit returns flow-control credit for command output the manager has consumed.+type CommandCredit struct {+ state protoimpl.MessageState `protogen:"open.v1"`+ CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"`+ Bytes int64 `protobuf:"varint,2,opt,name=bytes,proto3" json:"bytes,omitempty"`+ unknownFields protoimpl.UnknownFields+ sizeCache protoimpl.SizeCache+}++func (x *CommandCredit) Reset() {+ *x = CommandCredit{}+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ ms.StoreMessageInfo(mi)+}++func (x *CommandCredit) String() string {+ return protoimpl.X.MessageStringOf(x)+}++func (*CommandCredit) ProtoMessage() {}++func (x *CommandCredit) ProtoReflect() protoreflect.Message {+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]+ if x != nil {+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))+ if ms.LoadMessageInfo() == nil {+ ms.StoreMessageInfo(mi)+ }+ return ms+ }+ return mi.MessageOf(x)+}++// Deprecated: Use CommandCredit.ProtoReflect.Descriptor instead.+func (*CommandCredit) Descriptor() ([]byte, []int) {+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{12}+}++func (x *CommandCredit) GetCommandId() string {+ if x != nil {+ return x.CommandId+ }+ return ""+}++func (x *CommandCredit) GetBytes() int64 {+ if x != nil {+ return x.Bytes+ }+ return 0+}+type HttpRequest struct {state protoimpl.MessageState `protogen:"open.v1"`RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`@@ -1310,7 +1398,7 @@ type HttpRequest struct {func (x *HttpRequest) Reset() {*x = HttpRequest{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1322,7 +1410,7 @@ func (x *HttpRequest) String() string {func (*HttpRequest) ProtoMessage() {}func (x *HttpRequest) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[12]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1335,7 +1423,7 @@ func (x *HttpRequest) ProtoReflect() protoreflect.Message {// Deprecated: Use HttpRequest.ProtoReflect.Descriptor instead.func (*HttpRequest) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{12}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{13}}func (x *HttpRequest) GetRequestId() string {@@ -1392,7 +1480,7 @@ type HttpResponse struct {func (x *HttpResponse) Reset() {*x = HttpResponse{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1404,7 +1492,7 @@ func (x *HttpResponse) String() string {func (*HttpResponse) ProtoMessage() {}func (x *HttpResponse) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[13]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1417,7 +1505,7 @@ func (x *HttpResponse) ProtoReflect() protoreflect.Message {// Deprecated: Use HttpResponse.ProtoReflect.Descriptor instead.func (*HttpResponse) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{13}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{14}}func (x *HttpResponse) GetRequestId() string {@@ -1460,7 +1548,7 @@ type StreamData struct {func (x *StreamData) Reset() {*x = StreamData{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1472,7 +1560,7 @@ func (x *StreamData) String() string {func (*StreamData) ProtoMessage() {}func (x *StreamData) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[14]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1485,7 +1573,7 @@ func (x *StreamData) ProtoReflect() protoreflect.Message {// Deprecated: Use StreamData.ProtoReflect.Descriptor instead.func (*StreamData) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{14}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{15}}func (x *StreamData) GetRequestId() string {@@ -1518,7 +1606,7 @@ type StreamEnd struct {func (x *StreamEnd) Reset() {*x = StreamEnd{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1530,7 +1618,7 @@ func (x *StreamEnd) String() string {func (*StreamEnd) ProtoMessage() {}func (x *StreamEnd) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[15]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1543,7 +1631,7 @@ func (x *StreamEnd) ProtoReflect() protoreflect.Message {// Deprecated: Use StreamEnd.ProtoReflect.Descriptor instead.func (*StreamEnd) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{15}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{16}}func (x *StreamEnd) GetRequestId() string {@@ -1562,7 +1650,7 @@ type HeartbeatPing struct {func (x *HeartbeatPing) Reset() {*x = HeartbeatPing{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1574,7 +1662,7 @@ func (x *HeartbeatPing) String() string {func (*HeartbeatPing) ProtoMessage() {}func (x *HeartbeatPing) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[16]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1587,7 +1675,7 @@ func (x *HeartbeatPing) ProtoReflect() protoreflect.Message {// Deprecated: Use HeartbeatPing.ProtoReflect.Descriptor instead.func (*HeartbeatPing) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{16}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{17}}func (x *HeartbeatPing) GetId() string {@@ -1606,7 +1694,7 @@ type HeartbeatPong struct {func (x *HeartbeatPong) Reset() {*x = HeartbeatPong{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1618,7 +1706,7 @@ func (x *HeartbeatPong) String() string {func (*HeartbeatPong) ProtoMessage() {}func (x *HeartbeatPong) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[17]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1631,7 +1719,7 @@ func (x *HeartbeatPong) ProtoReflect() protoreflect.Message {// Deprecated: Use HeartbeatPong.ProtoReflect.Descriptor instead.func (*HeartbeatPong) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{17}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{18}}func (x *HeartbeatPong) GetId() string {@@ -1653,7 +1741,7 @@ type WebSocketStart struct {func (x *WebSocketStart) Reset() {*x = WebSocketStart{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1665,7 +1753,7 @@ func (x *WebSocketStart) String() string {func (*WebSocketStart) ProtoMessage() {}func (x *WebSocketStart) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[18]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1678,7 +1766,7 @@ func (x *WebSocketStart) ProtoReflect() protoreflect.Message {// Deprecated: Use WebSocketStart.ProtoReflect.Descriptor instead.func (*WebSocketStart) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{18}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{19}}func (x *WebSocketStart) GetStreamId() string {@@ -1720,7 +1808,7 @@ type WebSocketData struct {func (x *WebSocketData) Reset() {*x = WebSocketData{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1732,7 +1820,7 @@ func (x *WebSocketData) String() string {func (*WebSocketData) ProtoMessage() {}func (x *WebSocketData) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[19]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1745,7 +1833,7 @@ func (x *WebSocketData) ProtoReflect() protoreflect.Message {// Deprecated: Use WebSocketData.ProtoReflect.Descriptor instead.func (*WebSocketData) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{19}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{20}}func (x *WebSocketData) GetStreamId() string {@@ -1778,7 +1866,7 @@ type WebSocketClose struct {func (x *WebSocketClose) Reset() {*x = WebSocketClose{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1790,7 +1878,7 @@ func (x *WebSocketClose) String() string {func (*WebSocketClose) ProtoMessage() {}func (x *WebSocketClose) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[20]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1803,7 +1891,7 @@ func (x *WebSocketClose) ProtoReflect() protoreflect.Message {// Deprecated: Use WebSocketClose.ProtoReflect.Descriptor instead.func (*WebSocketClose) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{20}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{21}}func (x *WebSocketClose) GetStreamId() string {@@ -1833,7 +1921,7 @@ type EventLog struct {func (x *EventLog) Reset() {*x = EventLog{}- mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[22]ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))ms.StoreMessageInfo(mi)}@@ -1845,7 +1933,7 @@ func (x *EventLog) String() string {func (*EventLog) ProtoMessage() {}func (x *EventLog) ProtoReflect() protoreflect.Message {- mi := &file_tunnel_v1_tunnel_proto_msgTypes[21]+ mi := &file_tunnel_v1_tunnel_proto_msgTypes[22]if x != nil {ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))if ms.LoadMessageInfo() == nil {@@ -1858,7 +1946,7 @@ func (x *EventLog) ProtoReflect() protoreflect.Message {// Deprecated: Use EventLog.ProtoReflect.Descriptor instead.func (*EventLog) Descriptor() ([]byte, []int) {- return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{21}+ return file_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{22}}func (x *EventLog) GetType() string {@@ -1963,7 +2051,7 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +"file_chunk\x18\f \x01(\v2\x14.tunnel.v1.FileChunkH\x00R\tfileChunk\x12;\n" +"\fstream_close\x18\r \x01(\v2\x16.tunnel.v1.StreamCloseH\x00R\vstreamClose\x12A\n" +"\x0ecancel_request\x18\x0e \x01(\v2\x18.tunnel.v1.CancelRequestH\x00R\rcancelRequestB\t\n" +- "\apayload\"\xb4\x06\n" ++ "\apayload\"\xf7\x06\n" +"\x0eManagerMessage\x12;\n" +"\fhttp_request\x18\x01 \x01(\v2\x16.tunnel.v1.HttpRequestH\x00R\vhttpRequest\x12A\n" +"\x0eheartbeat_pong\x18\x02 \x01(\v2\x18.tunnel.v1.HeartbeatPongH\x00R\rheartbeatPong\x126\n" +@@ -1982,7 +2070,8 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +"\vstream_data\x18\f \x01(\v2\x15.tunnel.v1.StreamDataH\x00R\n" +"streamData\x125\n" +"\n" +- "stream_end\x18\r \x01(\v2\x14.tunnel.v1.StreamEndH\x00R\tstreamEndB\t\n" ++ "stream_end\x18\r \x01(\v2\x14.tunnel.v1.StreamEndH\x00R\tstreamEnd\x12A\n" ++ "\x0ecommand_credit\x18\x0e \x01(\v2\x18.tunnel.v1.CommandCreditH\x00R\rcommandCreditB\t\n" +"\apayload\"\xae\x01\n" +"\x0fRegisterRequest\x12\x1f\n" +"\vagent_token\x18\x01 \x01(\tR\n" +@@ -2024,12 +2113,17 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +"\n" +"CommandAck\x12\x1d\n" +"\n" +- "command_id\x18\x01 \x01(\tR\tcommandId\"^\n" ++ "command_id\x18\x01 \x01(\tR\tcommandId\"\xf3\x01\n" +"\rCommandOutput\x12\x1d\n" +"\n" +"command_id\x18\x01 \x01(\tR\tcommandId\x12\x12\n" +"\x04data\x18\x02 \x01(\fR\x04data\x12\x1a\n" +- "\bsequence\x18\x03 \x01(\x03R\bsequence\"\x8f\x02\n" ++ "\bsequence\x18\x03 \x01(\x03R\bsequence\x12\x16\n" ++ "\x06status\x18\x04 \x01(\x05R\x06status\x12?\n" ++ "\aheaders\x18\x05 \x03(\v2%.tunnel.v1.CommandOutput.HeadersEntryR\aheaders\x1a:\n" ++ "\fHeadersEntry\x12\x10\n" ++ "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" ++ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8f\x02\n" +"\x0fCommandComplete\x12\x1d\n" +"\n" +"command_id\x18\x01 \x01(\tR\tcommandId\x12\x16\n" +@@ -2068,7 +2162,11 @@ const file_tunnel_v1_tunnel_proto_rawDesc = "" +"\x05error\x18\x02 \x01(\tR\x05error\".\n" +"\rCancelRequest\x12\x1d\n" +"\n" +- "command_id\x18\x01 \x01(\tR\tcommandId\"\xfd\x01\n" ++ "command_id\x18\x01 \x01(\tR\tcommandId\"D\n" ++ "\rCommandCredit\x12\x1d\n" ++ "\n" ++ "command_id\x18\x01 \x01(\tR\tcommandId\x12\x14\n" ++ "\x05bytes\x18\x02 \x01(\x03R\x05bytes\"\xfd\x01\n" +"\vHttpRequest\x12\x1d\n" +"\n" +"request_id\x18\x01 \x01(\tR\trequestId\x12\x16\n" +@@ -2145,7 +2243,7 @@ func file_tunnel_v1_tunnel_proto_rawDescGZIP() []byte {return file_tunnel_v1_tunnel_proto_rawDescData}-var file_tunnel_v1_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 30)+var file_tunnel_v1_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 32)var file_tunnel_v1_tunnel_proto_goTypes = []any{(*AgentMessage)(nil), // 0: tunnel.v1.AgentMessage(*ManagerMessage)(nil), // 1: tunnel.v1.ManagerMessage@@ -2159,68 +2257,72 @@ var file_tunnel_v1_tunnel_proto_goTypes = []any{(*StreamOpen)(nil), // 9: tunnel.v1.StreamOpen(*StreamClose)(nil), // 10: tunnel.v1.StreamClose(*CancelRequest)(nil), // 11: tunnel.v1.CancelRequest- (*HttpRequest)(nil), // 12: tunnel.v1.HttpRequest- (*HttpResponse)(nil), // 13: tunnel.v1.HttpResponse- (*StreamData)(nil), // 14: tunnel.v1.StreamData- (*StreamEnd)(nil), // 15: tunnel.v1.StreamEnd- (*HeartbeatPing)(nil), // 16: tunnel.v1.HeartbeatPing- (*HeartbeatPong)(nil), // 17: tunnel.v1.HeartbeatPong- (*WebSocketStart)(nil), // 18: tunnel.v1.WebSocketStart- (*WebSocketData)(nil), // 19: tunnel.v1.WebSocketData- (*WebSocketClose)(nil), // 20: tunnel.v1.WebSocketClose- (*EventLog)(nil), // 21: tunnel.v1.EventLog- nil, // 22: tunnel.v1.CommandRequest.HeadersEntry- nil, // 23: tunnel.v1.CommandRequest.MetadataEntry- nil, // 24: tunnel.v1.CommandComplete.HeadersEntry- nil, // 25: tunnel.v1.FileChunk.MetadataEntry- nil, // 26: tunnel.v1.StreamOpen.HeadersEntry- nil, // 27: tunnel.v1.HttpRequest.HeadersEntry- nil, // 28: tunnel.v1.HttpResponse.HeadersEntry- nil, // 29: tunnel.v1.WebSocketStart.HeadersEntry+ (*CommandCredit)(nil), // 12: tunnel.v1.CommandCredit+ (*HttpRequest)(nil), // 13: tunnel.v1.HttpRequest+ (*HttpResponse)(nil), // 14: tunnel.v1.HttpResponse+ (*StreamData)(nil), // 15: tunnel.v1.StreamData+ (*StreamEnd)(nil), // 16: tunnel.v1.StreamEnd+ (*HeartbeatPing)(nil), // 17: tunnel.v1.HeartbeatPing+ (*HeartbeatPong)(nil), // 18: tunnel.v1.HeartbeatPong+ (*WebSocketStart)(nil), // 19: tunnel.v1.WebSocketStart+ (*WebSocketData)(nil), // 20: tunnel.v1.WebSocketData+ (*WebSocketClose)(nil), // 21: tunnel.v1.WebSocketClose+ (*EventLog)(nil), // 22: tunnel.v1.EventLog+ nil, // 23: tunnel.v1.CommandRequest.HeadersEntry+ nil, // 24: tunnel.v1.CommandRequest.MetadataEntry+ nil, // 25: tunnel.v1.CommandOutput.HeadersEntry+ nil, // 26: tunnel.v1.CommandComplete.HeadersEntry+ nil, // 27: tunnel.v1.FileChunk.MetadataEntry+ nil, // 28: tunnel.v1.StreamOpen.HeadersEntry+ nil, // 29: tunnel.v1.HttpRequest.HeadersEntry+ nil, // 30: tunnel.v1.HttpResponse.HeadersEntry+ nil, // 31: tunnel.v1.WebSocketStart.HeadersEntry}var file_tunnel_v1_tunnel_proto_depIdxs = []int32{- 13, // 0: tunnel.v1.AgentMessage.http_response:type_name -> tunnel.v1.HttpResponse- 16, // 1: tunnel.v1.AgentMessage.heartbeat_ping:type_name -> tunnel.v1.HeartbeatPing- 19, // 2: tunnel.v1.AgentMessage.ws_data:type_name -> tunnel.v1.WebSocketData- 20, // 3: tunnel.v1.AgentMessage.ws_close:type_name -> tunnel.v1.WebSocketClose- 14, // 4: tunnel.v1.AgentMessage.stream_data:type_name -> tunnel.v1.StreamData- 15, // 5: tunnel.v1.AgentMessage.stream_end:type_name -> tunnel.v1.StreamEnd+ 14, // 0: tunnel.v1.AgentMessage.http_response:type_name -> tunnel.v1.HttpResponse+ 17, // 1: tunnel.v1.AgentMessage.heartbeat_ping:type_name -> tunnel.v1.HeartbeatPing+ 20, // 2: tunnel.v1.AgentMessage.ws_data:type_name -> tunnel.v1.WebSocketData+ 21, // 3: tunnel.v1.AgentMessage.ws_close:type_name -> tunnel.v1.WebSocketClose+ 15, // 4: tunnel.v1.AgentMessage.stream_data:type_name -> tunnel.v1.StreamData+ 16, // 5: tunnel.v1.AgentMessage.stream_end:type_name -> tunnel.v1.StreamEnd2, // 6: tunnel.v1.AgentMessage.register:type_name -> tunnel.v1.RegisterRequest- 21, // 7: tunnel.v1.AgentMessage.event:type_name -> tunnel.v1.EventLog+ 22, // 7: tunnel.v1.AgentMessage.event:type_name -> tunnel.v1.EventLog5, // 8: tunnel.v1.AgentMessage.command_ack:type_name -> tunnel.v1.CommandAck6, // 9: tunnel.v1.AgentMessage.command_output:type_name -> tunnel.v1.CommandOutput7, // 10: tunnel.v1.AgentMessage.command_complete:type_name -> tunnel.v1.CommandComplete8, // 11: tunnel.v1.AgentMessage.file_chunk:type_name -> tunnel.v1.FileChunk10, // 12: tunnel.v1.AgentMessage.stream_close:type_name -> tunnel.v1.StreamClose11, // 13: tunnel.v1.AgentMessage.cancel_request:type_name -> tunnel.v1.CancelRequest- 12, // 14: tunnel.v1.ManagerMessage.http_request:type_name -> tunnel.v1.HttpRequest- 17, // 15: tunnel.v1.ManagerMessage.heartbeat_pong:type_name -> tunnel.v1.HeartbeatPong- 18, // 16: tunnel.v1.ManagerMessage.ws_start:type_name -> tunnel.v1.WebSocketStart- 19, // 17: tunnel.v1.ManagerMessage.ws_data:type_name -> tunnel.v1.WebSocketData- 20, // 18: tunnel.v1.ManagerMessage.ws_close:type_name -> tunnel.v1.WebSocketClose+ 13, // 14: tunnel.v1.ManagerMessage.http_request:type_name -> tunnel.v1.HttpRequest+ 18, // 15: tunnel.v1.ManagerMessage.heartbeat_pong:type_name -> tunnel.v1.HeartbeatPong+ 19, // 16: tunnel.v1.ManagerMessage.ws_start:type_name -> tunnel.v1.WebSocketStart+ 20, // 17: tunnel.v1.ManagerMessage.ws_data:type_name -> tunnel.v1.WebSocketData+ 21, // 18: tunnel.v1.ManagerMessage.ws_close:type_name -> tunnel.v1.WebSocketClose3, // 19: tunnel.v1.ManagerMessage.register_response:type_name -> tunnel.v1.RegisterResponse4, // 20: tunnel.v1.ManagerMessage.command_request:type_name -> tunnel.v1.CommandRequest9, // 21: tunnel.v1.ManagerMessage.stream_open:type_name -> tunnel.v1.StreamOpen10, // 22: tunnel.v1.ManagerMessage.stream_close:type_name -> tunnel.v1.StreamClose11, // 23: tunnel.v1.ManagerMessage.cancel_request:type_name -> tunnel.v1.CancelRequest8, // 24: tunnel.v1.ManagerMessage.file_chunk:type_name -> tunnel.v1.FileChunk- 14, // 25: tunnel.v1.ManagerMessage.stream_data:type_name -> tunnel.v1.StreamData- 15, // 26: tunnel.v1.ManagerMessage.stream_end:type_name -> tunnel.v1.StreamEnd- 22, // 27: tunnel.v1.CommandRequest.headers:type_name -> tunnel.v1.CommandRequest.HeadersEntry- 23, // 28: tunnel.v1.CommandRequest.metadata:type_name -> tunnel.v1.CommandRequest.MetadataEntry- 24, // 29: tunnel.v1.CommandComplete.headers:type_name -> tunnel.v1.CommandComplete.HeadersEntry- 25, // 30: tunnel.v1.FileChunk.metadata:type_name -> tunnel.v1.FileChunk.MetadataEntry- 26, // 31: tunnel.v1.StreamOpen.headers:type_name -> tunnel.v1.StreamOpen.HeadersEntry- 27, // 32: tunnel.v1.HttpRequest.headers:type_name -> tunnel.v1.HttpRequest.HeadersEntry- 28, // 33: tunnel.v1.HttpResponse.headers:type_name -> tunnel.v1.HttpResponse.HeadersEntry- 29, // 34: tunnel.v1.WebSocketStart.headers:type_name -> tunnel.v1.WebSocketStart.HeadersEntry- 0, // 35: tunnel.v1.TunnelService.Connect:input_type -> tunnel.v1.AgentMessage- 1, // 36: tunnel.v1.TunnelService.Connect:output_type -> tunnel.v1.ManagerMessage- 36, // [36:37] is the sub-list for method output_type- 35, // [35:36] is the sub-list for method input_type- 35, // [35:35] is the sub-list for extension type_name- 35, // [35:35] is the sub-list for extension extendee- 0, // [0:35] is the sub-list for field type_name+ 15, // 25: tunnel.v1.ManagerMessage.stream_data:type_name -> tunnel.v1.StreamData+ 16, // 26: tunnel.v1.ManagerMessage.stream_end:type_name -> tunnel.v1.StreamEnd+ 12, // 27: tunnel.v1.ManagerMessage.command_credit:type_name -> tunnel.v1.CommandCredit+ 23, // 28: tunnel.v1.CommandRequest.headers:type_name -> tunnel.v1.CommandRequest.HeadersEntry+ 24, // 29: tunnel.v1.CommandRequest.metadata:type_name -> tunnel.v1.CommandRequest.MetadataEntry+ 25, // 30: tunnel.v1.CommandOutput.headers:type_name -> tunnel.v1.CommandOutput.HeadersEntry+ 26, // 31: tunnel.v1.CommandComplete.headers:type_name -> tunnel.v1.CommandComplete.HeadersEntry+ 27, // 32: tunnel.v1.FileChunk.metadata:type_name -> tunnel.v1.FileChunk.MetadataEntry+ 28, // 33: tunnel.v1.StreamOpen.headers:type_name -> tunnel.v1.StreamOpen.HeadersEntry+ 29, // 34: tunnel.v1.HttpRequest.headers:type_name -> tunnel.v1.HttpRequest.HeadersEntry+ 30, // 35: tunnel.v1.HttpResponse.headers:type_name -> tunnel.v1.HttpResponse.HeadersEntry+ 31, // 36: tunnel.v1.WebSocketStart.headers:type_name -> tunnel.v1.WebSocketStart.HeadersEntry+ 0, // 37: tunnel.v1.TunnelService.Connect:input_type -> tunnel.v1.AgentMessage+ 1, // 38: tunnel.v1.TunnelService.Connect:output_type -> tunnel.v1.ManagerMessage+ 38, // [38:39] is the sub-list for method output_type+ 37, // [37:38] is the sub-list for method input_type+ 37, // [37:37] is the sub-list for extension type_name+ 37, // [37:37] is the sub-list for extension extendee+ 0, // [0:37] is the sub-list for field type_name}func init() { file_tunnel_v1_tunnel_proto_init() }@@ -2258,6 +2360,7 @@ func file_tunnel_v1_tunnel_proto_init() {(*ManagerMessage_FileChunk)(nil),(*ManagerMessage_StreamData)(nil),(*ManagerMessage_StreamEnd)(nil),+ (*ManagerMessage_CommandCredit)(nil),}type x struct{}out := protoimpl.TypeBuilder{@@ -2265,7 +2368,7 @@ func file_tunnel_v1_tunnel_proto_init() {GoPackagePath: reflect.TypeOf(x{}).PkgPath(),RawDescriptor: unsafe.Slice(unsafe.StringData(file_tunnel_v1_tunnel_proto_rawDesc), len(file_tunnel_v1_tunnel_proto_rawDesc)),NumEnums: 0,- NumMessages: 30,+ NumMessages: 32,NumExtensions: 0,NumServices: 1,},diff --git a/backend/proto/tunnel/v1/tunnel.proto b/backend/proto/tunnel/v1/tunnel.protoindex 49e028020..5477d6514 100644--- a/backend/proto/tunnel/v1/tunnel.proto+++ b/backend/proto/tunnel/v1/tunnel.proto@@ -50,6 +50,8 @@ message ManagerMessage {// older agents decode an unknown oneof case as a nil payload.StreamData stream_data = 12;StreamEnd stream_end = 13;+ // Only sent to agents that advertised command-credit-v1.+ CommandCredit command_credit = 14;}}@@ -93,6 +95,10 @@ message CommandOutput {string command_id = 1;bytes data = 2;int64 sequence = 3;+ // Set on the first chunk so the manager can stream the response before+ // command_complete arrives. Older managers ignore these fields.+ int32 status = 4;+ map<string, string> headers = 5;}message CommandComplete {@@ -130,6 +136,12 @@ message CancelRequest {string command_id = 1;}+// CommandCredit returns flow-control credit for command output the manager has consumed.+message CommandCredit {+ string command_id = 1;+ int64 bytes = 2;+}+message HttpRequest {string request_id = 1;string method = 2;