Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: harden edge environment access and health checks (#4489)
Kyle Mendelledb390d61c7d
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex 2a9b8b33a..d555b336e 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -23,6 +23,7 @@ import ("go.uber.org/fx/fxevent""github.com/getarcaneapp/arcane/backend/v2/internal/apikey"+ "github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -206,6 +207,7 @@ type initializeStartupStateParams struct {Docker *docker.DockerClientServiceSwarm *swarm.SwarmServiceRole *role.RoleService+ Auth *auth.AuthServiceUser *user.UserServiceApiKey *apikey.ApiKeyServiceVulnerability *vulnerability.VulnerabilityService@@ -286,6 +288,10 @@ func initializeStartupState(p initializeStartupStateParams) {// Global variables are manager-owned: import a legacy local .env.global once, then sync everywhere.p.Environment.SetVariableSyncer(p.Variable)+ p.Environment.SetUserInvalidator(func(userID string) {+ p.Role.InvalidateUser(userID)+ p.Auth.InvalidateUserTokenCache(userID)+ })if err := p.Variable.ImportLegacyLocalEnvFile(appCtx); err != nil {slog.WarnContext(appCtx, "Failed to import legacy global variables", "error", err)}@@ -429,7 +435,7 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config) error}defer func() { _ = resp.Body.Close() }()- body, _ := io.ReadAll(resp.Body)+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))switch {case resp.StatusCode == http.StatusOK:@@ -443,5 +449,5 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config) errorslog.DebugContext(ctx, "Pairing skipped - API key not recognized (agent may already be paired)", "managerUrl", cfg.ManagerApiUrl)return nil}- return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))+ return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))}diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.goindex c4efd0122..08333ad55 100644--- a/backend/internal/bootstrap/edge_bootstrap.go+++ b/backend/internal/bootstrap/edge_bootstrap.go@@ -111,7 +111,7 @@ func registerEdgeTunnelRoutes(return stopCleanup(stopCtx)},})- apiGroup.POST("/tunnel/poll", server.HandlePoll)+ apiGroup.POST("/tunnel/poll", server.HandlePoll, middleware.PerIPRateLimit(6000, 1200), middleware.PerAgentTokenRateLimit(60, 30))// Rate-limit agent mTLS enrollment per-IP. Enrollment is authenticated// only by the agent token, so we cap bursts to mitigate brute-force or// token-abuse attempts without impacting normal agent lifecycles.diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex ae5719f5f..fc01a26d2 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -331,6 +331,7 @@ func registerJobs(params registerJobsParams) error {Scheduler: params.Scheduler,Settings: params.Settings,Environment: params.Environment,+ JobSchedule: params.JobSchedule,AutoUpdate: params.AutoUpdate,ImageUpdateWatcher: params.ImageUpdateWatcher,FilesystemWatcher: params.FilesystemWatcher,@@ -410,6 +411,7 @@ type settingsSubscriptionsParams struct {Scheduler settingsEffectsSchedulerSettings settingsChangeSubscriberEnvironment timeoutSettingsEnvironment+ JobSchedule *job.JobServiceAutoUpdate *flow.JobImageUpdateWatcher *scheduler.ImageUpdateWatcher@@ -466,6 +468,16 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {rescheduleOn([]string{"autoUpdate", "autoUpdateInterval"}, params.AutoUpdate)+ subscribe([]string{"dockerClientRefreshInterval", "environmentHealthInterval"}, func(updates []libarcane.SettingUpdate) {+ changedKeys := make([]string, 0, len(updates))+ for _, update := range updates {+ changedKeys = append(changedKeys, update.Key)+ }+ if err := params.JobSchedule.RescheduleJobsForSettingKeys(params.LifecycleCtx, changedKeys); err != nil {+ slog.WarnContext(params.LifecycleCtx, "Failed to reschedule jobs after settings change", "keys", changedKeys, "error", err)+ }+ })+subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.goindex 5f70ab074..765075a3d 100644--- a/backend/internal/bootstrap/router_bootstrap.go+++ b/backend/internal/bootstrap/router_bootstrap.go@@ -26,6 +26,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/federated""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/telemetry"@@ -239,11 +240,17 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {))// Keyed per token for stored tokens, per source IP for everything else.apiGroup.Use(middleware.PerTokenRateLimitForPaths(- []string{"/api/webhooks/trigger/:token"}, 60, 10, deps.Webhook.Service().IsKnownToken,+ []string{"/api/webhooks/trigger/:token"}, 60, 10,+ func(c *echo.Context) string { return c.Param("token") },+ deps.Webhook.Service().IsKnownToken,))// Agent event ingestion sits outside the auth middleware, so it needs its own ceiling; agents batch events.- apiGroup.Use(middleware.PerIPRateLimitForPaths(- []string{"/api/events"}, 60, 30,+ apiGroup.Use(middleware.PerTokenRateLimitForPaths(+ []string{"/api/events"}, 600, 2*event.DaemonEventBatchSize,+ func(c *echo.Context) string {+ return strings.TrimSpace(c.Request().Header.Get(middleware.HeaderAgentToken))+ },+ deps.Environment.Service().IsKnownAccessToken,))handlerAppCtx := handlerutil.NewActivityAppContext(ctx)diff --git a/backend/internal/dashboard/handler.go b/backend/internal/dashboard/handler.goindex cd7efa142..d8422e4b2 100644--- a/backend/internal/dashboard/handler.go+++ b/backend/internal/dashboard/handler.go@@ -392,5 +392,8 @@ func classifyDashboardStreamErrorInternal(err error) (string, string) {if transportErr, ok := errors.AsType[*remenv.TransportError](err); ok {return transportErr.Error(), dashboard.StreamErrorCodeUnreachable}+ if statusErr, ok := errors.AsType[*remenv.StatusError](err); ok && (statusErr.StatusCode == http.StatusUnauthorized || statusErr.StatusCode == http.StatusForbidden) {+ return "Agent rejected the environment access token — regenerate the API key and update the agent", dashboard.StreamErrorCodeAuthFailed+ }return err.Error(), ""}diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex eea7f2759..7ef945d24 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -86,7 +86,11 @@ var ServiceOptions = fx.Options(appimages.NewApplicationImagesService,session.NewSessionService,passkey.NewPasskeyService,- environment.NewEnvironmentService,+ fx.Annotate(environment.NewEnvironmentService,+ fx.OnStart(func(ctx context.Context, service *environment.EnvironmentService) error {+ return service.LoadAccessTokens(ctx)+ }),+ ),apns.NewApnsService,notification.NewNotificationService,provideVulnerabilityService,@@ -204,10 +208,14 @@ var ServiceOptions = fx.Options(return settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)},func(service *s3domain.S3DestinationService, localEnvironment *environment.EnvironmentService) *s3domain.Module {- return s3domain.New(service, localEnvironment.SyncS3DestinationsToRemoteEnvironments)+ return s3domain.New(service, func(ctx context.Context) {+ localEnvironment.QueueRemoteSync(ctx, "S3 destinations", localEnvironment.SyncS3DestinationsToRemoteEnvironments)+ })},func(service *registry.ContainerRegistryService, localEnvironment *environment.EnvironmentService) *registry.Module {- return registry.New(service, localEnvironment.SyncRegistriesToRemoteEnvironments)+ return registry.New(service, func(ctx context.Context) {+ localEnvironment.QueueRemoteSync(ctx, "registries", localEnvironment.SyncRegistriesToRemoteEnvironments)+ })},func(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {return auth.New(service, userService, settingsService, passkeyService.BeginMFAAuthentication)diff --git a/backend/internal/environment/children/health/service.go b/backend/internal/environment/children/health/service.goindex 931c108e2..88c7a7a20 100644--- a/backend/internal/environment/children/health/service.go+++ b/backend/internal/environment/children/health/service.go@@ -29,7 +29,7 @@ func New(httpClient *http.Client, dockerService *docker.DockerClientService) *Se// Probe reports "online", "offline" or "error" for one environment. A custom// API URL always probes directly over HTTP.-func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customURL bool) (string, error) {+func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customURL bool, authenticated func(context.Context) (int, error)) (string, error) {if id == "0" && !customURL {reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)defer cancel()@@ -50,18 +50,22 @@ func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customURreturn "offline", errors.New("edge agent is not connected")}}-+ }+ if !customURL {reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second)defer cancel()- statusCode, _, err := edge.DoRequest(reqCtx, id, http.MethodGet, "/api/health", nil)- if err != nil {- return "offline", fmt.Errorf("health check via tunnel failed: %w", err)- }- if statusCode == http.StatusOK {+ statusCode, err := authenticated(reqCtx)+ switch {+ case err != nil:+ return "offline", fmt.Errorf("health check failed: %w", err)+ case statusCode == http.StatusOK:return "online", nil+ case statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden:+ return "error", fmt.Errorf("environment rejected the access token: status code %d", statusCode)+ default:+ return "error", fmt.Errorf("unexpected status code: %d", statusCode)}- return "error", fmt.Errorf("unexpected status code: %d", statusCode)}baseURL, err := httpx.NormalizeBaseURL(apiURL)diff --git a/backend/internal/environment/children/proxy/service.go b/backend/internal/environment/children/proxy/service.goindex 90a1e114b..20123c0f1 100644--- a/backend/internal/environment/children/proxy/service.go+++ b/backend/internal/environment/children/proxy/service.go@@ -38,6 +38,7 @@ type Service struct {client *remenv.ClientsettingsService *settings.SettingsServicesyncGate *utils.SyncGate+ syncLocks utils.KeyedMutex}// Target is a validated remote environment endpoint; build it with NewTarget.@@ -308,6 +309,12 @@ func (s *Service) fanOutSyncToEnvironment[Model, Item, Request any](toSyncItem func(Model) (Item, bool, error),wrap func([]Item) Request,) error {+ unlock, err := s.syncLocks.LockContext(ctx, target.id+"\x00"+path)+ if err != nil {+ return fmt.Errorf("sync cancelled while waiting for an in-flight sync: %w", err)+ }+ defer unlock()+var records []Modelif loadRecordsErr := s.db.WithContext(ctx).Find(&records).Error; loadRecordsErr != nil {return fmt.Errorf("failed to get %s: %w", kind, loadRecordsErr)diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.goindex bc204fbf2..5841c1c33 100644--- a/backend/internal/environment/handler.go+++ b/backend/internal/environment/handler.go@@ -453,7 +453,7 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updat}}(context.WithoutCancel(ctx), input.ID, updated.Name)}- if kit.FromPtr(updated.AccessToken) != "" && (kit.FromPtr(input.Body.AccessToken) != "" || input.Body.Name != nil) {+ if kit.FromPtr(updated.AccessToken) != "" && kit.FromPtr(input.Body.AccessToken) != "" {h.triggerEnvironmentResourceSync(ctx, input.ID, updated.Name, "environment update")}diff --git a/backend/internal/environment/helpers.go b/backend/internal/environment/helpers.goindex 5086ed58a..d9b7479ea 100644--- a/backend/internal/environment/helpers.go+++ b/backend/internal/environment/helpers.go@@ -6,6 +6,7 @@ import ("sync""github.com/samber/hot"+ "go.getarcane.app/kit/pkg")// edgeTokenCache maps agent tokens to environment IDs; the reverse index drops@@ -16,6 +17,12 @@ type edgeTokenCache struct {byEnvID map[string]string}+type accessTokenIndex struct {+ mu sync.RWMutex+ byEnvID map[string]string+ byHashed map[string]int+}+// remoteEnvSnapshotCache holds every enabled, visible, non-local environment// in memory so hot paths avoid a DB round trip.type remoteEnvSnapshotCache struct {@@ -38,6 +45,12 @@ const environmentCacheStripes = 64var environmentCacheSeed = maphash.MakeSeed()+type remoteSyncQueue struct {+ mu sync.Mutex+ running map[string]bool+ pending map[string]bool+}+// runtimeWatchers fans a coalesced wake-up out to environment liveness watchers.type runtimeWatchers struct {mu sync.Mutex@@ -64,6 +77,28 @@ func (c *edgeTokenCache) sync(envID, token string) {}}+func (i *accessTokenIndex) sync(envID, token string) {+ if envID == "" {+ return+ }++ i.mu.Lock()+ defer i.mu.Unlock()++ if previousHash, ok := i.byEnvID[envID]; ok {+ delete(i.byEnvID, envID)+ i.byHashed[previousHash]--+ if i.byHashed[previousHash] <= 0 {+ delete(i.byHashed, previousHash)+ }+ }+ if resolvedToken := strings.TrimSpace(token); resolvedToken != "" {+ hashed := kit.SHA256Hex(resolvedToken)+ i.byEnvID[envID] = hashed+ i.byHashed[hashed]+++ }+}+func isActiveRemoteEnvironment(env Environment) bool {return env.ID != "" && env.ID != "0" && env.Enabled && !env.Hidden}diff --git a/backend/internal/environment/service.go b/backend/internal/environment/service.goindex e44bdee79..f6e5ac13c 100644--- a/backend/internal/environment/service.go+++ b/backend/internal/environment/service.go@@ -13,6 +13,7 @@ import ("net/url""slices""strings"+ "sync""sync/atomic""time""uuid"@@ -68,10 +69,13 @@ type EnvironmentService struct {environmentCache *hot.HotCache[environmentCacheKey, Environment]environmentGens [environmentCacheStripes]atomic.Uint64accessTokens *hot.HotCache[string, string]+ accessTokenIndex *accessTokenIndexjobs *entityjobs.RegistryvariableSyncer VariableSyncer+ invalidateUser func(userID string)runtimeWatchers runtimeWatchers+ remoteSyncs remoteSyncQueuesyncGate *utils.SyncGateproxy *proxy.Servicehealth *health.Service@@ -162,13 +166,21 @@ func NewEnvironmentService(WithMissingSharedCache().WithJanitor().Build(),- accessTokens: hot.NewHotCache[string, string](hot.LRU, 1024).WithTTL(environmentCacheTTL).WithJanitor().Build(),+ accessTokens: hot.NewHotCache[string, string](hot.LRU, 1024).WithTTL(environmentCacheTTL).WithJanitor().Build(),+ accessTokenIndex: &accessTokenIndex{+ byEnvID: make(map[string]string),+ byHashed: make(map[string]int),+ },jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScope),runtimeWatchers: runtimeWatchers{chans: make(map[int]chan struct{})},- proxy: proxy.New(db, httpClient, settingsService, syncGate),- health: health.New(httpClient, dockerService),- snippets: snippets.New(eventService),- swarm: swarm.New(apiKeyService),+ remoteSyncs: remoteSyncQueue{+ running: make(map[string]bool),+ pending: make(map[string]bool),+ },+ proxy: proxy.New(db, httpClient, settingsService, syncGate),+ health: health.New(httpClient, dockerService),+ snippets: snippets.New(eventService),+ swarm: swarm.New(apiKeyService),}}@@ -219,6 +231,10 @@ func (s *EnvironmentService) SetVariableSyncer(syncer VariableSyncer) {s.variableSyncer = syncer}+func (s *EnvironmentService) SetUserInvalidator(invalidate func(userID string)) {+ s.invalidateUser = invalidate+}+func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context, token string) (string, error) {token = strings.TrimSpace(token)if token == "" {@@ -227,7 +243,15 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,staleEnvironmentID, wasCached := s.edgeTokens.byToken.Peek(token)if envID, ok, _ := s.edgeTokens.byToken.Get(token); ok {- return envID, nil+ cached, err := s.GetEnvironmentByIDCached(ctx, envID)+ if err != nil {+ return "", fmt.Errorf("failed to resolve cached edge environment: %w", err)+ }+ if cached.Enabled {+ return envID, nil+ }+ s.edgeTokens.sync(envID, "")+ return "", errors.New("invalid agent token")}if wasCached {s.edgeTokens.mu.Lock()@@ -241,6 +265,7 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,if err := s.db.WithContext(ctx).Select("id", "access_token").Where("is_edge = ?", true).+ Where("enabled = ?", true).Where("access_token = ?", token).First(&env).Error; err != nil {if !errors.Is(err, gorm.ErrRecordNotFound) {@@ -359,12 +384,13 @@ func (s *EnvironmentService) CreateEnvironment(ctx context.Context, env *Environreturn nil, fmt.Errorf("failed to create environment: %w", err)}- go s.createEnvironmentEvent(context.WithoutCancel(ctx), env.ID, env.Name, event.EventTypeEnvironmentCreate,+ s.createEnvironmentEvent(ctx, env.ID, env.Name, event.EventTypeEnvironmentCreate,"Environment Created", fmt.Sprintf("Environment '%s' was created", env.Name), event.EventSeveritySuccess, userID, username)if env.Enabled {s.registerHealthJob(ctx, env.ID)}+ s.accessTokenIndex.sync(env.ID, kit.FromPtr(env.AccessToken))s.remoteEnvs.put(*env)s.invalidateEnvironmentCache(env.ID)s.NotifyRuntimeStateChanged()@@ -431,9 +457,9 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, u}s.remoteEnvs.put(*updated)- if rawAccessToken, ok := updates["access_token"]; ok {- accessToken, _ := rawAccessToken.(string)- s.edgeTokens.sync(id, accessToken)+ if _, ok := updates["access_token"]; ok {+ s.edgeTokens.sync(id, "")+ s.accessTokenIndex.sync(id, kit.FromPtr(updated.AccessToken))}// Reconcile the per-environment health job when the enabled flag is toggled.@@ -443,12 +469,14 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, us.registerHealthJob(ctx, id)} else {s.jobs.Unregister(ctx, id)+ s.edgeTokens.sync(id, "")+ edge.GetRegistry().Unregister(id)}}}if id != LocalEnvironmentID {- go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, updated.Name, event.EventTypeEnvironmentUpdate,+ s.createEnvironmentEvent(ctx, id, updated.Name, event.EventTypeEnvironmentUpdate,"Environment Updated", fmt.Sprintf("Environment '%s' was updated", updated.Name), event.EventSeverityInfo, userID, username)}@@ -464,7 +492,7 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u// Stop the per-environment health job before the row is removed.s.jobs.Unregister(ctx, id)- var syncIDs []string+ var syncIDs, federatedUserIDs []stringif transactionErr := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {if loadSyncIDsErr := tx.Table("gitops_syncs").Where("environment_id = ?", id).@@ -482,6 +510,29 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, ureturn fmt.Errorf("failed to delete environment gitops syncs: %w", deleteSyncsErr)}}+ if loadFederatedUsersErr := tx.Table("federated_credentials").+ Where("environment_id = ?", id).+ Pluck("identity_user_id", &federatedUserIDs).Error; loadFederatedUsersErr != nil {+ return fmt.Errorf("failed to list environment federated credentials: %w", loadFederatedUsersErr)+ }+ if len(federatedUserIDs) > 0 {+ if deleteCredentialsErr := tx.Exec("DELETE FROM federated_credentials WHERE environment_id = ?", id).Error; deleteCredentialsErr != nil {+ return fmt.Errorf("failed to delete environment federated credentials: %w", deleteCredentialsErr)+ }+ if deleteUsersErr := tx.Exec("DELETE FROM users WHERE id IN ?", federatedUserIDs).Error; deleteUsersErr != nil {+ return fmt.Errorf("failed to delete environment federated service users: %w", deleteUsersErr)+ }+ }+ if deleteVariablesErr := tx.Exec(`DELETE FROM global_variables+ WHERE all_environments = ?+ AND id IN (SELECT global_variable_id FROM global_variable_environments WHERE environment_id = ?)+ AND id NOT IN (SELECT global_variable_id FROM global_variable_environments WHERE environment_id <> ?)`,+ false, id, id).Error; deleteVariablesErr != nil {+ return fmt.Errorf("failed to delete environment-scoped global variables: %w", deleteVariablesErr)+ }+ if deleteVariableScopesErr := tx.Exec("DELETE FROM global_variable_environments WHERE environment_id = ?", id).Error; deleteVariableScopesErr != nil {+ return fmt.Errorf("failed to delete environment global variable scopes: %w", deleteVariableScopesErr)+ }if deleteEnvironmentErr := tx.Delete(&Environment{}, "id = ?", id).Error; deleteEnvironmentErr != nil {return fmt.Errorf("failed to delete environment: %w", deleteEnvironmentErr)}@@ -501,7 +552,15 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u}}+ if s.invalidateUser != nil {+ for _, federatedUserID := range federatedUserIDs {+ s.invalidateUser(federatedUserID)+ }+ }+s.edgeTokens.sync(id, "")+ s.accessTokenIndex.sync(id, "")+ edge.GetRegistry().Unregister(id)s.ForgetSyncState(id)s.remoteEnvs.mu.Lock()delete(s.remoteEnvs.envs, id)@@ -510,7 +569,7 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, us.invalidateEnvironmentCache(id)s.NotifyRuntimeStateChanged()- go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, env.Name, event.EventTypeEnvironmentDelete,+ s.createEnvironmentEvent(ctx, id, env.Name, event.EventTypeEnvironmentDelete,"Environment Deleted", fmt.Sprintf("Environment '%s' was deleted", env.Name), event.EventSeverityWarning, userID, username)return nil@@ -524,7 +583,9 @@ func (s *EnvironmentService) createEnvironmentEvent(severity event.EventSeverity,userID, username *string,) {- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{+ eventCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)+ defer cancel()+ if _, err := s.eventService.CreateEvent(eventCtx, event.CreateEventRequest{Type: eventType,Severity: severity,Title: title,@@ -535,7 +596,9 @@ func (s *EnvironmentService) createEnvironmentEvent(UserID: userID,Username: username,EnvironmentID: new(envID),- })+ }); err != nil {+ slog.WarnContext(ctx, "Failed to record environment event", "environmentId", envID, "eventType", eventType, "error", err)+ }}// RegenerateEnvironmentApiKey rotates the key and updates its environment binding.@@ -566,7 +629,8 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, ns.invalidateEnvironmentCache(envID)s.NotifyRuntimeStateChanged()- s.edgeTokens.sync(envID, apiKey)+ s.edgeTokens.sync(envID, "")+ s.accessTokenIndex.sync(envID, apiKey)s.remoteEnvs.mu.Lock()s.remoteEnvs.revision++if envRecord, ok := s.remoteEnvs.envs[envID]; ok {@@ -591,8 +655,9 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, n}}s.remoteEnvs.mu.Unlock()+ edge.GetRegistry().Unregister(envID)- go s.createEnvironmentEvent(context.WithoutCancel(ctx), envID, envName, event.EventTypeEnvironmentApiKeyRegenerated,+ s.createEnvironmentEvent(ctx, envID, envName, event.EventTypeEnvironmentApiKeyRegenerated,"API Key Regenerated", "Environment API key was regenerated and status set to pending", event.EventSeverityInfo, new(userID), new(username))return nil@@ -630,6 +695,32 @@ func (s *EnvironmentService) ResolveEnvironmentByAccessToken(ctx context.Contextreturn &env, nil}+func (s *EnvironmentService) LoadAccessTokens(ctx context.Context) error {+ var envs []Environment+ if err := s.db.WithContext(ctx).+ Select("id", "access_token").+ Where("access_token IS NOT NULL AND access_token != ?", "").+ Find(&envs).Error; err != nil {+ return fmt.Errorf("failed to load environment access tokens: %w", err)+ }+ for _, env := range envs {+ s.accessTokenIndex.sync(env.ID, kit.FromPtr(env.AccessToken))+ }+ return nil+}++func (s *EnvironmentService) IsKnownAccessToken(token string) bool {+ token = strings.TrimSpace(token)+ if token == "" || len(token) > maxAccessTokenLength {+ return false+ }+ hashed := kit.SHA256Hex(token)+ s.accessTokenIndex.mu.RLock()+ defer s.accessTokenIndex.mu.RUnlock()+ _, ok := s.accessTokenIndex.byHashed[hashed]+ return ok+}+func (s *EnvironmentService) GetEnabledRegistryCredentials(ctx context.Context) ([]containerregistry.Credential, error) {var registries []registry.ContainerRegistryif err := s.db.WithContext(ctx).Where("enabled = ?", true).Find(®istries).Error; err != nil {@@ -710,8 +801,10 @@ func DisplayName(environmentID, storedName string) string {}const (- edgeTokenCacheTTL = time.Minute- environmentCacheTTL = 30 * time.Second+ edgeTokenCacheTTL = time.Minute+ environmentCacheTTL = 30 * time.Second+ remoteSyncTimeout = time.Minute+ maxAccessTokenLength = 128)// GetActiveRemoteEnvironmentSnapshot returns the latest in-process snapshot for@@ -782,7 +875,7 @@ func (s *EnvironmentService) invalidateEnvironmentCache(id string) {}func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, params pagination.QueryParams, accessibleEnvIDs []string) ([]environment.Environment, pagination.Response, error) {- if strings.TrimSpace(params.Filters["type"]) != "" {+ if strings.TrimSpace(params.Filters["type"]) != "" || strings.TrimSpace(params.Filters["status"]) != "" || params.Sort == "status" {var envs []Environmentif err := s.db.WithContext(ctx).Model(&Environment{}).@@ -866,6 +959,9 @@ func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, paraif !item.IsEdge {return strings.ToLower(strings.TrimSpace(filterValue)) == "http"}+ if strings.ToLower(strings.TrimSpace(filterValue)) == "polling" {+ return item.LastPollAt != nil+ }// Disconnected or poll-only agents classify by the transport they last used.switch strings.ToLower(strings.TrimSpace(kit.Ternary(kit.FromPtr(item.Connected) && item.EdgeTransport != nil,kit.FromPtr(item.EdgeTransport), kit.FromPtr(item.LastEdgeTransport)))) {@@ -942,7 +1038,7 @@ func (s *EnvironmentService) ListVisibleEnvironments(ctx context.Context) ([]env// ListRemoteEnvironmentIDs returns the IDs of enabled remote environments; it// satisfies agg.RemoteEnvironmentLister for aggregated stream handlers.func (s *EnvironmentService) ListRemoteEnvironmentIDs(ctx context.Context) ([]string, error) {- envs, err := s.ListRemoteEnvironments(ctx)+ envs, err := s.ListActiveRemoteEnvironments(ctx)if err != nil {return nil, err}@@ -955,37 +1051,32 @@ func (s *EnvironmentService) ListRemoteEnvironmentIDs(ctx context.Context) ([]st// ListRemoteEnvironments returns all non-local, enabled environments for syncing purposes.func (s *EnvironmentService) ListRemoteEnvironments(ctx context.Context) ([]Environment, error) {- for {- s.remoteEnvs.mu.RLock()- revision := s.remoteEnvs.revision- s.remoteEnvs.mu.RUnlock()+ s.remoteEnvs.mu.RLock()+ revision := s.remoteEnvs.revision+ s.remoteEnvs.mu.RUnlock()- var envs []Environment- err := s.db.WithContext(ctx).- Model(&Environment{}).- Where("id != ?", LocalEnvironmentID).- Where("enabled = ?", true).- Where("hidden = ?", false).- Find(&envs).Error- if err != nil {- return nil, fmt.Errorf("failed to list remote environments: %w", err)- }+ var envs []Environment+ if err := s.db.WithContext(ctx).+ Model(&Environment{}).+ Where("id != ?", LocalEnvironmentID).+ Where("enabled = ?", true).+ Where("hidden = ?", false).+ Find(&envs).Error; err != nil {+ return nil, fmt.Errorf("failed to list remote environments: %w", err)+ }- s.remoteEnvs.mu.Lock()- // Retry if a cache mutation or another refresh overtook the database read.- if s.remoteEnvs.revision != revision {- s.remoteEnvs.mu.Unlock()- continue- }- s.remoteEnvs.envs = make(map[string]Environment, len(envs))- for _, envRecord := range envs {- s.remoteEnvs.envs[envRecord.ID] = envRecord- }- s.remoteEnvs.seeded = true- s.remoteEnvs.revision++- s.remoteEnvs.mu.Unlock()+ s.remoteEnvs.mu.Lock()+ defer s.remoteEnvs.mu.Unlock()+ if s.remoteEnvs.revision != revision {return envs, nil}+ s.remoteEnvs.envs = make(map[string]Environment, len(envs))+ for _, envRecord := range envs {+ s.remoteEnvs.envs[envRecord.ID] = envRecord+ }+ s.remoteEnvs.seeded = true+ s.remoteEnvs.revision+++ return envs, nil}// SubscribeRuntimeChanges returns a channel that receives a coalesced wake-up@@ -1060,6 +1151,18 @@ func (s *EnvironmentService) ListEnabledEnvironmentIDs(ctx context.Context) ([]sreturn ids, nil}+func (s *EnvironmentService) ListEnabledRemoteEnvironments(ctx context.Context) ([]Environment, error) {+ var envs []Environment+ if err := s.db.WithContext(ctx).+ Model(&Environment{}).+ Where("enabled = ?", true).+ Where("id <> ?", LocalEnvironmentID).+ Find(&envs).Error; err != nil {+ return nil, fmt.Errorf("failed to list enabled remote environments: %w", err)+ }+ return envs, nil+}+// RescheduleHealthJobs re-registers every enabled environment's health job,// picking up a changed global interval.func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {@@ -1197,7 +1300,18 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, custreturn "error", err}- status, err := s.health.Probe(ctx, id, mo.PointerToOption(customApiUrl).OrElse(envRecord.ApiUrl), envRecord.IsEdge, customApiUrl != nil)+ status, err := s.health.Probe(ctx, id, mo.PointerToOption(customApiUrl).OrElse(envRecord.ApiUrl), envRecord.IsEdge, customApiUrl != nil,+ func(probeCtx context.Context) (int, error) {+ target, targetErr := proxy.NewTarget(envRecord.ID, envRecord.Name, envRecord.ApiUrl, envRecord.IsEdge, envRecord.AccessToken)+ if targetErr != nil {+ return 0, targetErr+ }+ resp, executeErr := s.proxy.Execute(probeCtx, target, http.MethodHead, "/api/environments/0/system/health", nil)+ if executeErr != nil {+ return 0, executeErr+ }+ return resp.StatusCode, nil+ })if customApiUrl == nil {if updateErr := s.updateEnvironmentStatus(ctx, id, status); updateErr != nil {slog.WarnContext(ctx, "Failed to persist environment status", "environmentId", id, "status", status, "error", updateErr)@@ -1212,29 +1326,25 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, cust}func (s *EnvironmentService) updateEnvironmentStatus(ctx context.Context, id, status string) error {- var currentEnv Environment- if err := s.db.WithContext(ctx).Select("status", "is_edge").Where("id = ?", id).First(¤tEnv).Error; err != nil {- return fmt.Errorf("failed to check environment status: %w", err)- }-- if currentEnv.Status == string(EnvironmentStatusPending) {- // Only a successful check pairs a direct environment; edge environments pair- // through the agent's tunnel, so manager reachability means nothing for them.- if currentEnv.IsEdge || status != string(EnvironmentStatusOnline) {- slog.DebugContext(ctx, "skipping status update for pending environment", "environmentId", id)- return nil- }- slog.InfoContext(ctx, "promoted pending direct environment to online via reachability check", "environmentId", id)- }-now := time.Now()updates := map[string]any{"status": status,- "last_seen": &now,"updated_at": &now,}- if err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {- return fmt.Errorf("failed to update environment status: %w", err)+ query := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id)+ if status == string(EnvironmentStatusOnline) {+ updates["last_seen"] = &now+ query = query.Where("status <> ? OR is_edge = ?", string(EnvironmentStatusPending), false)+ } else {+ query = query.Where("status <> ?", string(EnvironmentStatusPending))+ }+ result := query.Updates(updates)+ if result.Error != nil {+ return fmt.Errorf("failed to update environment status: %w", result.Error)+ }+ if result.RowsAffected == 0 {+ slog.DebugContext(ctx, "skipping status update for pending environment", "environmentId", id)+ return nil}// Direct environments have no tunnel callback, so this is their only liveness change.s.NotifyRuntimeStateChanged()@@ -1282,7 +1392,11 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contexupdates["status"] = string(EnvironmentStatusOffline)}- if err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {+ query := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id)+ if !connected {+ query = query.Where("status <> ?", string(EnvironmentStatusPending))+ }+ if err := query.Updates(updates).Error; err != nil {return fmt.Errorf("failed to update environment connection state: %w", err)}@@ -1370,26 +1484,59 @@ func (s *EnvironmentService) SyncS3DestinationsToRemoteEnvironments(ctx context.return s.syncToRemoteEnvironments(ctx, "S3 destinations", s.SyncS3DestinationsToEnvironment)}-func (s *EnvironmentService) syncToRemoteEnvironments(ctx context.Context, kind string, sync func(context.Context, string) error) error {+func (s *EnvironmentService) QueueRemoteSync(ctx context.Context, kind string, syncAll func(context.Context) error) {+ s.remoteSyncs.mu.Lock()+ defer s.remoteSyncs.mu.Unlock()+ if s.remoteSyncs.running[kind] {+ s.remoteSyncs.pending[kind] = true+ return+ }+ s.remoteSyncs.running[kind] = true++ go func(runCtx context.Context) {+ for {+ if err := syncAll(runCtx); err != nil {+ slog.WarnContext(runCtx, "Failed to fan out sync to remote environments", "kind", kind, "error", err)+ }+ s.remoteSyncs.mu.Lock()+ if !s.remoteSyncs.pending[kind] || runCtx.Err() != nil {+ delete(s.remoteSyncs.running, kind)+ delete(s.remoteSyncs.pending, kind)+ s.remoteSyncs.mu.Unlock()+ return+ }+ delete(s.remoteSyncs.pending, kind)+ s.remoteSyncs.mu.Unlock()+ }+ }(s.jobs.Context(ctx))+}++func (s *EnvironmentService) syncToRemoteEnvironments(ctx context.Context, kind string, syncEnvironment func(context.Context, string) error) error {envs, err := s.ListRemoteEnvironments(ctx)if err != nil {return fmt.Errorf("failed to list remote environments for %s sync: %w", kind, err)}- var failedCount int+ var failedCount atomic.Int32+ var wg sync.WaitGroupfor _, env := range envs {if strings.TrimSpace(kit.FromPtr(env.AccessToken)) == "" {slog.DebugContext(ctx, "Skipping sync for environment without access token", "kind", kind, "environmentId", env.ID, "environmentName", env.Name)continue}- if syncErr := sync(ctx, env.ID); syncErr != nil {- failedCount++- slog.WarnContext(ctx, "Failed to sync to remote environment", "kind", kind, "environmentId", env.ID, "environmentName", env.Name, "error", syncErr)- }+ wg.Go(func() {+ syncCtx, cancel := context.WithTimeout(ctx, remoteSyncTimeout)+ defer cancel()+ if syncErr := syncEnvironment(syncCtx, env.ID); syncErr != nil {+ failedCount.Add(1)+ slog.WarnContext(syncCtx, "Failed to sync to remote environment", "kind", kind, "environmentId", env.ID, "environmentName", env.Name, "error", syncErr)+ }+ })}+ wg.Wait()- if failedCount > 0 {- return fmt.Errorf("failed to sync %s to %d remote environment(s)", kind, failedCount)+ if failed := failedCount.Load(); failed > 0 {+ return fmt.Errorf("failed to sync %s to %d remote environment(s)", kind, failed)}return nil}diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.goindex 2fc853b14..21c6ee3dc 100644--- a/backend/internal/environment/service_test.go+++ b/backend/internal/environment/service_test.go@@ -72,6 +72,30 @@ type testGitOpsSyncRow struct {SyncInterval int}+type testFederatedCredentialRow struct {+ database.BaseModel+ Name string+ IdentityUserID string `gorm:"column:identity_user_id"`+ EnvironmentID *string `gorm:"column:environment_id"`+}++type testGlobalVariableRow struct {+ database.BaseModel+ Key string+ AllEnvironments bool `gorm:"column:all_environments"`+}++type testGlobalVariableEnvironmentRow struct {+ GlobalVariableID string `gorm:"column:global_variable_id;primaryKey"`+ EnvironmentID string `gorm:"column:environment_id;primaryKey"`+}++func (testFederatedCredentialRow) TableName() string { return "federated_credentials" }++func (testGlobalVariableRow) TableName() string { return "global_variables" }++func (testGlobalVariableEnvironmentRow) TableName() string { return "global_variable_environments" }+func TestEnvironmentService_OverlappingHealthCheckIsSkipped(t *testing.T) {gate := newAdmissionGateForEnvironmentTest(t)@@ -130,6 +154,9 @@ func setupEnvironmentServiceTestDB(t *testing.T) *database.DB {&event.Event{},&testProjectRow{},&testGitOpsSyncRow{},+ &testFederatedCredentialRow{},+ &testGlobalVariableRow{},+ &testGlobalVariableEnvironmentRow{},))sqlDB, err := db.DB()@@ -245,6 +272,80 @@ func TestEnvironmentService_DeleteEnvironment_CascadesGitOpsSyncs(t *testing.T)require.Contains(t, jobScheduler.removed, entityjobs.GitOpsSyncJobPrefix+syncID)}+func TestEnvironmentService_DeleteEnvironment_RemovesScopedCredentialsAndVariables(t *testing.T) {+ ctx := t.Context()+ db := setupEnvironmentServiceTestDB(t)++ createTestEnvironment(t, db, "env-delete-scoped", "http://scoped.example", nil)+ createTestEnvironment(t, db, "env-keep", "http://keep.example", nil)+ require.NoError(t, db.Create(&user.User{ID: "federated-user", Username: "federated-user"}).Error)+ require.NoError(t, db.Create(&testFederatedCredentialRow{+ ID: "credential-scoped",+ Name: "scoped",+ IdentityUserID: "federated-user",+ EnvironmentID: new("env-delete-scoped"),+ }).Error)+ require.NoError(t, db.Create(&[]testGlobalVariableRow{+ {ID: "var-only-deleted", Key: "ONLY_DELETED"},+ {ID: "var-shared", Key: "SHARED"},+ {ID: "var-all", Key: "ALL", AllEnvironments: true},+ }).Error)+ require.NoError(t, db.Create(&[]testGlobalVariableEnvironmentRow{+ {GlobalVariableID: "var-only-deleted", EnvironmentID: "env-delete-scoped"},+ {GlobalVariableID: "var-shared", EnvironmentID: "env-delete-scoped"},+ {GlobalVariableID: "var-shared", EnvironmentID: "env-keep"},+ }).Error)++ svc := NewEnvironmentService(db, http.DefaultClient, nil, event.NewEventService(db, nil, nil), nil, nil)+ var invalidatedUserIDs []string+ svc.SetUserInvalidator(func(userID string) { invalidatedUserIDs = append(invalidatedUserIDs, userID) })+ require.NoError(t, svc.DeleteEnvironment(ctx, "env-delete-scoped", nil, nil))+ require.Equal(t, []string{"federated-user"}, invalidatedUserIDs)++ var credentialCount, userCount int64+ require.NoError(t, db.Model(&testFederatedCredentialRow{}).Count(&credentialCount).Error)+ require.NoError(t, db.Model(&user.User{}).Where("id = ?", "federated-user").Count(&userCount).Error)+ require.Zero(t, credentialCount)+ require.Zero(t, userCount)++ var variableIDs []string+ require.NoError(t, db.Model(&testGlobalVariableRow{}).Order("id").Pluck("id", &variableIDs).Error)+ require.Equal(t, []string{"var-all", "var-shared"}, variableIDs)++ var scopes []testGlobalVariableEnvironmentRow+ require.NoError(t, db.Find(&scopes).Error)+ require.Equal(t, []testGlobalVariableEnvironmentRow{{GlobalVariableID: "var-shared", EnvironmentID: "env-keep"}}, scopes)+}++func TestEnvironmentService_IsKnownAccessToken_TracksTokenWrites(t *testing.T) {+ ctx := t.Context()+ db := setupEnvironmentServiceTestDB(t)+ createTestEnvironmentWithState(t, db, "env-known", "http://known.example", string(EnvironmentStatusOnline), false, new("seeded-token"))+ createTestEnvironmentWithState(t, db, "env-shared", "http://shared.example", string(EnvironmentStatusOnline), false, new("seeded-token"))+ svc := NewEnvironmentService(db, http.DefaultClient, nil, event.NewEventService(db, nil, nil), nil, nil)++ require.False(t, svc.IsKnownAccessToken("seeded-token"))+ require.NoError(t, svc.LoadAccessTokens(ctx))+ require.True(t, svc.IsKnownAccessToken("seeded-token"))+ require.False(t, svc.IsKnownAccessToken(strings.Repeat("x", maxAccessTokenLength+1)))++ require.NoError(t, svc.linkEnvironmentApiKey(ctx, "env-known", "key-rotated", "rotated-token", "", "", "env-known"))+ require.True(t, svc.IsKnownAccessToken("seeded-token"))+ require.True(t, svc.IsKnownAccessToken("rotated-token"))++ _, err := svc.UpdateEnvironment(ctx, "env-known", map[string]any{"access_token": "updated-token"}, nil, nil)+ require.NoError(t, err)+ require.False(t, svc.IsKnownAccessToken("rotated-token"))+ require.True(t, svc.IsKnownAccessToken("updated-token"))++ require.NoError(t, svc.DeleteEnvironment(ctx, "env-known", nil, nil))+ require.False(t, svc.IsKnownAccessToken("updated-token"))+ require.True(t, svc.IsKnownAccessToken("seeded-token"))++ require.NoError(t, svc.DeleteEnvironment(ctx, "env-shared", nil, nil))+ require.False(t, svc.IsKnownAccessToken("seeded-token"))+}+func createTestRegistry(t *testing.T, db *database.DB, id string) {t.Helper()@@ -737,9 +838,29 @@ func TestEnvironmentService_ResolveEdgeEnvironmentByToken_CachesAndInvalidatesOnrequire.NoError(t, err)require.Equal(t, "edge-auth", envID)- require.NoError(t, svc.DeleteEnvironment(ctx, "edge-auth", nil, nil))+ _, err = svc.UpdateEnvironment(ctx, "edge-auth", map[string]any{"enabled": false}, nil, nil)+ require.NoError(t, err)_, err = svc.ResolveEdgeEnvironmentByToken(ctx, newToken)require.Error(t, err)++ require.NoError(t, svc.linkEnvironmentApiKey(ctx, "edge-auth", "key-rotated", "edge-token-rotated", "", "", "edge-auth"))+ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")+ require.Error(t, err)++ _, err = svc.UpdateEnvironment(ctx, "edge-auth", map[string]any{"enabled": true}, nil, nil)+ require.NoError(t, err)+ envID, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")+ require.NoError(t, err)+ require.Equal(t, "edge-auth", envID)++ require.NoError(t, db.WithContext(ctx).Model(&Environment{}).Where("id = ?", "edge-auth").Update("enabled", false).Error)+ svc.invalidateEnvironmentCache("edge-auth")+ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")+ require.Error(t, err)++ require.NoError(t, svc.DeleteEnvironment(ctx, "edge-auth", nil, nil))+ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")+ require.Error(t, err)require.Contains(t, err.Error(), "invalid agent token")}diff --git a/backend/internal/event/service.go b/backend/internal/event/service.goindex aa4cc725e..4c3aac2ae 100644--- a/backend/internal/event/service.go+++ b/backend/internal/event/service.go@@ -39,11 +39,15 @@ import ()// maxConcurrentEdgeSyncs bounds in-flight event forwards to the manager.-const maxConcurrentEdgeSyncs = 16+const (+ maxConcurrentEdgeSyncs = 16+ edgeSyncQueueTimeout = time.Minute+ edgeSyncPublishTimeout = 10 * time.Second+)const (daemonEventQueueSize = 1024- daemonEventBatchSize = 64+ DaemonEventBatchSize = 64daemonEventFlushInterval = 250 * time.Millisecond)@@ -193,15 +197,19 @@ func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Ev}go func(parentCtx context.Context, outgoing *edge.TunnelEvent) {- syncCtx, cancel := context.WithTimeout(context.WithoutCancel(parentCtx), 10*time.Second)- defer cancel()-- if err := s.edgeSyncs.Acquire(syncCtx, 1); err != nil {- slog.WarnContext(syncCtx, "Dropped event sync to manager; too many in flight", "type", outgoing.Type, "error", err)+ baseCtx := context.WithoutCancel(parentCtx)+ acquireCtx, cancelAcquire := context.WithTimeout(baseCtx, edgeSyncQueueTimeout)+ acquireErr := s.edgeSyncs.Acquire(acquireCtx, 1)+ cancelAcquire()+ if acquireErr != nil {+ slog.WarnContext(baseCtx, "Dropped event sync to manager; too many in flight", "type", outgoing.Type, "error", acquireErr)return}defer s.edgeSyncs.Release(1)+ syncCtx, cancel := context.WithTimeout(baseCtx, edgeSyncPublishTimeout)+ defer cancel()+if err := edge.PublishEventToManager(syncCtx, outgoing); err != nil {if !errors.Is(err, edge.ErrNoActiveAgentTunnel) {slog.WarnContext(syncCtx, "Failed to sync event to manager over edge tunnel", "type", outgoing.Type, "error", err)@@ -718,7 +726,7 @@ func (s *EventService) RecordDockerEvent(ctx context.Context, msg events.Messagefunc (s *EventService) writeDaemonEvents(ctx context.Context) {defer close(s.daemonDone)- batch := make([]CreateEventRequest, 0, daemonEventBatchSize)+ batch := make([]CreateEventRequest, 0, DaemonEventBatchSize)var flush <-chan time.Timefor open := true; open; {select {@@ -729,7 +737,7 @@ func (s *EventService) writeDaemonEvents(ctx context.Context) {if flush == nil {flush = time.After(daemonEventFlushInterval)}- if len(batch) < daemonEventBatchSize {+ if len(batch) < DaemonEventBatchSize {continue}}diff --git a/backend/internal/event/service_test.go b/backend/internal/event/service_test.goindex a3a2e6f26..e2a1a374f 100644--- a/backend/internal/event/service_test.go+++ b/backend/internal/event/service_test.go@@ -843,7 +843,7 @@ func TestDaemonEventWriterBatchesAndFlushesOnStop(t *testing.T) {svc.RecordDockerEvent(t.Context(), events.Message{Type: events.ContainerEventType, Action: events.ActionStart, TimeNano: timeNano, Actor: events.Actor{ID: "web"}})}- const occurrences = daemonEventBatchSize*3 + 5+ const occurrences = DaemonEventBatchSize*3 + 5var wg sync.WaitGroupfor occurrence := range int64(occurrences) {for range 2 {diff --git a/backend/internal/middleware/environment.go b/backend/internal/middleware/environment.goindex e40296d4a..1f09b2374 100644--- a/backend/internal/middleware/environment.go+++ b/backend/internal/middleware/environment.go@@ -11,6 +11,7 @@ import ("log/slog""mime""mime/multipart"+ "net""net/http""os""path"@@ -37,14 +38,15 @@ const (apiEnvironmentsPrefix = "/api/environments/"environmentsPathMarker = "/environments/"- // proxyTimeout is generous because proxied operations such as image pulls can stream for minutes.- proxyTimeout = 30 * time.MinuteproxyDialTimeout = 5 * time.SecondmaxProxiedWorkspaceManifestBytes = 1024 * 1024+ maxProxiedWorkspacePrefixBytes = 16 * 1024 * 1024maxProxiedGitOpsSyncBodyBytes = 1024 * 1024)+var errProxiedBodyTooLarge = errors.New("proxied request body is too large")+// managementEndpointSet contains paths handled locally and never proxied to remote environments.var managementEndpointSet = map[string]struct{}{"/test": {},@@ -95,7 +97,11 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(}// Traced so direct agents continue the manager's trace.- httpClient := httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second, DialTimeout: proxyDialTimeout})+ httpClient := httpx.NewHTTPClient(httpxtypes.ClientOptions{+ TLSHandshakeTimeout: 10 * time.Second,+ DialTimeout: proxyDialTimeout,+ ResponseHeaderTimeout: edge.ProxyResponseHeaderTimeout,+ })httpClient.Transport = otelhttp.NewTransport(httpClient.Transport)m := &EnvironmentMiddleware{@@ -175,13 +181,23 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e}// SECURITY: Remote agents trust the manager as sudo, so per-environment authorization happens here.- if m.proxyPermissionDenied(c, perms, envID, suffix) {- return writeProxyError(c, http.StatusForbidden, "You don't have permission to perform this action on this environment")+ status := m.proxyPermissionStatus(c, perms, envID, suffix)+ if replay, isReplay := req.Body.(*proxiedReplayBody); isReplay {+ defer func() { _ = replay.Close() }()+ }+ switch status {+ case http.StatusOK:+ case http.StatusForbidden:+ return writeProxyError(c, status, "You don't have permission to perform this action on this environment")+ case http.StatusRequestEntityTooLarge:+ return writeProxyError(c, status, "Request body is too large")+ default:+ return writeProxyError(c, status, "Invalid request body")}tunnel, unavailable := m.activeTunnel(ctx, envID, apiURL)if unavailable {- return writeProxyError(c, http.StatusBadGateway, "Edge agent is not connected")+ return writeProxyError(c, http.StatusServiceUnavailable, "Edge agent is not connected")}proxyPath := path.Join(apiEnvironmentsPrefix, m.localID) + suffix@@ -200,7 +216,7 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e}// A multiplexed tunnel carries plain HTTP, so it shares the direct-agent path below.apiURL = edge.MuxAgentBaseURL- client = &http.Client{Transport: muxClient.Transport, Timeout: m.httpClient.Timeout}+ client = muxClient}target := strings.TrimRight(apiURL, "/") + proxyPath@@ -249,7 +265,8 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) eresp, err := client.Do(proxyReq)if err != nil {- return writeProxyError(c, http.StatusBadGateway, "Proxy request failed: "+err.Error())+ netErr, isNetErr := errors.AsType[net.Error](err)+ return writeProxyError(c, kit.Ternary(isNetErr && netErr.Timeout(), http.StatusGatewayTimeout, http.StatusBadGateway), "Proxy request failed: "+err.Error())}defer func() { _ = resp.Body.Close() }()@@ -272,7 +289,7 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e}// activeTunnel returns the environment's live tunnel, if any. Edge environments refresh poll demand on every-// request and wait briefly for the agent; unavailable reports an edge agent that never connected.+// request and wait briefly only for an agent with a fresh poll check-in; unavailable reports an offline edge agent.func (m *EnvironmentMiddleware) activeTunnel(ctx context.Context, envID, apiURL string) (tunnel *edge.AgentTunnel, unavailable bool) {if !strings.HasPrefix(strings.ToLower(strings.TrimSpace(apiURL)), "edge://") {tunnel, _ = m.registry.Active(envID).Get()@@ -282,6 +299,10 @@ func (m *EnvironmentMiddleware) activeTunnel(ctx context.Context, envID, apiURLif active, ok := m.registry.Active(envID).Get(); ok {return active, false}+ if edge.GetPollRuntimeRegistry().Get(envID, time.Now()).IsAbsent() {+ slog.WarnContext(ctx, "No active edge tunnel or recent poll check-in for environment", "environmentId", envID)+ return nil, true+ }tunnel, ok := m.registry.WaitForActive(ctx, envID, edge.DefaultTunnelAcquireTimeout()).Get()if !ok {slog.WarnContext(ctx, "No active edge tunnel for environment", "environmentId", envID)@@ -291,11 +312,11 @@ func (m *EnvironmentMiddleware) activeTunnel(ctx context.Context, envID, apiURLreturn tunnel, false}-// proxyPermissionDenied reports whether the caller lacks permission for the proxied request, mirroring local+// proxyPermissionStatus returns http.StatusOK when the caller may perform the proxied request, mirroring local// RequirePermission checks. Sudo bypasses it; routes without a permission mapping are denied by default.-func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz.PermissionSet, envID, suffix string) bool {+func (m *EnvironmentMiddleware) proxyPermissionStatus(c *echo.Context, ps *authz.PermissionSet, envID, suffix string) int {if m.matcher == nil || (ps != nil && ps.Sudo) {- return false+ return http.StatusOK}req := c.Request()@@ -321,22 +342,22 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authzslog.DebugContext(ctx, "Denying proxied upload session request: permission denied","method", method, "path", suffix, "kind", segments[1], "environmentId", envID)}- return denied+ return kit.Ternary(denied, http.StatusForbidden, http.StatusOK)}}perm, ok := m.matcher.Lookup(method, suffix).Get()if !ok {slog.WarnContext(ctx, "Denying proxied request with no known permission mapping", "method", method, "path", suffix, "environmentId", envID)- return true+ return http.StatusForbidden}if perm == "" {// Explicitly public route: allowed for any authenticated caller.- return false+ return http.StatusOK}if !ps.Allows(perm, kit.Ternary(authz.IsEnvScoped(perm), envID, "")) {slog.DebugContext(ctx, "Denying proxied request: permission denied", "method", method, "path", suffix, "permission", perm, "environmentId", envID)- return true+ return http.StatusForbidden}var required []string@@ -347,8 +368,8 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authzrequired, err = proxiedGitOpsSyncPermissions(req, method, segments)}if err != nil {- slog.DebugContext(ctx, "Denying proxied request with invalid body", "path", suffix, "environmentId", envID, "error", err)- return true+ slog.DebugContext(ctx, "Rejecting proxied request with invalid body", "path", suffix, "environmentId", envID, "error", err)+ return kit.Ternary(errors.Is(err, errProxiedBodyTooLarge), http.StatusRequestEntityTooLarge, http.StatusBadRequest)}// Resource-sorted container lists collect per-container stats, so they also need containers:read.if method == http.MethodGet && len(segments) == 1 && segments[0] == "containers" && container.IsResourceSort(req.URL.Query().Get("sort")) {@@ -358,10 +379,10 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authzif !ps.Allows(operationPermission, envID) {slog.DebugContext(ctx, "Denying proxied request: body-derived permission denied","path", suffix, "permission", operationPermission, "environmentId", envID)- return true+ return http.StatusForbidden}}- return false+ return http.StatusOK}func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error) {@@ -384,9 +405,13 @@ func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error)request.Body = &proxiedReplayBody{Reader: io.MultiReader(captured, originalBody), captured: captured, original: originalBody}}()- reader := multipart.NewReader(io.TeeReader(originalBody, captured), params["boundary"])+ prefix := &io.LimitedReader{R: originalBody, N: maxProxiedWorkspacePrefixBytes}+ reader := multipart.NewReader(io.TeeReader(prefix, captured), params["boundary"])for {part, nextErr := reader.NextPart()+ if nextErr != nil && prefix.N == 0 {+ return nil, fmt.Errorf("find workspace manifest part: %w", errProxiedBodyTooLarge)+ }if nextErr != nil {return nil, fmt.Errorf("find workspace manifest part: %w", nextErr)}@@ -398,6 +423,9 @@ func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error)}manifestJSON, readErr := io.ReadAll(io.LimitReader(part, maxProxiedWorkspaceManifestBytes+1))closeErr := part.Close()+ if readErr != nil && prefix.N == 0 {+ return nil, fmt.Errorf("read workspace manifest: %w", errProxiedBodyTooLarge)+ }if readErr != nil {return nil, fmt.Errorf("read workspace manifest: %w", readErr)}@@ -405,7 +433,7 @@ func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error)return nil, fmt.Errorf("close workspace manifest part: %w", closeErr)}if len(manifestJSON) > maxProxiedWorkspaceManifestBytes {- return nil, errors.New("workspace manifest is too large")+ return nil, fmt.Errorf("workspace manifest: %w", errProxiedBodyTooLarge)}var manifest volume.WorkspaceUpdateManifestif unmarshalErr := json.Unmarshal(manifestJSON, &manifest); unmarshalErr != nil {@@ -435,7 +463,7 @@ func proxiedGitOpsSyncPermissions(request *http.Request, method string, segmentsreturn nil, fmt.Errorf("read gitops sync body: %w", err)}if len(body) > maxProxiedGitOpsSyncBodyBytes {- return nil, errors.New("gitops sync body is too large")+ return nil, fmt.Errorf("gitops sync body: %w", errProxiedBodyTooLarge)}request.Body = io.NopCloser(bytes.NewReader(body))diff --git a/backend/internal/middleware/environment_test.go b/backend/internal/middleware/environment_test.goindex 08985490c..51f1c54bc 100644--- a/backend/internal/middleware/environment_test.go+++ b/backend/internal/middleware/environment_test.go@@ -34,7 +34,7 @@ func newTestEnvironmentMiddleware() *EnvironmentMiddleware {_ = creturn authz.SudoPermissionSet(), nil, true},- httpClient: &http.Client{Timeout: proxyTimeout},+ httpClient: &http.Client{},registry: edge.NewTunnelRegistry(),}}@@ -49,7 +49,7 @@ func attachMiddleware(router *echo.Echo, mw *EnvironmentMiddleware) *echo.Groupreturn api}-func TestEnvironmentMiddleware_ReturnsBadGatewayForEdgeResourcesWithoutTunnel(t *testing.T) {+func TestEnvironmentMiddleware_ReturnsServiceUnavailableForEdgeResourcesWithoutTunnel(t *testing.T) {middleware := newTestEnvironmentMiddleware()router := echo.New()api := attachMiddleware(router, middleware)@@ -65,7 +65,7 @@ func TestEnvironmentMiddleware_ReturnsBadGatewayForEdgeResourcesWithoutTunnel(trouter.ServeHTTP(recorder, req)- assert.Equal(t, http.StatusBadGateway, recorder.Code)+ assert.Equal(t, http.StatusServiceUnavailable, recorder.Code)assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")assert.False(t, localHandlerHit)}@@ -86,7 +86,7 @@ func TestEnvironmentMiddleware_ProxiesDashboardResourcesForRemoteEnvironments(trouter.ServeHTTP(recorder, req)- assert.Equal(t, http.StatusBadGateway, recorder.Code)+ assert.Equal(t, http.StatusServiceUnavailable, recorder.Code)assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")assert.False(t, localHandlerHit)}@@ -257,7 +257,7 @@ func TestEnvironmentMiddleware_LocalEnvironmentSkipsProxyPermissionCheck(t *test_ = creturn authz.NewPermissionSet(), nil, true},- httpClient: &http.Client{Timeout: proxyTimeout},+ httpClient: &http.Client{},registry: edge.NewTunnelRegistry(),}@@ -357,7 +357,7 @@ func TestEnvironmentMiddleware_ForwardsResolvedIconCatalogHeaderOnly(t *testing.u.Preferences.IconCatalog = tt.catalogreturn authz.SudoPermissionSet(), u, true},- httpClient: &http.Client{Timeout: proxyTimeout},+ httpClient: &http.Client{},registry: edge.NewTunnelRegistry(),}@@ -428,7 +428,7 @@ func TestProxyPermissionDeniedBlocksWriteForReadOnlyUser(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers/abc/restart"),"expected restart to be denied for a read-only user")}@@ -439,7 +439,7 @@ func TestProxyPermissionDeniedAllowsWriteForPermittedUser(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers/abc/restart"),"expected restart to be allowed for a user with containers:restart")}@@ -450,7 +450,7 @@ func TestProxyPermissionDeniedAllowsRead(t *testing.T) {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers"),"expected list to be allowed for a user with containers:list")}@@ -462,7 +462,7 @@ func TestProxyPermissionDeniedDeniesPermissionFromDifferentEnv(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers/abc/restart"),"expected denial: permission is scoped to a different environment")}@@ -470,7 +470,7 @@ func TestProxyPermissionDeniedSudoBypasses(t *testing.T) {m := newProxyAuthzMiddleware(containerMatcher())c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")- require.False(t, m.proxyPermissionDenied(c, authz.SudoPermissionSet(), proxyTestEnvID, "/containers/abc/restart"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, authz.SudoPermissionSet(), proxyTestEnvID, "/containers/abc/restart"),"expected sudo permission set to bypass the permission check")}@@ -481,7 +481,7 @@ func TestProxyPermissionDeniedDefaultDeniesUnmappedRoute(t *testing.T) {c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/unknown/resource")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/unknown/resource"),+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/unknown/resource"),"expected an unmapped proxied route to be denied by default")}@@ -491,7 +491,7 @@ func TestProxyPermissionDeniedAllowsPublicRoute(t *testing.T) {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/settings/public")- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/settings/public"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/settings/public"),"expected an explicitly public route to be allowed for any authenticated caller")}@@ -530,23 +530,23 @@ func TestProxyPermissionDeniedChecksVolumeWorkspaceManifestPermissions(t *testinreadOnly := authz.NewPermissionSet()readOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead)c, _ := newProxyVolumeWorkspaceContext(t, changes)- require.True(t, m.proxyPermissionDenied(c, readOnly, proxyTestEnvID, "/volumes/data/workspace"))+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, readOnly, proxyTestEnvID, "/volumes/data/workspace"))uploadOnly := authz.NewPermissionSet()uploadOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload)c, _ = newProxyVolumeWorkspaceContext(t, changes)- require.True(t, m.proxyPermissionDenied(c, uploadOnly, proxyTestEnvID, "/volumes/data/workspace"))+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, uploadOnly, proxyTestEnvID, "/volumes/data/workspace"))permitted := authz.NewPermissionSet()permitted.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload, authz.PermVolumesDelete)c, rawBody := newProxyVolumeWorkspaceContext(t, changes)- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))replayed, err := io.ReadAll(c.Request().Body)require.NoError(t, err)require.Equal(t, rawBody, replayed)c, rawBody = newProxyVolumeWorkspaceContext(t, changes, true)- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))replayed, err = io.ReadAll(c.Request().Body)require.NoError(t, err)require.Equal(t, rawBody, replayed)@@ -568,13 +568,13 @@ func TestProxyPermissionDeniedChecksGitOpsImportLifecyclePermission(t *testing.TcreateOnly := authz.NewPermissionSet()createOnly.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate)- require.True(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))- require.False(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))withLifecycle := authz.NewPermissionSet()withLifecycle.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate, authz.PermGitOpsLifecycle)c := newProxyGitOpsImportContext(hookBody)- require.False(t, m.proxyPermissionDenied(c, withLifecycle, proxyTestEnvID, "/gitops-syncs/import"))+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, withLifecycle, proxyTestEnvID, "/gitops-syncs/import"))replayed, err := io.ReadAll(c.Request().Body)require.NoError(t, err)require.Equal(t, hookBody, string(replayed))@@ -592,13 +592,13 @@ func TestProxyPermissionDeniedWSTerminalRequiresExec(t *testing.T) {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/ws/containers/abc/terminal")- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),"expected WS terminal to be denied without containers:exec")// Granting containers:exec allows the same stream.ps.AddEnv(proxyTestEnvID, authz.PermContainersExec)- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),"expected WS terminal to be allowed with containers:exec")}@@ -610,22 +610,22 @@ func TestProxyPermissionDeniedResourceSortRequiresRead(t *testing.T) {for _, sort := range []string{"cpuUsage", "memoryUsage"} {c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort="+sort)- require.True(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, listOnly, proxyTestEnvID, "/containers"),"expected resource sort %q to be denied with only containers:list", sort)}c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=name")- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, listOnly, proxyTestEnvID, "/containers"),"expected non-resource sorts to stay allowed with containers:list")c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, listOnly, proxyTestEnvID, "/containers"),"expected the plain list to stay allowed with containers:list")withRead := authz.NewPermissionSet()withRead.AddEnv(proxyTestEnvID, authz.PermContainersList, authz.PermContainersRead)c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=memoryUsage&order=desc")- require.False(t, m.proxyPermissionDenied(c, withRead, proxyTestEnvID, "/containers"),+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, withRead, proxyTestEnvID, "/containers"),"expected resource sort to be allowed with containers:list and containers:read")}diff --git a/backend/internal/middleware/ratelimit.go b/backend/internal/middleware/ratelimit.goindex 44f10e0f7..b59d83ba5 100644--- a/backend/internal/middleware/ratelimit.go+++ b/backend/internal/middleware/ratelimit.go@@ -164,6 +164,7 @@ func PerTokenRateLimitForPaths(paths []string,perMinute int,burst int,+ tokenOf func(c *echo.Context) string,isKnown func(token string) bool,) echo.MiddlewareFunc {if perMinute <= 0 {@@ -186,7 +187,7 @@ func PerTokenRateLimitForPaths(return next(c)}- token := c.Param("token")+ token := tokenOf(c)if token != "" && isKnown != nil && isKnown(token) {if !tokenLimiter.allow(kit.SHA256Hex(token)) {c.Response().Header().Set("Retry-After", "60")diff --git a/backend/internal/middleware/ratelimit_test.go b/backend/internal/middleware/ratelimit_test.goindex 4545d59c7..9a123f732 100644--- a/backend/internal/middleware/ratelimit_test.go+++ b/backend/internal/middleware/ratelimit_test.go@@ -4,6 +4,7 @@ import ("net/http""net/http/httptest""strconv"+ "strings""testing""time"@@ -175,7 +176,7 @@ func newTokenRateLimitTestRouterInternal(router := echo.New()router.IPExtractor = echo.ExtractIPDirect()- router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, perMinute, burst, isAuthentic))+ router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, perMinute, burst, func(c *echo.Context) string { return c.Param("token") }, isAuthentic))router.POST("/webhooks/trigger/:token", handler)return func(token string) int {@@ -273,7 +274,7 @@ func TestPerTokenRateLimitForPaths_AppliesOnlyToConfiguredPaths(t *testing.T) {router := echo.New()router.IPExtractor = echo.ExtractIPDirect()- router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, 60, 1, nil))+ router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, 60, 1, func(c *echo.Context) string { return c.Param("token") }, nil))router.POST("/webhooks/trigger/:token", func(c *echo.Context) error {return c.NoContent(http.StatusOK)})@@ -328,3 +329,29 @@ func TestIPRateLimiter_ProtectsCurrentKeyDuringSweep(t *testing.T) {require.Contains(t, limiter.limiters, "current")require.NotContains(t, limiter.limiters, "other")}++func TestPerTokenRateLimitForPaths_KeysByAgentTokenHeader(t *testing.T) {+ router := echo.New()+ router.IPExtractor = echo.ExtractIPDirect()+ router.Use(PerTokenRateLimitForPaths(+ []string{"/events"}, 60, 1,+ func(c *echo.Context) string { return c.Request().Header.Get(HeaderAgentToken) },+ func(token string) bool { return strings.HasPrefix(token, "known-") },+ ))+ router.POST("/events", okHandlerInternal)+ doReq := func(token string) int {+ req := httptest.NewRequest(http.MethodPost, "/events", http.NoBody)+ req.RemoteAddr = "192.0.2.10:4000"+ req.Header.Set(HeaderAgentToken, token)+ rec := httptest.NewRecorder()+ router.ServeHTTP(rec, req)+ return rec.Code+ }++ require.Equal(t, http.StatusOK, doReq("known-a"))+ require.Equal(t, http.StatusOK, doReq("known-b"))+ require.Equal(t, http.StatusTooManyRequests, doReq("known-a"))+ require.Equal(t, http.StatusOK, doReq("unknown-a"))+ require.Equal(t, http.StatusTooManyRequests, doReq("unknown-b"))+ require.Equal(t, http.StatusTooManyRequests, doReq(""))+}diff --git a/backend/internal/registry/handler.go b/backend/internal/registry/handler.goindex 6f1d9f60f..5b4f5eb32 100644--- a/backend/internal/registry/handler.go+++ b/backend/internal/registry/handler.go@@ -18,7 +18,7 @@ import (// ContainerRegistryHandler handles container registry management endpoints.type ContainerRegistryHandler struct {registryService *ContainerRegistryService- syncRemoteRegistries func(context.Context) error+ syncRemoteRegistries func(context.Context)}type ListContainerRegistriesInput struct {@@ -55,7 +55,7 @@ type SyncContainerRegistriesInput struct {}// NewHandler builds the container registry HTTP handler.-func NewHandler(registryService *ContainerRegistryService, syncRemoteRegistries func(context.Context) error) *ContainerRegistryHandler {+func NewHandler(registryService *ContainerRegistryService, syncRemoteRegistries func(context.Context)) *ContainerRegistryHandler {return &ContainerRegistryHandler{registryService: registryService, syncRemoteRegistries: syncRemoteRegistries}}@@ -207,9 +207,6 @@ func (h *ContainerRegistryHandler) triggerRemoteRegistrySync(ctx context.Contextreturn}- go func(syncCtx context.Context, syncReason string) {- if err := h.syncRemoteRegistries(syncCtx); err != nil {- slog.WarnContext(syncCtx, "Failed to fan out registry sync to remote environments", "reason", syncReason, "error", err.Error())- }- }(context.WithoutCancel(ctx), reason)+ slog.DebugContext(ctx, "Queueing registry sync to remote environments", "reason", reason)+ h.syncRemoteRegistries(ctx)}diff --git a/backend/internal/registry/module.go b/backend/internal/registry/module.goindex 5f0567b4c..ab9ddde95 100644--- a/backend/internal/registry/module.go+++ b/backend/internal/registry/module.go@@ -17,7 +17,7 @@ type Module struct {browseHandler *browse.Handler}-func New(service *ContainerRegistryService, syncRemoteRegistries func(context.Context) error) *Module {+func New(service *ContainerRegistryService, syncRemoteRegistries func(context.Context)) *Module {return &Module{service: service,handler: NewHandler(service, syncRemoteRegistries),diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.goindex 1bde751ba..efcbb9b48 100644--- a/backend/internal/s3/handler.go+++ b/backend/internal/s3/handler.go@@ -13,15 +13,14 @@ import ()const (- s3DestinationPath = "/backups/s3"- s3DestinationTag = "S3 Destinations"- s3ConnectionTestSuccess = "S3 connection test succeeded"- s3RemoteSyncFailureMessage = "Failed to fan out S3 destination sync to remote environments"+ s3DestinationPath = "/backups/s3"+ s3DestinationTag = "S3 Destinations"+ s3ConnectionTestSuccess = "S3 connection test succeeded")type s3DestinationHandler struct {service *S3DestinationService- syncRemoteDestinations func(context.Context) error+ syncRemoteDestinations func(context.Context)}type listS3DestinationsInput struct {@@ -168,11 +167,8 @@ func (h *s3DestinationHandler) test(ctx context.Context, input *testS3Destinatio}func (h *s3DestinationHandler) triggerRemoteSync(ctx context.Context, reason string) {- go func(syncCtx context.Context, syncReason string) {- if err := h.syncRemoteDestinations(syncCtx); err != nil {- slog.WarnContext(syncCtx, s3RemoteSyncFailureMessage, "reason", syncReason, "error", err.Error())- }- }(context.WithoutCancel(ctx), reason)+ slog.DebugContext(ctx, "Queueing S3 destination sync to remote environments", "reason", reason)+ h.syncRemoteDestinations(ctx)}func (h *s3DestinationHandler) testConfiguration(ctx context.Context, input *testS3DestinationConfigurationInput) (*handlerutil.Out[base.MessageResponse], error) {diff --git a/backend/internal/s3/module.go b/backend/internal/s3/module.goindex 827344e55..0b5544863 100644--- a/backend/internal/s3/module.go+++ b/backend/internal/s3/module.go@@ -13,10 +13,10 @@ import (type Module struct {service *S3DestinationService- syncRemoteDestinations func(context.Context) error+ syncRemoteDestinations func(context.Context)}-func New(service *S3DestinationService, syncRemoteDestinations func(context.Context) error) *Module {+func New(service *S3DestinationService, syncRemoteDestinations func(context.Context)) *Module {return &Module{service: service, syncRemoteDestinations: syncRemoteDestinations}}@@ -35,7 +35,7 @@ func (m *Module) RegisterRoutes(api huma.API) {RegisterS3Destinations(api, m.service, m.syncRemoteDestinations)}-func RegisterS3Destinations(api huma.API, service *S3DestinationService, syncRemoteDestinations func(context.Context) error) {+func RegisterS3Destinations(api huma.API, service *S3DestinationService, syncRemoteDestinations func(context.Context)) {handler := &s3DestinationHandler{service: service, syncRemoteDestinations: syncRemoteDestinations}handlerutil.RegisterSecured(api,diff --git a/backend/internal/variable/service.go b/backend/internal/variable/service.goindex c78d06364..fe5b7a74f 100644--- a/backend/internal/variable/service.go+++ b/backend/internal/variable/service.go@@ -30,6 +30,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")@@ -50,11 +51,12 @@ const (// .env.global file: locally via template.TemplateService, remotely via the existing// agent variables endpoint.type VariableService struct {- db *database.DB- proxyEnvironmentJSON func(context.Context, string, string, string, []byte, any) error- listEnabledEnvironmentIDs func(context.Context) ([]string, error)- settingsService *settings.SettingsService- kvService *kv.KVService+ db *database.DB+ proxyEnvironmentJSON func(context.Context, string, string, string, []byte, any) error+ listEnabledRemoteEnvironments func(context.Context) ([]environment.Environment, error)+ settingsService *settings.SettingsService+ kvService *kv.KVService+ importLocks utils.KeyedMutexstatusMu sync.RWMutexsyncStatus map[string]env.EnvironmentSyncStatus@@ -73,7 +75,7 @@ func NewVariableService(db *database.DB, environmentService *environment.Environ}if environmentService != nil {service.proxyEnvironmentJSON = environmentService.ProxyJSONRequest- service.listEnabledEnvironmentIDs = environmentService.ListEnabledEnvironmentIDs+ service.listEnabledRemoteEnvironments = environmentService.ListEnabledRemoteEnvironments}return service}@@ -319,6 +321,9 @@ func (s *VariableService) SyncEnvironment(ctx context.Context, envID string) err// ForgetSyncState makes the next SyncEnvironment resend regardless of change.func (s *VariableService) ForgetSyncState(envID string) {s.syncGate.Forget(envID)+ s.statusMu.Lock()+ delete(s.syncStatus, envID)+ s.statusMu.Unlock()}func (s *VariableService) syncEnvironmentInternal(ctx context.Context, envID string) error {@@ -359,16 +364,15 @@ func (s *VariableService) syncEnvironmentInternal(ctx context.Context, envID str// syncTargetsInternal returns the local environment plus every enabled// environment, deduplicated.-func (s *VariableService) syncTargetsInternal(ctx context.Context) []string {- envIDs, err := s.listEnabledEnvironmentIDs(ctx)+func (s *VariableService) syncTargetsInternal(ctx context.Context) []environment.Environment {+ remotes, err := s.listEnabledRemoteEnvironments(ctx)if err != nil {slog.ErrorContext(ctx, "Failed to list environments for global variable sync", "error", err)- envIDs = nil- }- if !slices.Contains(envIDs, environment.LocalEnvironmentID) {- envIDs = append([]string{environment.LocalEnvironmentID}, envIDs...)+ remotes = nil}- return envIDs+ return append([]environment.Environment{{ID: environment.LocalEnvironmentID}}, slices.DeleteFunc(remotes, func(remote environment.Environment) bool {+ return strings.TrimSpace(kit.FromPtr(remote.AccessToken)) == ""+ })...)}// SyncAll pushes the effective variable set to the local environment and every@@ -378,8 +382,8 @@ func (s *VariableService) SyncAll(ctx context.Context) []env.EnvironmentSyncStattargets := s.syncTargetsInternal(ctx)// An explicit sync exists to repair agents, so every target is resent even// when the manager's payload has not changed.- for _, envID := range targets {- s.ForgetSyncState(envID)+ for _, target := range targets {+ s.ForgetSyncState(target.ID)}s.syncEnvironmentsInternal(ctx, targets)return s.SyncStatuses()@@ -394,25 +398,33 @@ func (s *VariableService) SyncAllBackground(ctx context.Context) []env.Environmeslog.WarnContext(ctx, "Failed to sync global variables locally", "error", err)}- remoteIDs := slices.DeleteFunc(s.syncTargetsInternal(ctx), func(id string) bool { return id == environment.LocalEnvironmentID })- for _, envID := range remoteIDs {- s.recordSyncPendingInternal(envID)+ remotes := slices.DeleteFunc(s.syncTargetsInternal(ctx), func(target environment.Environment) bool {+ return target.ID == environment.LocalEnvironmentID+ })+ for _, target := range remotes {+ s.recordSyncPendingInternal(target.ID)}bgCtx := context.WithoutCancel(ctx)- go s.syncEnvironmentsInternal(bgCtx, remoteIDs)+ go s.syncEnvironmentsInternal(bgCtx, remotes)return s.SyncStatuses()}-func (s *VariableService) syncEnvironmentsInternal(ctx context.Context, envIDs []string) {+func (s *VariableService) syncEnvironmentsInternal(ctx context.Context, targets []environment.Environment) {var wg sync.WaitGroup- for _, envID := range envIDs {+ for _, target := range targets {wg.Go(func() {syncCtx, cancel := context.WithTimeout(ctx, variableSyncTimeout)defer cancel()- if err := s.SyncEnvironment(syncCtx, envID); err != nil {- slog.WarnContext(syncCtx, "Failed to sync global variables to environment", "environmentId", envID, "error", err)+ if target.IsEdge && !edge.HasActiveTunnel(target.ID) {+ if _, ok := edge.RequestTunnelAndWait(syncCtx, target.ID, edge.DefaultTunnelDemandTTL, edge.DefaultTunnelAcquireTimeout()).Get(); !ok {+ s.recordSyncPendingInternal(target.ID)+ return+ }+ }+ if err := s.SyncEnvironment(syncCtx, target.ID); err != nil {+ slog.WarnContext(syncCtx, "Failed to sync global variables to environment", "environmentId", target.ID, "error", err)}})}@@ -611,6 +623,12 @@ func (s *VariableService) ImportLegacyLocalEnvFile(ctx context.Context) error {// would not already produce for that environment as an env-scoped variable.// An import failure aborts the push so existing agent data is never wiped.func (s *VariableService) importRemoteLegacyVarsOnceInternal(ctx context.Context, envID string) error {+ unlock, err := s.importLocks.LockContext(ctx, envID)+ if err != nil {+ return err+ }+ defer unlock()+flag := globalVariablesImportedKVPrefix + envIDimported, err := s.kvService.GetBool(ctx, flag, false)if err != nil {diff --git a/backend/internal/variable/service_test.go b/backend/internal/variable/service_test.goindex 30c4a9fae..918c52201 100644--- a/backend/internal/variable/service_test.go+++ b/backend/internal/variable/service_test.go@@ -301,7 +301,6 @@ func TestSyncEnvironment_DirectMaterializesSecretsThroughAgentRoute(t *testing.T}).Error)environmentService := environment.NewEnvironmentService(db, nil, nil, nil, service.settingsService, nil)service.proxyEnvironmentJSON = environmentService.ProxyJSONRequest- service.listEnabledEnvironmentIDs = environmentService.ListEnabledEnvironmentIDscreateMaterializedSecretVariableInternal(t, service)require.NoError(t, service.SyncEnvironment(t.Context(), "env-direct"))diff --git a/backend/internal/webhook/service_test.go b/backend/internal/webhook/service_test.goindex 94dd7fba9..7c323689c 100644--- a/backend/internal/webhook/service_test.go+++ b/backend/internal/webhook/service_test.go@@ -179,7 +179,7 @@ func TestKnownTokenRateLimit_ExhaustedIPDoesNotBlockFirstUse(t *testing.T) {require.NoError(t, sqlDB.Close())router := echo.New()router.IPExtractor = echo.ExtractIPDirect()- router.Use(middleware.PerTokenRateLimitForPaths([]string{"/trigger/:token"}, 1, 1, svc.IsKnownToken))+ router.Use(middleware.PerTokenRateLimitForPaths([]string{"/trigger/:token"}, 1, 1, func(c *echo.Context) string { return c.Param("token") }, svc.IsKnownToken))router.POST("/trigger/:token", func(c *echo.Context) error { return c.NoContent(http.StatusAccepted) })request := func(token string) int {req := httptest.NewRequest(http.MethodPost, "/trigger/"+token, http.NoBody)diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.goindex b68a71cca..0638a111a 100644--- a/backend/pkg/libarcane/edge/client.go+++ b/backend/pkg/libarcane/edge/client.go@@ -10,6 +10,7 @@ import ("log/slog""net""net/http"+ "net/url""slices""strings""sync"@@ -124,6 +125,13 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)if errCh != nil {defer close(errCh)}+ if configured := strings.ToLower(strings.TrimSpace(c.cfg.EdgeTransport)); configured != "" && NormalizeEdgeTransport(configured) != configured {+ slog.WarnContext(ctx, "Unknown EDGE_TRANSPORT value; using the default transport", "configured", c.cfg.EdgeTransport, "transport", NormalizeEdgeTransport(configured))+ }+ if parsed, err := url.Parse(httpx.ManagerBaseURL(c.cfg.ManagerApiUrl)); err == nil && strings.Trim(parsed.Path, "/") != "" && c.managerGRPCAddr != "" {+ slog.WarnContext(ctx, "MANAGER_API_URL has a path prefix that gRPC cannot route; gRPC edge tunnel transport is unavailable", "pathPrefix", parsed.Path)+ c.managerGRPCAddr = ""+ }// Log only the endpoints this agent will dial.poll := UsePollEdgeTransport(c.cfg)attrs := []any{@@ -147,7 +155,15 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)reconnectBackoff := backoff.NewExponentialBackOff()reconnectBackoff.InitialInterval = c.reconnectIntervalreconnectBackoff.MaxInterval = maxReconnectInterval- serve := kit.Ternary(UsePollEdgeTransport(c.cfg), c.connectAndServePoll, c.connectAndServeManagedTunnel)+ serve := func(ctx context.Context) error {+ if err := EnsureAgentMTLSAssets(ctx, c.cfg); err != nil {+ return err+ }+ if err := ValidateAgentMTLSConfig(c.cfg); err != nil {+ return err+ }+ return kit.Ternary(poll, c.connectAndServePoll, c.connectAndServeManagedTunnel)(ctx)+ }for {if ctx.Err() != nil {@@ -448,7 +464,7 @@ func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, wif transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {c.beginCommandRequestTransfer(ctx, conn, transferID, msg)} else {- workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })+ c.startCommandRequest(ctx, conn, msg, workers)}case MessageTypeFileChunk:c.handleCommandRequestChunk(ctx, msg, workers)@@ -464,7 +480,12 @@ func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, w}}case MessageTypeCancelRequest:- slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)+ if value, ok := c.commandCancels.LoadAndDelete(msg.ID); ok {+ if cancel, isCancel := value.(context.CancelFunc); isCancel {+ cancel()+ slog.DebugContext(ctx, "Cancelled edge command on manager request", "id", msg.ID)+ }+ }case MessageTypeCommandCredit:if value, ok := c.commandRecorders.Load(msg.ID); ok {if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {@@ -544,6 +565,13 @@ func (c *TunnelClient) handleCommandRequestChunk(ctx context.Context, msg *Tunnereturn}+ if transfer.body.Len()+len(msg.Body) > maxLegacyCommandBodyBytes {+ if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {+ transfer.stop()+ c.sendCommandComplete(transfer.conn, transfer.request.ID, http.StatusRequestEntityTooLarge, "command body is too large for a legacy edge tunnel; upgrade the manager and agent")+ }+ return+ }_, _ = transfer.body.Write(msg.Body)transfer.nextSequence++if !msg.EOF {@@ -552,11 +580,21 @@ func (c *TunnelClient) handleCommandRequestChunk(ctx context.Context, msg *Tunneif c.requestTransfers.CompareAndDelete(msg.ID, transfer) {transfer.stop()- transfer.request.Body = append([]byte(nil), transfer.body.Bytes()...)- workers.Go(func() { c.handleCommandRequest(ctx, transfer.conn, transfer.request) })+ transfer.request.Body = transfer.body.Bytes()+ c.startCommandRequest(ctx, transfer.conn, transfer.request, workers)}}+func (c *TunnelClient) startCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {+ commandCtx, cancel := context.WithCancel(ctx)+ c.commandCancels.Store(msg.ID, cancel)+ workers.Go(func() {+ defer c.commandCancels.Delete(msg.ID)+ defer cancel()+ c.handleCommandRequest(commandCtx, conn, msg)+ })+}+// Deprecated: legacy edge message protocol; remove in v3.func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {if !ValidateEdgeCommand(msg.Command, msg.Method, msg.Path, false) {@@ -571,7 +609,7 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnreturn}- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)+ reqCtx, cancel := context.WithTimeout(ctx, kit.Ternary(msg.TimeoutMillis > 0, min(time.Duration(msg.TimeoutMillis)*time.Millisecond, c.requestTimeout), c.requestTimeout))defer cancel()req, err := c.buildLocalHTTPRequest(reqCtx, msg)@@ -1094,12 +1132,11 @@ func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (if cfg.AgentToken == "" {return nil, errors.New("AGENT_TOKEN is required")}-- if err := EnsureAgentMTLSAssets(ctx, cfg); err != nil {- return nil, err+ if parsed, err := url.Parse(httpx.ManagerBaseURL(cfg.ManagerApiUrl)); NormalizeEdgeTransport(cfg.EdgeTransport) == EdgeTransportGRPC && err == nil && strings.Trim(parsed.Path, "/") != "" {+ return nil, errors.New("gRPC transport cannot use a MANAGER_API_URL with a path prefix; use EDGE_TRANSPORT=websocket or auto")}- if err := ValidateAgentMTLSConfig(cfg); err != nil {- return nil, err+ if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeDisabled && !managerUsesTLS(cfg) {+ return nil, errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https")}client := NewTunnelClient(cfg, handler)@@ -1136,8 +1173,8 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {if err != nil {if resp != nil && resp.Body != nil {defer func() { _ = resp.Body.Close() }()- body, _ := io.ReadAll(resp.Body)- return fmt.Errorf("failed to connect to manager websocket endpoint (status: %d, body: %s): %w", resp.StatusCode, string(body), err)+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))+ return fmt.Errorf("failed to connect to manager websocket endpoint (status: %d, body: %s): %w", resp.StatusCode, strings.TrimSpace(string(body)), err)}return fmt.Errorf("failed to connect to manager websocket endpoint: %w", err)}diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.goindex 9273fa0ec..fa18a6b75 100644--- a/backend/pkg/libarcane/edge/client_test.go+++ b/backend/pkg/libarcane/edge/client_test.go@@ -942,18 +942,36 @@ func TestStartTunnelClient_GRPCValidation(t *testing.T) {require.Error(t, err)assert.Contains(t, err.Error(), "MANAGER_API_URL to use https")})+}- t.Run("required mtls auto-enrollment failure surfaces", func(t *testing.T) {- _, err := StartTunnelClient(ctx, &Config{- EdgeAgent: true,- EdgeTransport: EdgeTransportGRPC,- ManagerApiUrl: "https://127.0.0.1:1/api",- AgentToken: "token",- EdgeMTLSMode: EdgeMTLSModeRequired,- }, http.NotFoundHandler())- require.Error(t, err)- assert.Contains(t, err.Error(), "edge mTLS enrollment request failed")- })+func TestTunnelClient_StartWithErrorChan_RetriesMTLSEnrollmentEachAttempt(t *testing.T) {+ client := NewTunnelClient(&Config{+ EdgeAgent: true,+ EdgeTransport: EdgeTransportGRPC,+ ManagerApiUrl: "https://127.0.0.1:1/api",+ AgentToken: "token",+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSAssetsDir: t.TempDir(),+ }, http.NotFoundHandler())+ client.reconnectInterval = 10 * time.Millisecond++ ctx, cancel := context.WithCancel(t.Context())+ errCh := make(chan error, 1)+ go client.StartWithErrorChan(ctx, errCh)++ for range 2 {+ select {+ case err := <-errCh:+ require.ErrorContains(t, err, "edge mTLS enrollment request failed")+ case <-time.After(5 * time.Second):+ t.Fatal("timed out waiting for an mTLS enrollment attempt")+ }+ }+ cancel()+ require.Eventually(t, func() bool {+ _, open := <-errCh+ return !open+ }, 5*time.Second, 10*time.Millisecond)}func TestTunnelClient_connectAndServeGRPC_EmptyManagerAddress(t *testing.T) {diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.goindex 4fe269774..7e11f91ba 100644--- a/backend/pkg/libarcane/edge/client_transport_poll.go+++ b/backend/pkg/libarcane/edge/client_transport_poll.go@@ -213,8 +213,8 @@ func (c *TunnelClient) pollTunnelControl(ctx context.Context, httpClient *http.Cdefer func() { _ = resp.Body.Close() }()if resp.StatusCode != http.StatusOK {- respBody, _ := io.ReadAll(resp.Body)- return nil, fmt.Errorf("poll request failed with status %d: %s", resp.StatusCode, string(respBody))+ respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 512))+ return nil, fmt.Errorf("poll request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))}var pollResp TunnelPollResponsediff --git a/backend/pkg/libarcane/edge/command_client.go b/backend/pkg/libarcane/edge/command_client.goindex 4bbec4e66..41a570a81 100644--- a/backend/pkg/libarcane/edge/command_client.go+++ b/backend/pkg/libarcane/edge/command_client.go@@ -161,6 +161,11 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C// Release an agent still waiting to send output that nobody will read.credit(math.MaxInt32)}+ if ctx.Err() != nil && slices.Contains(tunnel.Capabilities, tunnelCapabilityProtoParity) {+ if cancelErr := tunnel.Conn.Send(&TunnelMessage{ID: requestID, Type: MessageTypeCancelRequest}); cancelErr != nil {+ slog.DebugContext(context.WithoutCancel(ctx), "Failed to send edge command cancel", "id", requestID, "error", cancelErr)+ }+ }return nil, err}diff --git a/backend/pkg/libarcane/edge/commands.go b/backend/pkg/libarcane/edge/commands.goindex 6f5462b78..55fcfe575 100644--- a/backend/pkg/libarcane/edge/commands.go+++ b/backend/pkg/libarcane/edge/commands.go@@ -284,23 +284,28 @@ var commandRoutesIndex = buildCommandRouteIndex(commandRoutes)// ResolveEdgeCommandName returns the edge command for a request, or None when the allowlist does not cover it.func ResolveEdgeCommandName(method, requestPath string, stream bool) mo.Option[string] {requestPath, _, _ = strings.Cut(strings.TrimSpace(requestPath), "?")- current := commandRoutesIndex[commandRouteKey{Method: kit.Ternary(stream, "", strings.ToUpper(strings.TrimSpace(method))), Stream: stream}]- for part := range strings.SplitSeq(strings.Trim(requestPath, "/"), "/") {- if current == nil {- break- }- if next := current.static[part]; next != nil {- current = next- } else if part != "" {- current = current.param- } else {- current = nil- }- }- if current == nil || current.route == nil {+ root := commandRoutesIndex[commandRouteKey{Method: kit.Ternary(stream, "", strings.ToUpper(strings.TrimSpace(method))), Stream: stream}]+ route := root.match(strings.Split(strings.Trim(requestPath, "/"), "/"))+ if route == nil {return mo.None[string]()}- return mo.Some(current.route.CommandName)+ return mo.Some(route.CommandName)+}++func (n *commandRouteNode) match(parts []string) *commandRoute {+ if n == nil {+ return nil+ }+ if len(parts) == 0 {+ return n.route+ }+ if route := n.static[parts[0]].match(parts[1:]); route != nil {+ return route+ }+ if parts[0] == "" {+ return nil+ }+ return n.param.match(parts[1:])}// ValidateEdgeCommand reports whether commandName is the allowlisted command for the request.diff --git a/backend/pkg/libarcane/edge/commands_test.go b/backend/pkg/libarcane/edge/commands_test.goindex 91968c3c8..4c5c580a3 100644--- a/backend/pkg/libarcane/edge/commands_test.go+++ b/backend/pkg/libarcane/edge/commands_test.go@@ -41,6 +41,11 @@ func TestResolveEdgeCommandName(t *testing.T) {shouldHit: true,},{name: "volume workspace list", method: "GET", path: "/api/environments/0/volumes/data/workspace", command: "volume.workspace.list", shouldHit: true},+ {name: "volume named backups inspect", method: "GET", path: "/api/environments/0/volumes/backups", command: "volume.inspect", shouldHit: true},+ {name: "volume named backups workspace", method: "GET", path: "/api/environments/0/volumes/backups/workspace", command: "volume.workspace.list", shouldHit: true},+ {name: "volume named counts usage", method: "GET", path: "/api/environments/0/volumes/counts/usage", command: "volume.usage", shouldHit: true},+ {name: "volume named sizes backups", method: "GET", path: "/api/environments/0/volumes/sizes/backups", command: "volume.backup.list", shouldHit: true},+ {name: "volume backup download keeps static branch", method: "GET", path: "/api/environments/0/volumes/backups/b1/download", command: "volume.backup.download", shouldHit: true},{name: "volume workspace read file",method: "GET",diff --git a/backend/pkg/libarcane/edge/event_sync.go b/backend/pkg/libarcane/edge/event_sync.goindex 02c6b27de..5d9156dca 100644--- a/backend/pkg/libarcane/edge/event_sync.go+++ b/backend/pkg/libarcane/edge/event_sync.go@@ -12,6 +12,8 @@ import ("sync""time""uuid"++ "go.getarcane.app/kit/pkg")const (@@ -69,7 +71,10 @@ func PublishEventToManager(ctx context.Context, event *TunnelEvent) error {if sink.conn.IsClosed() {return ErrNoActiveAgentTunnel}- return sink.conn.Send(&TunnelMessage{ID: uuid.New().String(), Type: MessageTypeEvent, Event: event})+ if err := sink.conn.Send(&TunnelMessage{ID: uuid.New().String(), Type: MessageTypeEvent, Event: event}); err != nil {+ return kit.Ternary(errors.Is(err, ErrTunnelConnectionClosed), fmt.Errorf("%w: %w", ErrNoActiveAgentTunnel, err), err)+ }+ return nil}body, err := json.Marshal(event)@@ -83,7 +88,7 @@ func PublishEventToManager(ctx context.Context, event *TunnelEvent) error {req.Header.Set("Content-Type", "application/json")resp, err := sink.client.Do(req)if err != nil {- return fmt.Errorf("send edge event: %w", err)+ return kit.Ternary(errors.Is(err, ErrTunnelConnectionClosed), fmt.Errorf("send edge event: %w: %w", ErrNoActiveAgentTunnel, err), fmt.Errorf("send edge event: %w", err))}defer func() { _ = resp.Body.Close() }()if resp.StatusCode != http.StatusNoContent {diff --git a/backend/pkg/libarcane/edge/mux.go b/backend/pkg/libarcane/edge/mux.goindex 8a2d53ca4..fea17ac08 100644--- a/backend/pkg/libarcane/edge/mux.go+++ b/backend/pkg/libarcane/edge/mux.go@@ -37,6 +37,8 @@ const (muxWriteTimeout = 30 * time.Second// muxDialTimeout bounds a stream open on its own, because http.Transport detaches dials from request cancellation.muxDialTimeout = 30 * time.Second+ // ProxyResponseHeaderTimeout is generous because proxied operations such as deploys can run for minutes before responding.+ ProxyResponseHeaderTimeout = 30 * time.Minute)// errTunnelMuxActive rejects legacy messages once a connection carries a multiplexed session.@@ -76,6 +78,7 @@ func newMuxHTTPClient(dial func(ctx context.Context) (net.Conn, error)) *http.ClIdleConnTimeout: 30 * time.Second,MaxIdleConnsPerHost: 32,ExpectContinueTimeout: time.Second,+ ResponseHeaderTimeout: ProxyResponseHeaderTimeout,ReadBufferSize: 64 << 10,WriteBufferSize: 64 << 10,})}diff --git a/backend/pkg/libarcane/edge/mux_manager.go b/backend/pkg/libarcane/edge/mux_manager.goindex 94289ad11..e63560300 100644--- a/backend/pkg/libarcane/edge/mux_manager.go+++ b/backend/pkg/libarcane/edge/mux_manager.go@@ -15,8 +15,11 @@ import ("github.com/samber/mo")-// maxMuxEventBytes bounds one agent event posted over a multiplexed session.-const maxMuxEventBytes = 1 << 20+const (+ // maxMuxEventBytes bounds one agent event posted over a multiplexed session.+ maxMuxEventBytes = 1 << 20+ muxRegistrationPingTimeout = 3 * time.Second+)// dial opens a stream for the session client. Callers that found the tunnel during the register handshake// wait for the session to start.@@ -47,10 +50,31 @@ func (m *muxSession) close() {if session != nil {_ = session.Close()}- m.client.CloseIdleConnections()})}+func (m *muxSession) alive(ctx context.Context, timeout time.Duration) bool {+ m.mu.Lock()+ session := m.session+ m.mu.Unlock()+ if session == nil {+ return true+ }+ pinged := make(chan error, 1)+ go func() {+ _, err := session.Ping()+ pinged <- err+ }()+ select {+ case err := <-pinged:+ return err == nil+ case <-time.After(timeout):+ return false+ case <-ctx.Done():+ return true+ }+}+// HTTPClient returns the client for a tunnel that negotiated the multiplexed data plane. Requests use// MuxAgentBaseURL as their origin and are served by the agent's own router.func (t *AgentTunnel) HTTPClient() mo.Option[*http.Client] {diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.goindex 428602d64..8ee9a78c0 100644--- a/backend/pkg/libarcane/edge/poll_control.go+++ b/backend/pkg/libarcane/edge/poll_control.go@@ -1,9 +1,6 @@package edgeimport (- "encoding/json/v2"- "errors"- "io""log/slog""net/http""strings"@@ -180,15 +177,6 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {req := c.Request()ctx := req.Context()- if req.Body != nil {- var pollReq TunnelPollRequest- err := json.UnmarshalRead(req.Body, &pollReq)- _ = req.Body.Close()- if err != nil && !errors.Is(err, http.ErrBodyReadAfterClose) && !errors.Is(err, io.EOF) {- return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid poll payload"})- }- }-// Proxy-terminated mTLS consumes the client certificate, so the token remains the environment claim.token, source := agentToken(req.Header.Values)if token == "" {diff --git a/backend/pkg/libarcane/edge/proxy.go b/backend/pkg/libarcane/edge/proxy.goindex db2465e54..a2a64e47d 100644--- a/backend/pkg/libarcane/edge/proxy.go+++ b/backend/pkg/libarcane/edge/proxy.go@@ -20,7 +20,8 @@ const (DefaultProxyTimeout = 5 * time.MinuteDefaultTunnelAcquirePollEvery = 100 * time.Millisecond// Deprecated: legacy edge message protocol; remove in v3.- tunnelStreamHeader = "X-Arcane-Tunnel-Stream"+ tunnelStreamHeader = "X-Arcane-Tunnel-Stream"+ maxLegacyCommandBodyBytes = 256 << 20)// browserSecurityHeaders must not cross a legacy tunnel: the agent's CORS check would reject the browser Origin,@@ -153,7 +154,7 @@ func (s *grpcResponseState) handleTunnelMessage(method string, incoming *TunnelMs.writeBody(incoming.Body)if incoming.Error != "" && incoming.Status >= http.StatusBadRequest {done, status, headers, body = true, incoming.Status, s.cleanHeaders(), s.respBody.Bytes()- err = errors.New(incoming.Error)+ err = &commandStatusError{status: incoming.Status, message: incoming.Error}} else {done = true_, headers, body = s.finish()@@ -268,7 +269,10 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) evar body []byteif req.Body != nil {var err error- body, err = io.ReadAll(req.Body)+ body, err = io.ReadAll(http.MaxBytesReader(c.Response(), req.Body, maxLegacyCommandBodyBytes))+ if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {+ return c.JSON(http.StatusRequestEntityTooLarge, map[string]any{"error": "request body is too large for a legacy edge tunnel; upgrade the agent"})+ }if err != nil {slog.ErrorContext(ctx, "Failed to read request body for tunnel proxy", "error", err)return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to read request body"})@@ -301,6 +305,9 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) eif resp, unwrapErr := echo.UnwrapResponse(c.Response()); unwrapErr == nil && resp.Committed {panic(http.ErrAbortHandler)}+ if statusErr, ok := errors.AsType[*commandStatusError](err); ok {+ return c.JSON(statusErr.status, map[string]any{"error": statusErr.message})+ }if proxyCtx.Err() != nil {return c.JSON(http.StatusGatewayTimeout, map[string]any{"error": "request timed out"})}@@ -311,6 +318,10 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) ereturn nil}+func (e *commandStatusError) Error() string {+ return e.message+}+// cleanHeaders returns the response headers without the internal stream marker.func (s *grpcResponseState) cleanHeaders() map[string]string {headers := maps.Clone(s.respHeaders)diff --git a/backend/pkg/libarcane/edge/registry.go b/backend/pkg/libarcane/edge/registry.goindex 61a5f5b38..35f017f70 100644--- a/backend/pkg/libarcane/edge/registry.go+++ b/backend/pkg/libarcane/edge/registry.go@@ -204,6 +204,10 @@ func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunneif envID == "" {return false, false, "environment ID is required", nil}+ if existing, ok := r.Get(envID).Get(); ok && existing != tunnel && existing.mux != nil && existing.connected() && !existing.mux.alive(ctx, muxRegistrationPingTimeout) {+ slog.InfoContext(ctx, "Existing edge tunnel session failed a liveness ping; replacing it", "environmentId", envID)+ _ = existing.CloseWithReason("edge tunnel liveness ping failed")+ }result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResult, bool, error) {var result registerSessionResultdiff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.goindex a194ba057..8c81eb478 100644--- a/backend/pkg/libarcane/edge/tls.go+++ b/backend/pkg/libarcane/edge/tls.go@@ -62,6 +62,7 @@ const (var (generatedAssetNameSanitizer = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)managerCALocks utils.KeyedMutex+ errEnrollUnreachable = errors.New("edge mTLS enrollment request failed")// caKeyEncrypt is a variable so tests can force encryption failures.caKeyEncrypt = libcrypto.Encrypt)@@ -185,7 +186,7 @@ func GenerateManagerClientMTLSAssetsWithContext(ctx context.Context, cfg *ConfiguriSAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)// The URI SAN is the stable identity, so renaming an environment keeps its certificate.- existing, reason := loadClientCertificate(certPath, keyPath, time.Now())+ existing, reason := loadClientCertificate(certPath, keyPath, time.Now(), agentMTLSRenewBefore)certIssued := reason != "" || !supportedGeneratedKey(existing.Leaf.PublicKey) ||existing.Leaf.CheckSignatureFrom(ca.Leaf) != nil || (uriSAN != nil && !certificateHasURISAN(existing.Leaf, uriSAN))if certIssued {@@ -282,8 +283,7 @@ func GeneratedManagerClientMTLSCertPath(cfg *Config, envID string) (string, erro// EnsureAgentMTLSAssets downloads manager-generated client certificates when edge mTLS is// enabled without configured client cert/key files, and renews them before they expire.func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {- if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled ||- (strings.TrimSpace(cfg.EdgeMTLSCertFile) != "" && strings.TrimSpace(cfg.EdgeMTLSKeyFile) != "") {+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {return nil}@@ -293,18 +293,27 @@ func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {}certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)+ configuredCert, configuredKey := strings.TrimSpace(cfg.EdgeMTLSCertFile), strings.TrimSpace(cfg.EdgeMTLSKeyFile)+ if configuredCert != "" && configuredKey != "" && (filepath.Clean(configuredCert) != certPath || filepath.Clean(configuredKey) != keyPath) {+ return nil+ }- _, reason := loadClientCertificate(certPath, keyPath, time.Now())+ _, reason := loadClientCertificate(certPath, keyPath, time.Now(), agentMTLSRenewBefore)if reason != "" {if fileExists(certPath) {slog.WarnContext(ctx, "Existing edge mTLS assets need renewal; enrolling new assets", "reason", reason, "certPath", certPath)}if enrollErr := enrollAgentMTLSAssets(ctx, cfg, assetsDir, certPath, keyPath); enrollErr != nil {- if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired {+ _, currentReason := loadClientCertificate(certPath, keyPath, time.Now(), 0)+ switch {+ case currentReason == "" && errors.Is(enrollErr, errEnrollUnreachable):+ slog.WarnContext(ctx, "Edge mTLS renewal failed; keeping the current client certificate until it expires", "error", enrollErr, "certPath", certPath)+ case NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired:slog.WarnContext(ctx, "Edge mTLS enrollment failed; proceeding without client certificate", "error", enrollErr)return nil+ default:+ return enrollErr}- return enrollErr}}@@ -314,8 +323,10 @@ func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr)}}- cfg.EdgeMTLSCertFile = certPath- cfg.EdgeMTLSKeyFile = keyPath+ if configuredCert != certPath || configuredKey != keyPath {+ cfg.EdgeMTLSCertFile = certPath+ cfg.EdgeMTLSKeyFile = keyPath+ }if caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName); fileExists(caPath) && strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {cfg.EdgeMTLSCAFile = caPath}@@ -333,7 +344,12 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPathreturn errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https for certificate enrollment")}- httpClient, clientErr := NewManagerHTTPClient(cfg, 30*time.Second)+ httpClient, clientErr := NewManagerHTTPClient(&Config{+ EdgeMTLSMode: cfg.EdgeMTLSMode,+ EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,+ EdgeMTLSServerName: cfg.EdgeMTLSServerName,+ ManagerApiUrl: cfg.ManagerApiUrl,+ }, 30*time.Second)if clientErr != nil {return fmt.Errorf("failed to configure edge mTLS enrollment client: %w", clientErr)}@@ -347,7 +363,12 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPathresp, doErr := httpClient.Do(req)if doErr != nil {- return fmt.Errorf("edge mTLS enrollment request failed: %w", doErr)+ _, certRejected := errors.AsType[*tls.CertificateVerificationError](doErr)+ _, alert := errors.AsType[tls.AlertError](doErr)+ if certRejected || alert {+ return fmt.Errorf("edge mTLS enrollment handshake failed: %w", doErr)+ }+ return fmt.Errorf("%w: %w", errEnrollUnreachable, doErr)}defer func() { _ = resp.Body.Close() }()if resp.StatusCode != http.StatusOK {@@ -373,7 +394,7 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPathreturn fmt.Errorf("failed to write edge mTLS asset %s: %w", file.Name, writeErr)}}- if _, writtenReason := loadClientCertificate(certPath, keyPath, time.Now()); writtenReason != "" {+ if _, writtenReason := loadClientCertificate(certPath, keyPath, time.Now(), 0); writtenReason != "" {return fmt.Errorf("edge mTLS enrollment wrote unusable assets: %s", writtenReason)}return nil@@ -404,7 +425,7 @@ func buildManagerClientTLSConfig(cfg *Config) (*tls.Config, error) {if certPath == "" || keyPath == "" {return tlsConfig, nil}- cert, reason := loadClientCertificate(certPath, keyPath, time.Now())+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now(), 0)if reason != "" {if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeOptional {slog.WarnContext(ctx, "Ignoring unusable optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "reason", reason)@@ -534,12 +555,31 @@ func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, bo// os rather than acfs: the assets dir may be configured anywhere on the host.certPEM, certErr := os.ReadFile(certPath)rawKey, keyErr := os.ReadFile(keyPath)- keyPEM := ""- if ciphertext, encrypted := strings.CutPrefix(strings.TrimSpace(string(rawKey)), caKeyEncryptedPrefix); certErr == nil && keyErr == nil && encrypted {- keyPEM, _ = libcrypto.Decrypt(ciphertext)+ for _, readErr := range []error{certErr, keyErr} {+ if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {+ return tls.Certificate{}, false, fmt.Errorf("failed to read edge mTLS CA: %w", readErr)+ }}- if ca, pairErr := tls.X509KeyPair(certPEM, []byte(keyPEM)); pairErr == nil && ca.Leaf.IsCA && supportedGeneratedKey(ca.Leaf.PublicKey) {- return ca, false, nil+ ciphertext, encrypted := strings.CutPrefix(strings.TrimSpace(string(rawKey)), caKeyEncryptedPrefix)+ switch {+ case certErr != nil && keyErr != nil:+ case certErr != nil || keyErr != nil:+ slog.WarnContext(ctx, "Edge mTLS CA is incomplete; regenerating it", "certPath", certPath, "keyPath", keyPath)+ case !encrypted:+ slog.WarnContext(ctx, "Edge mTLS CA key is not in the supported encrypted format; regenerating the CA", "keyPath", keyPath)+ default:+ keyPEM, decryptErr := libcrypto.Decrypt(ciphertext)+ if decryptErr != nil {+ return tls.Certificate{}, false, fmt.Errorf("failed to decrypt edge mTLS CA private key %s: %w", keyPath, decryptErr)+ }+ ca, pairErr := tls.X509KeyPair(certPEM, []byte(keyPEM))+ if pairErr != nil {+ return tls.Certificate{}, false, fmt.Errorf("failed to load edge mTLS CA key pair from %s: %w", assetsDir, pairErr)+ }+ if ca.Leaf.IsCA && supportedGeneratedKey(ca.Leaf.PublicKey) {+ return ca, false, nil+ }+ slog.WarnContext(ctx, "Edge mTLS CA uses an unsupported algorithm; regenerating it", "certPath", certPath)}privateKey, err := certgen.GenerateMLDSA87PrivateKey()@@ -566,14 +606,14 @@ func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, boif err != nil {return tls.Certificate{}, false, fmt.Errorf("failed to marshal CA private key: %w", err)}- ciphertext, err := caKeyEncrypt(string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})))+ encryptedKey, err := caKeyEncrypt(string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})))if err != nil {return tls.Certificate{}, false, fmt.Errorf("failed to encrypt edge mTLS CA private key: %w", err)}- if ciphertext == "" {+ if encryptedKey == "" {return tls.Certificate{}, false, errors.New("failed to encrypt edge mTLS CA private key: encrypted payload is empty")}- if writeErr := atomic.WriteFile(keyPath, []byte(caKeyEncryptedPrefix+ciphertext), 0o600); writeErr != nil {+ if writeErr := atomic.WriteFile(keyPath, []byte(caKeyEncryptedPrefix+encryptedKey), 0o600); writeErr != nil {return tls.Certificate{}, false, writeErr}@@ -582,8 +622,8 @@ func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, bo}// loadClientCertificate loads a client key pair, or returns why it must be (re)issued: unreadable,-// mismatched, outside its validity window, or within agentMTLSRenewBefore of expiry.-func loadClientCertificate(certPath, keyPath string, now time.Time) (tls.Certificate, string) {+// mismatched, outside its validity window, or within renewBefore of expiry.+func loadClientCertificate(certPath, keyPath string, now time.Time, renewBefore time.Duration) (tls.Certificate, string) {cert, err := tls.LoadX509KeyPair(certPath, keyPath)switch {case err != nil:@@ -592,7 +632,7 @@ func loadClientCertificate(certPath, keyPath string, now time.Time) (tls.Certifireturn tls.Certificate{}, "certificate is not valid before " + cert.Leaf.NotBefore.UTC().Format(time.RFC3339)case !now.Before(cert.Leaf.NotAfter):return tls.Certificate{}, "certificate expired at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)- case now.Add(agentMTLSRenewBefore).After(cert.Leaf.NotAfter):+ case renewBefore > 0 && now.Add(renewBefore).After(cert.Leaf.NotAfter):return tls.Certificate{}, "certificate expires soon at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)}return cert, ""diff --git a/backend/pkg/libarcane/edge/tls_test.go b/backend/pkg/libarcane/edge/tls_test.goindex e7e6f8861..b301ac6fe 100644--- a/backend/pkg/libarcane/edge/tls_test.go+++ b/backend/pkg/libarcane/edge/tls_test.go@@ -8,6 +8,7 @@ import ("crypto/rand""crypto/tls""crypto/x509"+ "encoding/json/v2""encoding/pem""errors""fmt"@@ -192,6 +193,51 @@ func TestEnsureAgentMTLSAssets_LimitsEnrollmentErrorBody(t *testing.T) {require.LessOrEqual(t, len(err.Error()), maxEnrollResponseBytes+128)}+func TestEnsureAgentMTLSAssets_EnrollsWithoutUnusableGeneratedCert(t *testing.T) {+ generated, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSAssetsDir: t.TempDir(),+ AppURL: "https://manager.example.com",+ }, "env-renew", "Renew")+ require.NoError(t, err)++ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.TLS == nil || len(r.TLS.PeerCertificates) > 0 {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ _ = json.MarshalWrite(w, enrollMTLSResponse{Files: generated.Files})+ }))+ t.Cleanup(server.Close)++ caPath := filepath.Join(t.TempDir(), "manager.crt")+ require.NoError(t, os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{+ Type: "CERTIFICATE",+ Bytes: server.Certificate().Raw,+ }), 0o600))++ assetsDir := filepath.Join(t.TempDir(), "assets")+ require.NoError(t, os.MkdirAll(assetsDir, 0o700))+ certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)+ keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)+ require.NoError(t, os.WriteFile(certPath, []byte("not a certificate"), 0o600))+ require.NoError(t, os.WriteFile(keyPath, []byte("not a key"), 0o600))++ cfg := &Config{+ ManagerApiUrl: server.URL + "/api",+ AgentToken: "valid-token",+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSCAFile: caPath,+ EdgeMTLSCertFile: certPath,+ EdgeMTLSKeyFile: keyPath,+ EdgeMTLSAssetsDir: assetsDir,+ }++ require.NoError(t, EnsureAgentMTLSAssets(t.Context(), cfg))+ _, reason := loadClientCertificate(certPath, keyPath, time.Now(), 0)+ require.Empty(t, reason)+}+func TestEnsureAgentMTLSAssets_UsesDownloadedCAPathWhenPresent(t *testing.T) {assetsDir := t.TempDir()caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)@@ -380,13 +426,41 @@ func TestLoadClientCertificate_RenewsExpiredCertificate(t *testing.T) {certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)- cert, reason := loadClientCertificate(certPath, keyPath, time.Now())+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now(), agentMTLSRenewBefore)require.Empty(t, reason)- _, reason = loadClientCertificate(certPath, keyPath, cert.Leaf.NotAfter.Add(24*time.Hour))+ _, reason = loadClientCertificate(certPath, keyPath, cert.Leaf.NotAfter.Add(24*time.Hour), agentMTLSRenewBefore)require.Contains(t, reason, "expired")}+func TestLoadClientCertificate_RenewBeforeOnlyAppliesWhenRequested(t *testing.T) {+ assetsDir := t.TempDir()+ assets, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{+ EdgeMTLSMode: EdgeMTLSModeRequired,+ EdgeMTLSAssetsDir: t.TempDir(),+ AppURL: "https://manager.example.com",+ }, "env-window", "Window")+ require.NoError(t, err)++ for _, file := range assets.Files {+ targetPath := filepath.Join(assetsDir, filepath.Base(file.Name))+ perm := kit.Ternary(file.Permissions == "0600", 0o600, utils.FilePerm)+ require.NoError(t, os.WriteFile(targetPath, []byte(file.Content), perm))+ }++ certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)+ keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now(), 0)+ require.Empty(t, reason)++ nearExpiry := cert.Leaf.NotAfter.Add(-24 * time.Hour)+ _, reason = loadClientCertificate(certPath, keyPath, nearExpiry, 0)+ require.Empty(t, reason, "a certificate inside the renewal window stays usable when no window is requested")++ _, reason = loadClientCertificate(certPath, keyPath, nearExpiry, agentMTLSRenewBefore)+ require.Contains(t, reason, "expires soon")+}+func TestGenerateManagerClientMTLSAssets_ReissuesMismatchedKeyPair(t *testing.T) {cfg := &Config{EdgeMTLSMode: EdgeMTLSModeRequired,diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.goindex bcd67bb36..fe47084b8 100644--- a/backend/pkg/libarcane/edge/types.go+++ b/backend/pkg/libarcane/edge/types.go@@ -44,6 +44,11 @@ type commandRequestTransfer struct {timer *time.Timer}+type commandStatusError struct {+ status int+ message string+}+// TunnelClient represents the agent-side tunnel clienttype TunnelClient struct {cfg *Config@@ -62,6 +67,7 @@ type TunnelClient struct {activeStreams sync.Map // map[string]*activeWSStreamrequestTransfers sync.Map // map[string]*commandRequestTransfercommandRecorders sync.Map // map[string]*commandResponseRecorder awaiting credit+ commandCancels sync.MaptransportPreferenceMu sync.RWMutexpreferWebSocketUntil time.TimegrpcFailureStreak intdiff --git a/backend/pkg/remenv/client.go b/backend/pkg/remenv/client.goindex c3c0fe059..24db6b578 100644--- a/backend/pkg/remenv/client.go+++ b/backend/pkg/remenv/client.go@@ -10,8 +10,8 @@ import ("net""net/http""strings"- "time"+ "go.getarcane.app/kit/pkg""go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp")@@ -92,16 +92,14 @@ type Client struct {}func NewClient(httpClient *http.Client, tunnel TunnelTransport) *Client {- client := &http.Client{Timeout: 30 * time.Second}- if httpClient != nil {- copied := *httpClient- client = &copied- }- client.Transport = otelhttp.NewTransport(client.Transport)-+ base := kit.FromPtr(httpClient)return &Client{- httpClient: client,- tunnel: tunnel,+ httpClient: &http.Client{+ Transport: otelhttp.NewTransport(base.Transport),+ CheckRedirect: base.CheckRedirect,+ Jar: base.Jar,+ },+ tunnel: tunnel,}}diff --git a/backend/pkg/utils/httpx/request.go b/backend/pkg/utils/httpx/request.goindex 2e64fd434..46cd34f39 100644--- a/backend/pkg/utils/httpx/request.go+++ b/backend/pkg/utils/httpx/request.go@@ -153,6 +153,7 @@ func NewHTTPClient(options httpxtypes.ClientOptions) *http.Client {MaxIdleConnsPerHost: 32,IdleConnTimeout: 90 * time.Second,TLSHandshakeTimeout: options.TLSHandshakeTimeout,+ ResponseHeaderTimeout: options.ResponseHeaderTimeout,ExpectContinueTimeout: 1 * time.Second,ForceAttemptHTTP2: true,}diff --git a/frontend/src/lib/types/shared.ts b/frontend/src/lib/types/shared.tsindex 1648a972d..cad6eca9c 100644--- a/frontend/src/lib/types/shared.ts+++ b/frontend/src/lib/types/shared.ts@@ -238,7 +238,7 @@ export interface DashboardSnapshot {export type DashboardStreamEventType = 'snapshot' | 'error' | 'pending' | 'heartbeat';-export type DashboardStreamErrorCode = 'agent_incompatible' | 'unreachable' | '';+export type DashboardStreamErrorCode = 'agent_incompatible' | 'unreachable' | 'auth_failed' | '';export interface DashboardStreamEvent {type: DashboardStreamEventType;diff --git a/tests/spec/cli.spec.ts b/tests/spec/cli.spec.tsindex 08a5c877c..fb30da61a 100644--- a/tests/spec/cli.spec.ts+++ b/tests/spec/cli.spec.ts@@ -626,8 +626,8 @@ test.describe('arcane-cli e2e', () => {// - a DENIED request returns 403 — the permission check runs before the tunnel// lookup, so it short-circuits regardless of connectivity;// - an AUTHORIZED request gets past the permission check and reaches the tunnel-// stage, which returns 502 ("edge agent is not connected").-// The 403-vs-502 split is exactly what proves the permission is enforced per+// stage, which returns 503 ("Edge agent is not connected").+// The 403-vs-503 split is exactly what proves the permission is enforced per// environment and is not a blanket block. The local environment ("0") is served// directly (never proxied) and must remain unaffected.test.describe('arcane-cli remote environment RBAC', () => {@@ -692,7 +692,7 @@ test.describe('arcane-cli remote environment RBAC', () => {// Create a remote edge environment. No agent will ever connect to it; it// exists only so the proxy treats requests to it as remote (edge://) and- // reaches either the permission check (403) or the tunnel lookup (502).+ // reaches either the permission check (403) or the tunnel lookup (503).const envResponse = await adminFetch('/api/environments', {method: 'POST',body: JSON.stringify({@@ -777,7 +777,7 @@ test.describe('arcane-cli remote environment RBAC', () => {});});- test('authorizes a remote write for an admin key, reaching the disconnected agent (502, not 403)', async () => {+ test('authorizes a remote write for an admin key, reaching the disconnected agent (503, not 403)', async () => {await withAdminConfig(remoteEnvId, async (config) => {const error = await runExpectingFailure(config.configPath, ['volumes',@@ -787,7 +787,7 @@ test.describe('arcane-cli remote environment RBAC', () => {]);// The admin key passes the permission check, so the request reaches the// edge tunnel stage and fails only because no agent is connected.- expect(error).toContain('status 502');+ expect(error).toContain('status 503');expect(error).not.toContain('status 403');});});diff --git a/types/dashboard/dashboard.go b/types/dashboard/dashboard.goindex 35dcca89f..99263d30a 100644--- a/types/dashboard/dashboard.go+++ b/types/dashboard/dashboard.go@@ -28,6 +28,7 @@ const (StreamErrorCodeAgentIncompatible = "agent_incompatible"StreamErrorCodeUnreachable = "unreachable"+ StreamErrorCodeAuthFailed = "auth_failed")type ActionItemSeverity stringdiff --git a/types/httpx/client.go b/types/httpx/client.goindex 090f632c0..bfa54aec4 100644--- a/types/httpx/client.go+++ b/types/httpx/client.go@@ -5,7 +5,8 @@ import "time"// ClientOptions configures timeouts for an outbound HTTP client.// Durations are used as provided; a zero Timeout disables the request timeout.type ClientOptions struct {- Timeout time.Duration- TLSHandshakeTimeout time.Duration- DialTimeout time.Duration+ Timeout time.Duration+ TLSHandshakeTimeout time.Duration+ DialTimeout time.Duration+ ResponseHeaderTimeout time.Duration}