Skip to content

Modern Docker Management, Designed for Everyone

fix: harden edge environment access and health checks (#4489)

Kyle Mendelledb390d61c7d
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 2a9b8b33a..d555b336e 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -23,6 +23,7 @@ import (
"go.uber.org/fx/fxevent"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -206,6 +207,7 @@ type initializeStartupStateParams struct {
Docker *docker.DockerClientService
Swarm *swarm.SwarmService
Role *role.RoleService
+ Auth *auth.AuthService
User *user.UserService
ApiKey *apikey.ApiKeyService
Vulnerability *vulnerability.VulnerabilityService
@@ -286,6 +288,10 @@ func initializeStartupState(p initializeStartupStateParams) {
// Global variables are manager-owned: import a legacy local .env.global once, then sync everywhere.
p.Environment.SetVariableSyncer(p.Variable)
+ p.Environment.SetUserInvalidator(func(userID string) {
+ p.Role.InvalidateUser(userID)
+ p.Auth.InvalidateUserTokenCache(userID)
+ })
if err := p.Variable.ImportLegacyLocalEnvFile(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to import legacy global variables", "error", err)
}
@@ -429,7 +435,7 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config) error
}
defer func() { _ = resp.Body.Close() }()
- body, _ := io.ReadAll(resp.Body)
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
switch {
case resp.StatusCode == http.StatusOK:
@@ -443,5 +449,5 @@ func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config) error
slog.DebugContext(ctx, "Pairing skipped - API key not recognized (agent may already be paired)", "managerUrl", cfg.ManagerApiUrl)
return nil
}
- return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, string(body))
+ return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.go
index c4efd0122..08333ad55 100644
--- a/backend/internal/bootstrap/edge_bootstrap.go
+++ b/backend/internal/bootstrap/edge_bootstrap.go
@@ -111,7 +111,7 @@ func registerEdgeTunnelRoutes(
return stopCleanup(stopCtx)
},
})
- apiGroup.POST("/tunnel/poll", server.HandlePoll)
+ apiGroup.POST("/tunnel/poll", server.HandlePoll, middleware.PerIPRateLimit(6000, 1200), middleware.PerAgentTokenRateLimit(60, 30))
// Rate-limit agent mTLS enrollment per-IP. Enrollment is authenticated
// only by the agent token, so we cap bursts to mitigate brute-force or
// token-abuse attempts without impacting normal agent lifecycles.
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index ae5719f5f..fc01a26d2 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -331,6 +331,7 @@ func registerJobs(params registerJobsParams) error {
Scheduler: params.Scheduler,
Settings: params.Settings,
Environment: params.Environment,
+ JobSchedule: params.JobSchedule,
AutoUpdate: params.AutoUpdate,
ImageUpdateWatcher: params.ImageUpdateWatcher,
FilesystemWatcher: params.FilesystemWatcher,
@@ -410,6 +411,7 @@ type settingsSubscriptionsParams struct {
Scheduler settingsEffectsScheduler
Settings settingsChangeSubscriber
Environment timeoutSettingsEnvironment
+ JobSchedule *job.JobService
AutoUpdate *flow.Job
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
@@ -466,6 +468,16 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {
rescheduleOn([]string{"autoUpdate", "autoUpdateInterval"}, params.AutoUpdate)
+ subscribe([]string{"dockerClientRefreshInterval", "environmentHealthInterval"}, func(updates []libarcane.SettingUpdate) {
+ changedKeys := make([]string, 0, len(updates))
+ for _, update := range updates {
+ changedKeys = append(changedKeys, update.Key)
+ }
+ if err := params.JobSchedule.RescheduleJobsForSettingKeys(params.LifecycleCtx, changedKeys); err != nil {
+ slog.WarnContext(params.LifecycleCtx, "Failed to reschedule jobs after settings change", "keys", changedKeys, "error", err)
+ }
+ })
+
subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {
if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 5f70ab074..765075a3d 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -26,6 +26,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/federated"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/telemetry"
@@ -239,11 +240,17 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
))
// Keyed per token for stored tokens, per source IP for everything else.
apiGroup.Use(middleware.PerTokenRateLimitForPaths(
- []string{"/api/webhooks/trigger/:token"}, 60, 10, deps.Webhook.Service().IsKnownToken,
+ []string{"/api/webhooks/trigger/:token"}, 60, 10,
+ func(c *echo.Context) string { return c.Param("token") },
+ deps.Webhook.Service().IsKnownToken,
))
// Agent event ingestion sits outside the auth middleware, so it needs its own ceiling; agents batch events.
- apiGroup.Use(middleware.PerIPRateLimitForPaths(
- []string{"/api/events"}, 60, 30,
+ apiGroup.Use(middleware.PerTokenRateLimitForPaths(
+ []string{"/api/events"}, 600, 2*event.DaemonEventBatchSize,
+ func(c *echo.Context) string {
+ return strings.TrimSpace(c.Request().Header.Get(middleware.HeaderAgentToken))
+ },
+ deps.Environment.Service().IsKnownAccessToken,
))
handlerAppCtx := handlerutil.NewActivityAppContext(ctx)
diff --git a/backend/internal/dashboard/handler.go b/backend/internal/dashboard/handler.go
index cd7efa142..d8422e4b2 100644
--- a/backend/internal/dashboard/handler.go
+++ b/backend/internal/dashboard/handler.go
@@ -392,5 +392,8 @@ func classifyDashboardStreamErrorInternal(err error) (string, string) {
if transportErr, ok := errors.AsType[*remenv.TransportError](err); ok {
return transportErr.Error(), dashboard.StreamErrorCodeUnreachable
}
+ if statusErr, ok := errors.AsType[*remenv.StatusError](err); ok && (statusErr.StatusCode == http.StatusUnauthorized || statusErr.StatusCode == http.StatusForbidden) {
+ return "Agent rejected the environment access token — regenerate the API key and update the agent", dashboard.StreamErrorCodeAuthFailed
+ }
return err.Error(), ""
}
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index eea7f2759..7ef945d24 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -86,7 +86,11 @@ var ServiceOptions = fx.Options(
appimages.NewApplicationImagesService,
session.NewSessionService,
passkey.NewPasskeyService,
- environment.NewEnvironmentService,
+ fx.Annotate(environment.NewEnvironmentService,
+ fx.OnStart(func(ctx context.Context, service *environment.EnvironmentService) error {
+ return service.LoadAccessTokens(ctx)
+ }),
+ ),
apns.NewApnsService,
notification.NewNotificationService,
provideVulnerabilityService,
@@ -204,10 +208,14 @@ var ServiceOptions = fx.Options(
return settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)
},
func(service *s3domain.S3DestinationService, localEnvironment *environment.EnvironmentService) *s3domain.Module {
- return s3domain.New(service, localEnvironment.SyncS3DestinationsToRemoteEnvironments)
+ return s3domain.New(service, func(ctx context.Context) {
+ localEnvironment.QueueRemoteSync(ctx, "S3 destinations", localEnvironment.SyncS3DestinationsToRemoteEnvironments)
+ })
},
func(service *registry.ContainerRegistryService, localEnvironment *environment.EnvironmentService) *registry.Module {
- return registry.New(service, localEnvironment.SyncRegistriesToRemoteEnvironments)
+ return registry.New(service, func(ctx context.Context) {
+ localEnvironment.QueueRemoteSync(ctx, "registries", localEnvironment.SyncRegistriesToRemoteEnvironments)
+ })
},
func(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {
return auth.New(service, userService, settingsService, passkeyService.BeginMFAAuthentication)
diff --git a/backend/internal/environment/children/health/service.go b/backend/internal/environment/children/health/service.go
index 931c108e2..88c7a7a20 100644
--- a/backend/internal/environment/children/health/service.go
+++ b/backend/internal/environment/children/health/service.go
@@ -29,7 +29,7 @@ func New(httpClient *http.Client, dockerService *docker.DockerClientService) *Se
// Probe reports "online", "offline" or "error" for one environment. A custom
// API URL always probes directly over HTTP.
-func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customURL bool) (string, error) {
+func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customURL bool, authenticated func(context.Context) (int, error)) (string, error) {
if id == "0" && !customURL {
reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
@@ -50,18 +50,22 @@ func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customUR
return "offline", errors.New("edge agent is not connected")
}
}
-
+ }
+ if !customURL {
reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
- statusCode, _, err := edge.DoRequest(reqCtx, id, http.MethodGet, "/api/health", nil)
- if err != nil {
- return "offline", fmt.Errorf("health check via tunnel failed: %w", err)
- }
- if statusCode == http.StatusOK {
+ statusCode, err := authenticated(reqCtx)
+ switch {
+ case err != nil:
+ return "offline", fmt.Errorf("health check failed: %w", err)
+ case statusCode == http.StatusOK:
return "online", nil
+ case statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden:
+ return "error", fmt.Errorf("environment rejected the access token: status code %d", statusCode)
+ default:
+ return "error", fmt.Errorf("unexpected status code: %d", statusCode)
}
- return "error", fmt.Errorf("unexpected status code: %d", statusCode)
}
baseURL, err := httpx.NormalizeBaseURL(apiURL)
diff --git a/backend/internal/environment/children/proxy/service.go b/backend/internal/environment/children/proxy/service.go
index 90a1e114b..20123c0f1 100644
--- a/backend/internal/environment/children/proxy/service.go
+++ b/backend/internal/environment/children/proxy/service.go
@@ -38,6 +38,7 @@ type Service struct {
client *remenv.Client
settingsService *settings.SettingsService
syncGate *utils.SyncGate
+ syncLocks utils.KeyedMutex
}
// Target is a validated remote environment endpoint; build it with NewTarget.
@@ -308,6 +309,12 @@ func (s *Service) fanOutSyncToEnvironment[Model, Item, Request any](
toSyncItem func(Model) (Item, bool, error),
wrap func([]Item) Request,
) error {
+ unlock, err := s.syncLocks.LockContext(ctx, target.id+"\x00"+path)
+ if err != nil {
+ return fmt.Errorf("sync cancelled while waiting for an in-flight sync: %w", err)
+ }
+ defer unlock()
+
var records []Model
if loadRecordsErr := s.db.WithContext(ctx).Find(&records).Error; loadRecordsErr != nil {
return fmt.Errorf("failed to get %s: %w", kind, loadRecordsErr)
diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.go
index bc204fbf2..5841c1c33 100644
--- a/backend/internal/environment/handler.go
+++ b/backend/internal/environment/handler.go
@@ -453,7 +453,7 @@ func (h *EnvironmentHandler) UpdateEnvironment(ctx context.Context, input *Updat
}
}(context.WithoutCancel(ctx), input.ID, updated.Name)
}
- if kit.FromPtr(updated.AccessToken) != "" && (kit.FromPtr(input.Body.AccessToken) != "" || input.Body.Name != nil) {
+ if kit.FromPtr(updated.AccessToken) != "" && kit.FromPtr(input.Body.AccessToken) != "" {
h.triggerEnvironmentResourceSync(ctx, input.ID, updated.Name, "environment update")
}
diff --git a/backend/internal/environment/helpers.go b/backend/internal/environment/helpers.go
index 5086ed58a..d9b7479ea 100644
--- a/backend/internal/environment/helpers.go
+++ b/backend/internal/environment/helpers.go
@@ -6,6 +6,7 @@ import (
"sync"
"github.com/samber/hot"
+ "go.getarcane.app/kit/pkg"
)
// edgeTokenCache maps agent tokens to environment IDs; the reverse index drops
@@ -16,6 +17,12 @@ type edgeTokenCache struct {
byEnvID map[string]string
}
+type accessTokenIndex struct {
+ mu sync.RWMutex
+ byEnvID map[string]string
+ byHashed map[string]int
+}
+
// remoteEnvSnapshotCache holds every enabled, visible, non-local environment
// in memory so hot paths avoid a DB round trip.
type remoteEnvSnapshotCache struct {
@@ -38,6 +45,12 @@ const environmentCacheStripes = 64
var environmentCacheSeed = maphash.MakeSeed()
+type remoteSyncQueue struct {
+ mu sync.Mutex
+ running map[string]bool
+ pending map[string]bool
+}
+
// runtimeWatchers fans a coalesced wake-up out to environment liveness watchers.
type runtimeWatchers struct {
mu sync.Mutex
@@ -64,6 +77,28 @@ func (c *edgeTokenCache) sync(envID, token string) {
}
}
+func (i *accessTokenIndex) sync(envID, token string) {
+ if envID == "" {
+ return
+ }
+
+ i.mu.Lock()
+ defer i.mu.Unlock()
+
+ if previousHash, ok := i.byEnvID[envID]; ok {
+ delete(i.byEnvID, envID)
+ i.byHashed[previousHash]--
+ if i.byHashed[previousHash] <= 0 {
+ delete(i.byHashed, previousHash)
+ }
+ }
+ if resolvedToken := strings.TrimSpace(token); resolvedToken != "" {
+ hashed := kit.SHA256Hex(resolvedToken)
+ i.byEnvID[envID] = hashed
+ i.byHashed[hashed]++
+ }
+}
+
func isActiveRemoteEnvironment(env Environment) bool {
return env.ID != "" && env.ID != "0" && env.Enabled && !env.Hidden
}
diff --git a/backend/internal/environment/service.go b/backend/internal/environment/service.go
index e44bdee79..f6e5ac13c 100644
--- a/backend/internal/environment/service.go
+++ b/backend/internal/environment/service.go
@@ -13,6 +13,7 @@ import (
"net/url"
"slices"
"strings"
+ "sync"
"sync/atomic"
"time"
"uuid"
@@ -68,10 +69,13 @@ type EnvironmentService struct {
environmentCache *hot.HotCache[environmentCacheKey, Environment]
environmentGens [environmentCacheStripes]atomic.Uint64
accessTokens *hot.HotCache[string, string]
+ accessTokenIndex *accessTokenIndex
jobs *entityjobs.Registry
variableSyncer VariableSyncer
+ invalidateUser func(userID string)
runtimeWatchers runtimeWatchers
+ remoteSyncs remoteSyncQueue
syncGate *utils.SyncGate
proxy *proxy.Service
health *health.Service
@@ -162,13 +166,21 @@ func NewEnvironmentService(
WithMissingSharedCache().
WithJanitor().
Build(),
- accessTokens: hot.NewHotCache[string, string](hot.LRU, 1024).WithTTL(environmentCacheTTL).WithJanitor().Build(),
+ accessTokens: hot.NewHotCache[string, string](hot.LRU, 1024).WithTTL(environmentCacheTTL).WithJanitor().Build(),
+ accessTokenIndex: &accessTokenIndex{
+ byEnvID: make(map[string]string),
+ byHashed: make(map[string]int),
+ },
jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScope),
runtimeWatchers: runtimeWatchers{chans: make(map[int]chan struct{})},
- proxy: proxy.New(db, httpClient, settingsService, syncGate),
- health: health.New(httpClient, dockerService),
- snippets: snippets.New(eventService),
- swarm: swarm.New(apiKeyService),
+ remoteSyncs: remoteSyncQueue{
+ running: make(map[string]bool),
+ pending: make(map[string]bool),
+ },
+ proxy: proxy.New(db, httpClient, settingsService, syncGate),
+ health: health.New(httpClient, dockerService),
+ snippets: snippets.New(eventService),
+ swarm: swarm.New(apiKeyService),
}
}
@@ -219,6 +231,10 @@ func (s *EnvironmentService) SetVariableSyncer(syncer VariableSyncer) {
s.variableSyncer = syncer
}
+func (s *EnvironmentService) SetUserInvalidator(invalidate func(userID string)) {
+ s.invalidateUser = invalidate
+}
+
func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context, token string) (string, error) {
token = strings.TrimSpace(token)
if token == "" {
@@ -227,7 +243,15 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,
staleEnvironmentID, wasCached := s.edgeTokens.byToken.Peek(token)
if envID, ok, _ := s.edgeTokens.byToken.Get(token); ok {
- return envID, nil
+ cached, err := s.GetEnvironmentByIDCached(ctx, envID)
+ if err != nil {
+ return "", fmt.Errorf("failed to resolve cached edge environment: %w", err)
+ }
+ if cached.Enabled {
+ return envID, nil
+ }
+ s.edgeTokens.sync(envID, "")
+ return "", errors.New("invalid agent token")
}
if wasCached {
s.edgeTokens.mu.Lock()
@@ -241,6 +265,7 @@ func (s *EnvironmentService) ResolveEdgeEnvironmentByToken(ctx context.Context,
if err := s.db.WithContext(ctx).
Select("id", "access_token").
Where("is_edge = ?", true).
+ Where("enabled = ?", true).
Where("access_token = ?", token).
First(&env).Error; err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
@@ -359,12 +384,13 @@ func (s *EnvironmentService) CreateEnvironment(ctx context.Context, env *Environ
return nil, fmt.Errorf("failed to create environment: %w", err)
}
- go s.createEnvironmentEvent(context.WithoutCancel(ctx), env.ID, env.Name, event.EventTypeEnvironmentCreate,
+ s.createEnvironmentEvent(ctx, env.ID, env.Name, event.EventTypeEnvironmentCreate,
"Environment Created", fmt.Sprintf("Environment '%s' was created", env.Name), event.EventSeveritySuccess, userID, username)
if env.Enabled {
s.registerHealthJob(ctx, env.ID)
}
+ s.accessTokenIndex.sync(env.ID, kit.FromPtr(env.AccessToken))
s.remoteEnvs.put(*env)
s.invalidateEnvironmentCache(env.ID)
s.NotifyRuntimeStateChanged()
@@ -431,9 +457,9 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, u
}
s.remoteEnvs.put(*updated)
- if rawAccessToken, ok := updates["access_token"]; ok {
- accessToken, _ := rawAccessToken.(string)
- s.edgeTokens.sync(id, accessToken)
+ if _, ok := updates["access_token"]; ok {
+ s.edgeTokens.sync(id, "")
+ s.accessTokenIndex.sync(id, kit.FromPtr(updated.AccessToken))
}
// Reconcile the per-environment health job when the enabled flag is toggled.
@@ -443,12 +469,14 @@ func (s *EnvironmentService) UpdateEnvironment(ctx context.Context, id string, u
s.registerHealthJob(ctx, id)
} else {
s.jobs.Unregister(ctx, id)
+ s.edgeTokens.sync(id, "")
+ edge.GetRegistry().Unregister(id)
}
}
}
if id != LocalEnvironmentID {
- go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, updated.Name, event.EventTypeEnvironmentUpdate,
+ s.createEnvironmentEvent(ctx, id, updated.Name, event.EventTypeEnvironmentUpdate,
"Environment Updated", fmt.Sprintf("Environment '%s' was updated", updated.Name), event.EventSeverityInfo, userID, username)
}
@@ -464,7 +492,7 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
// Stop the per-environment health job before the row is removed.
s.jobs.Unregister(ctx, id)
- var syncIDs []string
+ var syncIDs, federatedUserIDs []string
if transactionErr := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if loadSyncIDsErr := tx.Table("gitops_syncs").
Where("environment_id = ?", id).
@@ -482,6 +510,29 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
return fmt.Errorf("failed to delete environment gitops syncs: %w", deleteSyncsErr)
}
}
+ if loadFederatedUsersErr := tx.Table("federated_credentials").
+ Where("environment_id = ?", id).
+ Pluck("identity_user_id", &federatedUserIDs).Error; loadFederatedUsersErr != nil {
+ return fmt.Errorf("failed to list environment federated credentials: %w", loadFederatedUsersErr)
+ }
+ if len(federatedUserIDs) > 0 {
+ if deleteCredentialsErr := tx.Exec("DELETE FROM federated_credentials WHERE environment_id = ?", id).Error; deleteCredentialsErr != nil {
+ return fmt.Errorf("failed to delete environment federated credentials: %w", deleteCredentialsErr)
+ }
+ if deleteUsersErr := tx.Exec("DELETE FROM users WHERE id IN ?", federatedUserIDs).Error; deleteUsersErr != nil {
+ return fmt.Errorf("failed to delete environment federated service users: %w", deleteUsersErr)
+ }
+ }
+ if deleteVariablesErr := tx.Exec(`DELETE FROM global_variables
+ WHERE all_environments = ?
+ AND id IN (SELECT global_variable_id FROM global_variable_environments WHERE environment_id = ?)
+ AND id NOT IN (SELECT global_variable_id FROM global_variable_environments WHERE environment_id <> ?)`,
+ false, id, id).Error; deleteVariablesErr != nil {
+ return fmt.Errorf("failed to delete environment-scoped global variables: %w", deleteVariablesErr)
+ }
+ if deleteVariableScopesErr := tx.Exec("DELETE FROM global_variable_environments WHERE environment_id = ?", id).Error; deleteVariableScopesErr != nil {
+ return fmt.Errorf("failed to delete environment global variable scopes: %w", deleteVariableScopesErr)
+ }
if deleteEnvironmentErr := tx.Delete(&Environment{}, "id = ?", id).Error; deleteEnvironmentErr != nil {
return fmt.Errorf("failed to delete environment: %w", deleteEnvironmentErr)
}
@@ -501,7 +552,15 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
}
}
+ if s.invalidateUser != nil {
+ for _, federatedUserID := range federatedUserIDs {
+ s.invalidateUser(federatedUserID)
+ }
+ }
+
s.edgeTokens.sync(id, "")
+ s.accessTokenIndex.sync(id, "")
+ edge.GetRegistry().Unregister(id)
s.ForgetSyncState(id)
s.remoteEnvs.mu.Lock()
delete(s.remoteEnvs.envs, id)
@@ -510,7 +569,7 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
s.invalidateEnvironmentCache(id)
s.NotifyRuntimeStateChanged()
- go s.createEnvironmentEvent(context.WithoutCancel(ctx), id, env.Name, event.EventTypeEnvironmentDelete,
+ s.createEnvironmentEvent(ctx, id, env.Name, event.EventTypeEnvironmentDelete,
"Environment Deleted", fmt.Sprintf("Environment '%s' was deleted", env.Name), event.EventSeverityWarning, userID, username)
return nil
@@ -524,7 +583,9 @@ func (s *EnvironmentService) createEnvironmentEvent(
severity event.EventSeverity,
userID, username *string,
) {
- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
+ eventCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+ defer cancel()
+ if _, err := s.eventService.CreateEvent(eventCtx, event.CreateEventRequest{
Type: eventType,
Severity: severity,
Title: title,
@@ -535,7 +596,9 @@ func (s *EnvironmentService) createEnvironmentEvent(
UserID: userID,
Username: username,
EnvironmentID: new(envID),
- })
+ }); err != nil {
+ slog.WarnContext(ctx, "Failed to record environment event", "environmentId", envID, "eventType", eventType, "error", err)
+ }
}
// RegenerateEnvironmentApiKey rotates the key and updates its environment binding.
@@ -566,7 +629,8 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, n
s.invalidateEnvironmentCache(envID)
s.NotifyRuntimeStateChanged()
- s.edgeTokens.sync(envID, apiKey)
+ s.edgeTokens.sync(envID, "")
+ s.accessTokenIndex.sync(envID, apiKey)
s.remoteEnvs.mu.Lock()
s.remoteEnvs.revision++
if envRecord, ok := s.remoteEnvs.envs[envID]; ok {
@@ -591,8 +655,9 @@ func (s *EnvironmentService) linkEnvironmentApiKey(ctx context.Context, envID, n
}
}
s.remoteEnvs.mu.Unlock()
+ edge.GetRegistry().Unregister(envID)
- go s.createEnvironmentEvent(context.WithoutCancel(ctx), envID, envName, event.EventTypeEnvironmentApiKeyRegenerated,
+ s.createEnvironmentEvent(ctx, envID, envName, event.EventTypeEnvironmentApiKeyRegenerated,
"API Key Regenerated", "Environment API key was regenerated and status set to pending", event.EventSeverityInfo, new(userID), new(username))
return nil
@@ -630,6 +695,32 @@ func (s *EnvironmentService) ResolveEnvironmentByAccessToken(ctx context.Context
return &env, nil
}
+func (s *EnvironmentService) LoadAccessTokens(ctx context.Context) error {
+ var envs []Environment
+ if err := s.db.WithContext(ctx).
+ Select("id", "access_token").
+ Where("access_token IS NOT NULL AND access_token != ?", "").
+ Find(&envs).Error; err != nil {
+ return fmt.Errorf("failed to load environment access tokens: %w", err)
+ }
+ for _, env := range envs {
+ s.accessTokenIndex.sync(env.ID, kit.FromPtr(env.AccessToken))
+ }
+ return nil
+}
+
+func (s *EnvironmentService) IsKnownAccessToken(token string) bool {
+ token = strings.TrimSpace(token)
+ if token == "" || len(token) > maxAccessTokenLength {
+ return false
+ }
+ hashed := kit.SHA256Hex(token)
+ s.accessTokenIndex.mu.RLock()
+ defer s.accessTokenIndex.mu.RUnlock()
+ _, ok := s.accessTokenIndex.byHashed[hashed]
+ return ok
+}
+
func (s *EnvironmentService) GetEnabledRegistryCredentials(ctx context.Context) ([]containerregistry.Credential, error) {
var registries []registry.ContainerRegistry
if err := s.db.WithContext(ctx).Where("enabled = ?", true).Find(&registries).Error; err != nil {
@@ -710,8 +801,10 @@ func DisplayName(environmentID, storedName string) string {
}
const (
- edgeTokenCacheTTL = time.Minute
- environmentCacheTTL = 30 * time.Second
+ edgeTokenCacheTTL = time.Minute
+ environmentCacheTTL = 30 * time.Second
+ remoteSyncTimeout = time.Minute
+ maxAccessTokenLength = 128
)
// GetActiveRemoteEnvironmentSnapshot returns the latest in-process snapshot for
@@ -782,7 +875,7 @@ func (s *EnvironmentService) invalidateEnvironmentCache(id string) {
}
func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, params pagination.QueryParams, accessibleEnvIDs []string) ([]environment.Environment, pagination.Response, error) {
- if strings.TrimSpace(params.Filters["type"]) != "" {
+ if strings.TrimSpace(params.Filters["type"]) != "" || strings.TrimSpace(params.Filters["status"]) != "" || params.Sort == "status" {
var envs []Environment
if err := s.db.WithContext(ctx).
Model(&Environment{}).
@@ -866,6 +959,9 @@ func (s *EnvironmentService) ListEnvironmentsPaginated(ctx context.Context, para
if !item.IsEdge {
return strings.ToLower(strings.TrimSpace(filterValue)) == "http"
}
+ if strings.ToLower(strings.TrimSpace(filterValue)) == "polling" {
+ return item.LastPollAt != nil
+ }
// Disconnected or poll-only agents classify by the transport they last used.
switch strings.ToLower(strings.TrimSpace(kit.Ternary(kit.FromPtr(item.Connected) && item.EdgeTransport != nil,
kit.FromPtr(item.EdgeTransport), kit.FromPtr(item.LastEdgeTransport)))) {
@@ -942,7 +1038,7 @@ func (s *EnvironmentService) ListVisibleEnvironments(ctx context.Context) ([]env
// ListRemoteEnvironmentIDs returns the IDs of enabled remote environments; it
// satisfies agg.RemoteEnvironmentLister for aggregated stream handlers.
func (s *EnvironmentService) ListRemoteEnvironmentIDs(ctx context.Context) ([]string, error) {
- envs, err := s.ListRemoteEnvironments(ctx)
+ envs, err := s.ListActiveRemoteEnvironments(ctx)
if err != nil {
return nil, err
}
@@ -955,37 +1051,32 @@ func (s *EnvironmentService) ListRemoteEnvironmentIDs(ctx context.Context) ([]st
// ListRemoteEnvironments returns all non-local, enabled environments for syncing purposes.
func (s *EnvironmentService) ListRemoteEnvironments(ctx context.Context) ([]Environment, error) {
- for {
- s.remoteEnvs.mu.RLock()
- revision := s.remoteEnvs.revision
- s.remoteEnvs.mu.RUnlock()
+ s.remoteEnvs.mu.RLock()
+ revision := s.remoteEnvs.revision
+ s.remoteEnvs.mu.RUnlock()
- var envs []Environment
- err := s.db.WithContext(ctx).
- Model(&Environment{}).
- Where("id != ?", LocalEnvironmentID).
- Where("enabled = ?", true).
- Where("hidden = ?", false).
- Find(&envs).Error
- if err != nil {
- return nil, fmt.Errorf("failed to list remote environments: %w", err)
- }
+ var envs []Environment
+ if err := s.db.WithContext(ctx).
+ Model(&Environment{}).
+ Where("id != ?", LocalEnvironmentID).
+ Where("enabled = ?", true).
+ Where("hidden = ?", false).
+ Find(&envs).Error; err != nil {
+ return nil, fmt.Errorf("failed to list remote environments: %w", err)
+ }
- s.remoteEnvs.mu.Lock()
- // Retry if a cache mutation or another refresh overtook the database read.
- if s.remoteEnvs.revision != revision {
- s.remoteEnvs.mu.Unlock()
- continue
- }
- s.remoteEnvs.envs = make(map[string]Environment, len(envs))
- for _, envRecord := range envs {
- s.remoteEnvs.envs[envRecord.ID] = envRecord
- }
- s.remoteEnvs.seeded = true
- s.remoteEnvs.revision++
- s.remoteEnvs.mu.Unlock()
+ s.remoteEnvs.mu.Lock()
+ defer s.remoteEnvs.mu.Unlock()
+ if s.remoteEnvs.revision != revision {
return envs, nil
}
+ s.remoteEnvs.envs = make(map[string]Environment, len(envs))
+ for _, envRecord := range envs {
+ s.remoteEnvs.envs[envRecord.ID] = envRecord
+ }
+ s.remoteEnvs.seeded = true
+ s.remoteEnvs.revision++
+ return envs, nil
}
// SubscribeRuntimeChanges returns a channel that receives a coalesced wake-up
@@ -1060,6 +1151,18 @@ func (s *EnvironmentService) ListEnabledEnvironmentIDs(ctx context.Context) ([]s
return ids, nil
}
+func (s *EnvironmentService) ListEnabledRemoteEnvironments(ctx context.Context) ([]Environment, error) {
+ var envs []Environment
+ if err := s.db.WithContext(ctx).
+ Model(&Environment{}).
+ Where("enabled = ?", true).
+ Where("id <> ?", LocalEnvironmentID).
+ Find(&envs).Error; err != nil {
+ return nil, fmt.Errorf("failed to list enabled remote environments: %w", err)
+ }
+ return envs, nil
+}
+
// RescheduleHealthJobs re-registers every enabled environment's health job,
// picking up a changed global interval.
func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {
@@ -1197,7 +1300,18 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, cust
return "error", err
}
- status, err := s.health.Probe(ctx, id, mo.PointerToOption(customApiUrl).OrElse(envRecord.ApiUrl), envRecord.IsEdge, customApiUrl != nil)
+ status, err := s.health.Probe(ctx, id, mo.PointerToOption(customApiUrl).OrElse(envRecord.ApiUrl), envRecord.IsEdge, customApiUrl != nil,
+ func(probeCtx context.Context) (int, error) {
+ target, targetErr := proxy.NewTarget(envRecord.ID, envRecord.Name, envRecord.ApiUrl, envRecord.IsEdge, envRecord.AccessToken)
+ if targetErr != nil {
+ return 0, targetErr
+ }
+ resp, executeErr := s.proxy.Execute(probeCtx, target, http.MethodHead, "/api/environments/0/system/health", nil)
+ if executeErr != nil {
+ return 0, executeErr
+ }
+ return resp.StatusCode, nil
+ })
if customApiUrl == nil {
if updateErr := s.updateEnvironmentStatus(ctx, id, status); updateErr != nil {
slog.WarnContext(ctx, "Failed to persist environment status", "environmentId", id, "status", status, "error", updateErr)
@@ -1212,29 +1326,25 @@ func (s *EnvironmentService) TestConnection(ctx context.Context, id string, cust
}
func (s *EnvironmentService) updateEnvironmentStatus(ctx context.Context, id, status string) error {
- var currentEnv Environment
- if err := s.db.WithContext(ctx).Select("status", "is_edge").Where("id = ?", id).First(&currentEnv).Error; err != nil {
- return fmt.Errorf("failed to check environment status: %w", err)
- }
-
- if currentEnv.Status == string(EnvironmentStatusPending) {
- // Only a successful check pairs a direct environment; edge environments pair
- // through the agent's tunnel, so manager reachability means nothing for them.
- if currentEnv.IsEdge || status != string(EnvironmentStatusOnline) {
- slog.DebugContext(ctx, "skipping status update for pending environment", "environmentId", id)
- return nil
- }
- slog.InfoContext(ctx, "promoted pending direct environment to online via reachability check", "environmentId", id)
- }
-
now := time.Now()
updates := map[string]any{
"status": status,
- "last_seen": &now,
"updated_at": &now,
}
- if err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {
- return fmt.Errorf("failed to update environment status: %w", err)
+ query := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id)
+ if status == string(EnvironmentStatusOnline) {
+ updates["last_seen"] = &now
+ query = query.Where("status <> ? OR is_edge = ?", string(EnvironmentStatusPending), false)
+ } else {
+ query = query.Where("status <> ?", string(EnvironmentStatusPending))
+ }
+ result := query.Updates(updates)
+ if result.Error != nil {
+ return fmt.Errorf("failed to update environment status: %w", result.Error)
+ }
+ if result.RowsAffected == 0 {
+ slog.DebugContext(ctx, "skipping status update for pending environment", "environmentId", id)
+ return nil
}
// Direct environments have no tunnel callback, so this is their only liveness change.
s.NotifyRuntimeStateChanged()
@@ -1282,7 +1392,11 @@ func (s *EnvironmentService) UpdateEnvironmentConnectionState(ctx context.Contex
updates["status"] = string(EnvironmentStatusOffline)
}
- if err := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id).Updates(updates).Error; err != nil {
+ query := s.db.WithContext(ctx).Model(&Environment{}).Where("id = ?", id)
+ if !connected {
+ query = query.Where("status <> ?", string(EnvironmentStatusPending))
+ }
+ if err := query.Updates(updates).Error; err != nil {
return fmt.Errorf("failed to update environment connection state: %w", err)
}
@@ -1370,26 +1484,59 @@ func (s *EnvironmentService) SyncS3DestinationsToRemoteEnvironments(ctx context.
return s.syncToRemoteEnvironments(ctx, "S3 destinations", s.SyncS3DestinationsToEnvironment)
}
-func (s *EnvironmentService) syncToRemoteEnvironments(ctx context.Context, kind string, sync func(context.Context, string) error) error {
+func (s *EnvironmentService) QueueRemoteSync(ctx context.Context, kind string, syncAll func(context.Context) error) {
+ s.remoteSyncs.mu.Lock()
+ defer s.remoteSyncs.mu.Unlock()
+ if s.remoteSyncs.running[kind] {
+ s.remoteSyncs.pending[kind] = true
+ return
+ }
+ s.remoteSyncs.running[kind] = true
+
+ go func(runCtx context.Context) {
+ for {
+ if err := syncAll(runCtx); err != nil {
+ slog.WarnContext(runCtx, "Failed to fan out sync to remote environments", "kind", kind, "error", err)
+ }
+ s.remoteSyncs.mu.Lock()
+ if !s.remoteSyncs.pending[kind] || runCtx.Err() != nil {
+ delete(s.remoteSyncs.running, kind)
+ delete(s.remoteSyncs.pending, kind)
+ s.remoteSyncs.mu.Unlock()
+ return
+ }
+ delete(s.remoteSyncs.pending, kind)
+ s.remoteSyncs.mu.Unlock()
+ }
+ }(s.jobs.Context(ctx))
+}
+
+func (s *EnvironmentService) syncToRemoteEnvironments(ctx context.Context, kind string, syncEnvironment func(context.Context, string) error) error {
envs, err := s.ListRemoteEnvironments(ctx)
if err != nil {
return fmt.Errorf("failed to list remote environments for %s sync: %w", kind, err)
}
- var failedCount int
+ var failedCount atomic.Int32
+ var wg sync.WaitGroup
for _, env := range envs {
if strings.TrimSpace(kit.FromPtr(env.AccessToken)) == "" {
slog.DebugContext(ctx, "Skipping sync for environment without access token", "kind", kind, "environmentId", env.ID, "environmentName", env.Name)
continue
}
- if syncErr := sync(ctx, env.ID); syncErr != nil {
- failedCount++
- slog.WarnContext(ctx, "Failed to sync to remote environment", "kind", kind, "environmentId", env.ID, "environmentName", env.Name, "error", syncErr)
- }
+ wg.Go(func() {
+ syncCtx, cancel := context.WithTimeout(ctx, remoteSyncTimeout)
+ defer cancel()
+ if syncErr := syncEnvironment(syncCtx, env.ID); syncErr != nil {
+ failedCount.Add(1)
+ slog.WarnContext(syncCtx, "Failed to sync to remote environment", "kind", kind, "environmentId", env.ID, "environmentName", env.Name, "error", syncErr)
+ }
+ })
}
+ wg.Wait()
- if failedCount > 0 {
- return fmt.Errorf("failed to sync %s to %d remote environment(s)", kind, failedCount)
+ if failed := failedCount.Load(); failed > 0 {
+ return fmt.Errorf("failed to sync %s to %d remote environment(s)", kind, failed)
}
return nil
}
diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.go
index 2fc853b14..21c6ee3dc 100644
--- a/backend/internal/environment/service_test.go
+++ b/backend/internal/environment/service_test.go
@@ -72,6 +72,30 @@ type testGitOpsSyncRow struct {
SyncInterval int
}
+type testFederatedCredentialRow struct {
+ database.BaseModel
+ Name string
+ IdentityUserID string `gorm:"column:identity_user_id"`
+ EnvironmentID *string `gorm:"column:environment_id"`
+}
+
+type testGlobalVariableRow struct {
+ database.BaseModel
+ Key string
+ AllEnvironments bool `gorm:"column:all_environments"`
+}
+
+type testGlobalVariableEnvironmentRow struct {
+ GlobalVariableID string `gorm:"column:global_variable_id;primaryKey"`
+ EnvironmentID string `gorm:"column:environment_id;primaryKey"`
+}
+
+func (testFederatedCredentialRow) TableName() string { return "federated_credentials" }
+
+func (testGlobalVariableRow) TableName() string { return "global_variables" }
+
+func (testGlobalVariableEnvironmentRow) TableName() string { return "global_variable_environments" }
+
func TestEnvironmentService_OverlappingHealthCheckIsSkipped(t *testing.T) {
gate := newAdmissionGateForEnvironmentTest(t)
@@ -130,6 +154,9 @@ func setupEnvironmentServiceTestDB(t *testing.T) *database.DB {
&event.Event{},
&testProjectRow{},
&testGitOpsSyncRow{},
+ &testFederatedCredentialRow{},
+ &testGlobalVariableRow{},
+ &testGlobalVariableEnvironmentRow{},
))
sqlDB, err := db.DB()
@@ -245,6 +272,80 @@ func TestEnvironmentService_DeleteEnvironment_CascadesGitOpsSyncs(t *testing.T)
require.Contains(t, jobScheduler.removed, entityjobs.GitOpsSyncJobPrefix+syncID)
}
+func TestEnvironmentService_DeleteEnvironment_RemovesScopedCredentialsAndVariables(t *testing.T) {
+ ctx := t.Context()
+ db := setupEnvironmentServiceTestDB(t)
+
+ createTestEnvironment(t, db, "env-delete-scoped", "http://scoped.example", nil)
+ createTestEnvironment(t, db, "env-keep", "http://keep.example", nil)
+ require.NoError(t, db.Create(&user.User{ID: "federated-user", Username: "federated-user"}).Error)
+ require.NoError(t, db.Create(&testFederatedCredentialRow{
+ ID: "credential-scoped",
+ Name: "scoped",
+ IdentityUserID: "federated-user",
+ EnvironmentID: new("env-delete-scoped"),
+ }).Error)
+ require.NoError(t, db.Create(&[]testGlobalVariableRow{
+ {ID: "var-only-deleted", Key: "ONLY_DELETED"},
+ {ID: "var-shared", Key: "SHARED"},
+ {ID: "var-all", Key: "ALL", AllEnvironments: true},
+ }).Error)
+ require.NoError(t, db.Create(&[]testGlobalVariableEnvironmentRow{
+ {GlobalVariableID: "var-only-deleted", EnvironmentID: "env-delete-scoped"},
+ {GlobalVariableID: "var-shared", EnvironmentID: "env-delete-scoped"},
+ {GlobalVariableID: "var-shared", EnvironmentID: "env-keep"},
+ }).Error)
+
+ svc := NewEnvironmentService(db, http.DefaultClient, nil, event.NewEventService(db, nil, nil), nil, nil)
+ var invalidatedUserIDs []string
+ svc.SetUserInvalidator(func(userID string) { invalidatedUserIDs = append(invalidatedUserIDs, userID) })
+ require.NoError(t, svc.DeleteEnvironment(ctx, "env-delete-scoped", nil, nil))
+ require.Equal(t, []string{"federated-user"}, invalidatedUserIDs)
+
+ var credentialCount, userCount int64
+ require.NoError(t, db.Model(&testFederatedCredentialRow{}).Count(&credentialCount).Error)
+ require.NoError(t, db.Model(&user.User{}).Where("id = ?", "federated-user").Count(&userCount).Error)
+ require.Zero(t, credentialCount)
+ require.Zero(t, userCount)
+
+ var variableIDs []string
+ require.NoError(t, db.Model(&testGlobalVariableRow{}).Order("id").Pluck("id", &variableIDs).Error)
+ require.Equal(t, []string{"var-all", "var-shared"}, variableIDs)
+
+ var scopes []testGlobalVariableEnvironmentRow
+ require.NoError(t, db.Find(&scopes).Error)
+ require.Equal(t, []testGlobalVariableEnvironmentRow{{GlobalVariableID: "var-shared", EnvironmentID: "env-keep"}}, scopes)
+}
+
+func TestEnvironmentService_IsKnownAccessToken_TracksTokenWrites(t *testing.T) {
+ ctx := t.Context()
+ db := setupEnvironmentServiceTestDB(t)
+ createTestEnvironmentWithState(t, db, "env-known", "http://known.example", string(EnvironmentStatusOnline), false, new("seeded-token"))
+ createTestEnvironmentWithState(t, db, "env-shared", "http://shared.example", string(EnvironmentStatusOnline), false, new("seeded-token"))
+ svc := NewEnvironmentService(db, http.DefaultClient, nil, event.NewEventService(db, nil, nil), nil, nil)
+
+ require.False(t, svc.IsKnownAccessToken("seeded-token"))
+ require.NoError(t, svc.LoadAccessTokens(ctx))
+ require.True(t, svc.IsKnownAccessToken("seeded-token"))
+ require.False(t, svc.IsKnownAccessToken(strings.Repeat("x", maxAccessTokenLength+1)))
+
+ require.NoError(t, svc.linkEnvironmentApiKey(ctx, "env-known", "key-rotated", "rotated-token", "", "", "env-known"))
+ require.True(t, svc.IsKnownAccessToken("seeded-token"))
+ require.True(t, svc.IsKnownAccessToken("rotated-token"))
+
+ _, err := svc.UpdateEnvironment(ctx, "env-known", map[string]any{"access_token": "updated-token"}, nil, nil)
+ require.NoError(t, err)
+ require.False(t, svc.IsKnownAccessToken("rotated-token"))
+ require.True(t, svc.IsKnownAccessToken("updated-token"))
+
+ require.NoError(t, svc.DeleteEnvironment(ctx, "env-known", nil, nil))
+ require.False(t, svc.IsKnownAccessToken("updated-token"))
+ require.True(t, svc.IsKnownAccessToken("seeded-token"))
+
+ require.NoError(t, svc.DeleteEnvironment(ctx, "env-shared", nil, nil))
+ require.False(t, svc.IsKnownAccessToken("seeded-token"))
+}
+
func createTestRegistry(t *testing.T, db *database.DB, id string) {
t.Helper()
@@ -737,9 +838,29 @@ func TestEnvironmentService_ResolveEdgeEnvironmentByToken_CachesAndInvalidatesOn
require.NoError(t, err)
require.Equal(t, "edge-auth", envID)
- require.NoError(t, svc.DeleteEnvironment(ctx, "edge-auth", nil, nil))
+ _, err = svc.UpdateEnvironment(ctx, "edge-auth", map[string]any{"enabled": false}, nil, nil)
+ require.NoError(t, err)
_, err = svc.ResolveEdgeEnvironmentByToken(ctx, newToken)
require.Error(t, err)
+
+ require.NoError(t, svc.linkEnvironmentApiKey(ctx, "edge-auth", "key-rotated", "edge-token-rotated", "", "", "edge-auth"))
+ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")
+ require.Error(t, err)
+
+ _, err = svc.UpdateEnvironment(ctx, "edge-auth", map[string]any{"enabled": true}, nil, nil)
+ require.NoError(t, err)
+ envID, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")
+ require.NoError(t, err)
+ require.Equal(t, "edge-auth", envID)
+
+ require.NoError(t, db.WithContext(ctx).Model(&Environment{}).Where("id = ?", "edge-auth").Update("enabled", false).Error)
+ svc.invalidateEnvironmentCache("edge-auth")
+ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")
+ require.Error(t, err)
+
+ require.NoError(t, svc.DeleteEnvironment(ctx, "edge-auth", nil, nil))
+ _, err = svc.ResolveEdgeEnvironmentByToken(ctx, "edge-token-rotated")
+ require.Error(t, err)
require.Contains(t, err.Error(), "invalid agent token")
}
diff --git a/backend/internal/event/service.go b/backend/internal/event/service.go
index aa4cc725e..4c3aac2ae 100644
--- a/backend/internal/event/service.go
+++ b/backend/internal/event/service.go
@@ -39,11 +39,15 @@ import (
)
// maxConcurrentEdgeSyncs bounds in-flight event forwards to the manager.
-const maxConcurrentEdgeSyncs = 16
+const (
+ maxConcurrentEdgeSyncs = 16
+ edgeSyncQueueTimeout = time.Minute
+ edgeSyncPublishTimeout = 10 * time.Second
+)
const (
daemonEventQueueSize = 1024
- daemonEventBatchSize = 64
+ DaemonEventBatchSize = 64
daemonEventFlushInterval = 250 * time.Millisecond
)
@@ -193,15 +197,19 @@ func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Ev
}
go func(parentCtx context.Context, outgoing *edge.TunnelEvent) {
- syncCtx, cancel := context.WithTimeout(context.WithoutCancel(parentCtx), 10*time.Second)
- defer cancel()
-
- if err := s.edgeSyncs.Acquire(syncCtx, 1); err != nil {
- slog.WarnContext(syncCtx, "Dropped event sync to manager; too many in flight", "type", outgoing.Type, "error", err)
+ baseCtx := context.WithoutCancel(parentCtx)
+ acquireCtx, cancelAcquire := context.WithTimeout(baseCtx, edgeSyncQueueTimeout)
+ acquireErr := s.edgeSyncs.Acquire(acquireCtx, 1)
+ cancelAcquire()
+ if acquireErr != nil {
+ slog.WarnContext(baseCtx, "Dropped event sync to manager; too many in flight", "type", outgoing.Type, "error", acquireErr)
return
}
defer s.edgeSyncs.Release(1)
+ syncCtx, cancel := context.WithTimeout(baseCtx, edgeSyncPublishTimeout)
+ defer cancel()
+
if err := edge.PublishEventToManager(syncCtx, outgoing); err != nil {
if !errors.Is(err, edge.ErrNoActiveAgentTunnel) {
slog.WarnContext(syncCtx, "Failed to sync event to manager over edge tunnel", "type", outgoing.Type, "error", err)
@@ -718,7 +726,7 @@ func (s *EventService) RecordDockerEvent(ctx context.Context, msg events.Message
func (s *EventService) writeDaemonEvents(ctx context.Context) {
defer close(s.daemonDone)
- batch := make([]CreateEventRequest, 0, daemonEventBatchSize)
+ batch := make([]CreateEventRequest, 0, DaemonEventBatchSize)
var flush <-chan time.Time
for open := true; open; {
select {
@@ -729,7 +737,7 @@ func (s *EventService) writeDaemonEvents(ctx context.Context) {
if flush == nil {
flush = time.After(daemonEventFlushInterval)
}
- if len(batch) < daemonEventBatchSize {
+ if len(batch) < DaemonEventBatchSize {
continue
}
}
diff --git a/backend/internal/event/service_test.go b/backend/internal/event/service_test.go
index a3a2e6f26..e2a1a374f 100644
--- a/backend/internal/event/service_test.go
+++ b/backend/internal/event/service_test.go
@@ -843,7 +843,7 @@ func TestDaemonEventWriterBatchesAndFlushesOnStop(t *testing.T) {
svc.RecordDockerEvent(t.Context(), events.Message{Type: events.ContainerEventType, Action: events.ActionStart, TimeNano: timeNano, Actor: events.Actor{ID: "web"}})
}
- const occurrences = daemonEventBatchSize*3 + 5
+ const occurrences = DaemonEventBatchSize*3 + 5
var wg sync.WaitGroup
for occurrence := range int64(occurrences) {
for range 2 {
diff --git a/backend/internal/middleware/environment.go b/backend/internal/middleware/environment.go
index e40296d4a..1f09b2374 100644
--- a/backend/internal/middleware/environment.go
+++ b/backend/internal/middleware/environment.go
@@ -11,6 +11,7 @@ import (
"log/slog"
"mime"
"mime/multipart"
+ "net"
"net/http"
"os"
"path"
@@ -37,14 +38,15 @@ const (
apiEnvironmentsPrefix = "/api/environments/"
environmentsPathMarker = "/environments/"
- // proxyTimeout is generous because proxied operations such as image pulls can stream for minutes.
- proxyTimeout = 30 * time.Minute
proxyDialTimeout = 5 * time.Second
maxProxiedWorkspaceManifestBytes = 1024 * 1024
+ maxProxiedWorkspacePrefixBytes = 16 * 1024 * 1024
maxProxiedGitOpsSyncBodyBytes = 1024 * 1024
)
+var errProxiedBodyTooLarge = errors.New("proxied request body is too large")
+
// managementEndpointSet contains paths handled locally and never proxied to remote environments.
var managementEndpointSet = map[string]struct{}{
"/test": {},
@@ -95,7 +97,11 @@ func NewEnvProxyMiddlewareWithParamAndRegistry(
}
// Traced so direct agents continue the manager's trace.
- httpClient := httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: proxyTimeout, TLSHandshakeTimeout: 10 * time.Second, DialTimeout: proxyDialTimeout})
+ httpClient := httpx.NewHTTPClient(httpxtypes.ClientOptions{
+ TLSHandshakeTimeout: 10 * time.Second,
+ DialTimeout: proxyDialTimeout,
+ ResponseHeaderTimeout: edge.ProxyResponseHeaderTimeout,
+ })
httpClient.Transport = otelhttp.NewTransport(httpClient.Transport)
m := &EnvironmentMiddleware{
@@ -175,13 +181,23 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e
}
// SECURITY: Remote agents trust the manager as sudo, so per-environment authorization happens here.
- if m.proxyPermissionDenied(c, perms, envID, suffix) {
- return writeProxyError(c, http.StatusForbidden, "You don't have permission to perform this action on this environment")
+ status := m.proxyPermissionStatus(c, perms, envID, suffix)
+ if replay, isReplay := req.Body.(*proxiedReplayBody); isReplay {
+ defer func() { _ = replay.Close() }()
+ }
+ switch status {
+ case http.StatusOK:
+ case http.StatusForbidden:
+ return writeProxyError(c, status, "You don't have permission to perform this action on this environment")
+ case http.StatusRequestEntityTooLarge:
+ return writeProxyError(c, status, "Request body is too large")
+ default:
+ return writeProxyError(c, status, "Invalid request body")
}
tunnel, unavailable := m.activeTunnel(ctx, envID, apiURL)
if unavailable {
- return writeProxyError(c, http.StatusBadGateway, "Edge agent is not connected")
+ return writeProxyError(c, http.StatusServiceUnavailable, "Edge agent is not connected")
}
proxyPath := path.Join(apiEnvironmentsPrefix, m.localID) + suffix
@@ -200,7 +216,7 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e
}
// A multiplexed tunnel carries plain HTTP, so it shares the direct-agent path below.
apiURL = edge.MuxAgentBaseURL
- client = &http.Client{Transport: muxClient.Transport, Timeout: m.httpClient.Timeout}
+ client = muxClient
}
target := strings.TrimRight(apiURL, "/") + proxyPath
@@ -249,7 +265,8 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e
resp, err := client.Do(proxyReq)
if err != nil {
- return writeProxyError(c, http.StatusBadGateway, "Proxy request failed: "+err.Error())
+ netErr, isNetErr := errors.AsType[net.Error](err)
+ return writeProxyError(c, kit.Ternary(isNetErr && netErr.Timeout(), http.StatusGatewayTimeout, http.StatusBadGateway), "Proxy request failed: "+err.Error())
}
defer func() { _ = resp.Body.Close() }()
@@ -272,7 +289,7 @@ func (m *EnvironmentMiddleware) Handle(c *echo.Context, next echo.HandlerFunc) e
}
// activeTunnel returns the environment's live tunnel, if any. Edge environments refresh poll demand on every
-// request and wait briefly for the agent; unavailable reports an edge agent that never connected.
+// request and wait briefly only for an agent with a fresh poll check-in; unavailable reports an offline edge agent.
func (m *EnvironmentMiddleware) activeTunnel(ctx context.Context, envID, apiURL string) (tunnel *edge.AgentTunnel, unavailable bool) {
if !strings.HasPrefix(strings.ToLower(strings.TrimSpace(apiURL)), "edge://") {
tunnel, _ = m.registry.Active(envID).Get()
@@ -282,6 +299,10 @@ func (m *EnvironmentMiddleware) activeTunnel(ctx context.Context, envID, apiURL
if active, ok := m.registry.Active(envID).Get(); ok {
return active, false
}
+ if edge.GetPollRuntimeRegistry().Get(envID, time.Now()).IsAbsent() {
+ slog.WarnContext(ctx, "No active edge tunnel or recent poll check-in for environment", "environmentId", envID)
+ return nil, true
+ }
tunnel, ok := m.registry.WaitForActive(ctx, envID, edge.DefaultTunnelAcquireTimeout()).Get()
if !ok {
slog.WarnContext(ctx, "No active edge tunnel for environment", "environmentId", envID)
@@ -291,11 +312,11 @@ func (m *EnvironmentMiddleware) activeTunnel(ctx context.Context, envID, apiURL
return tunnel, false
}
-// proxyPermissionDenied reports whether the caller lacks permission for the proxied request, mirroring local
+// proxyPermissionStatus returns http.StatusOK when the caller may perform the proxied request, mirroring local
// RequirePermission checks. Sudo bypasses it; routes without a permission mapping are denied by default.
-func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz.PermissionSet, envID, suffix string) bool {
+func (m *EnvironmentMiddleware) proxyPermissionStatus(c *echo.Context, ps *authz.PermissionSet, envID, suffix string) int {
if m.matcher == nil || (ps != nil && ps.Sudo) {
- return false
+ return http.StatusOK
}
req := c.Request()
@@ -321,22 +342,22 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz
slog.DebugContext(ctx, "Denying proxied upload session request: permission denied",
"method", method, "path", suffix, "kind", segments[1], "environmentId", envID)
}
- return denied
+ return kit.Ternary(denied, http.StatusForbidden, http.StatusOK)
}
}
perm, ok := m.matcher.Lookup(method, suffix).Get()
if !ok {
slog.WarnContext(ctx, "Denying proxied request with no known permission mapping", "method", method, "path", suffix, "environmentId", envID)
- return true
+ return http.StatusForbidden
}
if perm == "" {
// Explicitly public route: allowed for any authenticated caller.
- return false
+ return http.StatusOK
}
if !ps.Allows(perm, kit.Ternary(authz.IsEnvScoped(perm), envID, "")) {
slog.DebugContext(ctx, "Denying proxied request: permission denied", "method", method, "path", suffix, "permission", perm, "environmentId", envID)
- return true
+ return http.StatusForbidden
}
var required []string
@@ -347,8 +368,8 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz
required, err = proxiedGitOpsSyncPermissions(req, method, segments)
}
if err != nil {
- slog.DebugContext(ctx, "Denying proxied request with invalid body", "path", suffix, "environmentId", envID, "error", err)
- return true
+ slog.DebugContext(ctx, "Rejecting proxied request with invalid body", "path", suffix, "environmentId", envID, "error", err)
+ return kit.Ternary(errors.Is(err, errProxiedBodyTooLarge), http.StatusRequestEntityTooLarge, http.StatusBadRequest)
}
// Resource-sorted container lists collect per-container stats, so they also need containers:read.
if method == http.MethodGet && len(segments) == 1 && segments[0] == "containers" && container.IsResourceSort(req.URL.Query().Get("sort")) {
@@ -358,10 +379,10 @@ func (m *EnvironmentMiddleware) proxyPermissionDenied(c *echo.Context, ps *authz
if !ps.Allows(operationPermission, envID) {
slog.DebugContext(ctx, "Denying proxied request: body-derived permission denied",
"path", suffix, "permission", operationPermission, "environmentId", envID)
- return true
+ return http.StatusForbidden
}
}
- return false
+ return http.StatusOK
}
func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error) {
@@ -384,9 +405,13 @@ func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error)
request.Body = &proxiedReplayBody{Reader: io.MultiReader(captured, originalBody), captured: captured, original: originalBody}
}()
- reader := multipart.NewReader(io.TeeReader(originalBody, captured), params["boundary"])
+ prefix := &io.LimitedReader{R: originalBody, N: maxProxiedWorkspacePrefixBytes}
+ reader := multipart.NewReader(io.TeeReader(prefix, captured), params["boundary"])
for {
part, nextErr := reader.NextPart()
+ if nextErr != nil && prefix.N == 0 {
+ return nil, fmt.Errorf("find workspace manifest part: %w", errProxiedBodyTooLarge)
+ }
if nextErr != nil {
return nil, fmt.Errorf("find workspace manifest part: %w", nextErr)
}
@@ -398,6 +423,9 @@ func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error)
}
manifestJSON, readErr := io.ReadAll(io.LimitReader(part, maxProxiedWorkspaceManifestBytes+1))
closeErr := part.Close()
+ if readErr != nil && prefix.N == 0 {
+ return nil, fmt.Errorf("read workspace manifest: %w", errProxiedBodyTooLarge)
+ }
if readErr != nil {
return nil, fmt.Errorf("read workspace manifest: %w", readErr)
}
@@ -405,7 +433,7 @@ func proxiedVolumeWorkspacePermissions(request *http.Request) ([]string, error)
return nil, fmt.Errorf("close workspace manifest part: %w", closeErr)
}
if len(manifestJSON) > maxProxiedWorkspaceManifestBytes {
- return nil, errors.New("workspace manifest is too large")
+ return nil, fmt.Errorf("workspace manifest: %w", errProxiedBodyTooLarge)
}
var manifest volume.WorkspaceUpdateManifest
if unmarshalErr := json.Unmarshal(manifestJSON, &manifest); unmarshalErr != nil {
@@ -435,7 +463,7 @@ func proxiedGitOpsSyncPermissions(request *http.Request, method string, segments
return nil, fmt.Errorf("read gitops sync body: %w", err)
}
if len(body) > maxProxiedGitOpsSyncBodyBytes {
- return nil, errors.New("gitops sync body is too large")
+ return nil, fmt.Errorf("gitops sync body: %w", errProxiedBodyTooLarge)
}
request.Body = io.NopCloser(bytes.NewReader(body))
diff --git a/backend/internal/middleware/environment_test.go b/backend/internal/middleware/environment_test.go
index 08985490c..51f1c54bc 100644
--- a/backend/internal/middleware/environment_test.go
+++ b/backend/internal/middleware/environment_test.go
@@ -34,7 +34,7 @@ func newTestEnvironmentMiddleware() *EnvironmentMiddleware {
_ = c
return authz.SudoPermissionSet(), nil, true
},
- httpClient: &http.Client{Timeout: proxyTimeout},
+ httpClient: &http.Client{},
registry: edge.NewTunnelRegistry(),
}
}
@@ -49,7 +49,7 @@ func attachMiddleware(router *echo.Echo, mw *EnvironmentMiddleware) *echo.Group
return api
}
-func TestEnvironmentMiddleware_ReturnsBadGatewayForEdgeResourcesWithoutTunnel(t *testing.T) {
+func TestEnvironmentMiddleware_ReturnsServiceUnavailableForEdgeResourcesWithoutTunnel(t *testing.T) {
middleware := newTestEnvironmentMiddleware()
router := echo.New()
api := attachMiddleware(router, middleware)
@@ -65,7 +65,7 @@ func TestEnvironmentMiddleware_ReturnsBadGatewayForEdgeResourcesWithoutTunnel(t
router.ServeHTTP(recorder, req)
- assert.Equal(t, http.StatusBadGateway, recorder.Code)
+ assert.Equal(t, http.StatusServiceUnavailable, recorder.Code)
assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")
assert.False(t, localHandlerHit)
}
@@ -86,7 +86,7 @@ func TestEnvironmentMiddleware_ProxiesDashboardResourcesForRemoteEnvironments(t
router.ServeHTTP(recorder, req)
- assert.Equal(t, http.StatusBadGateway, recorder.Code)
+ assert.Equal(t, http.StatusServiceUnavailable, recorder.Code)
assert.Contains(t, recorder.Body.String(), "Edge agent is not connected")
assert.False(t, localHandlerHit)
}
@@ -257,7 +257,7 @@ func TestEnvironmentMiddleware_LocalEnvironmentSkipsProxyPermissionCheck(t *test
_ = c
return authz.NewPermissionSet(), nil, true
},
- httpClient: &http.Client{Timeout: proxyTimeout},
+ httpClient: &http.Client{},
registry: edge.NewTunnelRegistry(),
}
@@ -357,7 +357,7 @@ func TestEnvironmentMiddleware_ForwardsResolvedIconCatalogHeaderOnly(t *testing.
u.Preferences.IconCatalog = tt.catalog
return authz.SudoPermissionSet(), u, true
},
- httpClient: &http.Client{Timeout: proxyTimeout},
+ httpClient: &http.Client{},
registry: edge.NewTunnelRegistry(),
}
@@ -428,7 +428,7 @@ func TestProxyPermissionDeniedBlocksWriteForReadOnlyUser(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers/abc/restart"),
"expected restart to be denied for a read-only user")
}
@@ -439,7 +439,7 @@ func TestProxyPermissionDeniedAllowsWriteForPermittedUser(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers/abc/restart"),
"expected restart to be allowed for a user with containers:restart")
}
@@ -450,7 +450,7 @@ func TestProxyPermissionDeniedAllowsRead(t *testing.T) {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers"),
"expected list to be allowed for a user with containers:list")
}
@@ -462,7 +462,7 @@ func TestProxyPermissionDeniedDeniesPermissionFromDifferentEnv(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/containers/abc/restart"),
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/containers/abc/restart"),
"expected denial: permission is scoped to a different environment")
}
@@ -470,7 +470,7 @@ func TestProxyPermissionDeniedSudoBypasses(t *testing.T) {
m := newProxyAuthzMiddleware(containerMatcher())
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/containers/abc/restart")
- require.False(t, m.proxyPermissionDenied(c, authz.SudoPermissionSet(), proxyTestEnvID, "/containers/abc/restart"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, authz.SudoPermissionSet(), proxyTestEnvID, "/containers/abc/restart"),
"expected sudo permission set to bypass the permission check")
}
@@ -481,7 +481,7 @@ func TestProxyPermissionDeniedDefaultDeniesUnmappedRoute(t *testing.T) {
c := newProxyRequestContext(http.MethodPost, "/api/environments/"+proxyTestEnvID+"/unknown/resource")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/unknown/resource"),
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/unknown/resource"),
"expected an unmapped proxied route to be denied by default")
}
@@ -491,7 +491,7 @@ func TestProxyPermissionDeniedAllowsPublicRoute(t *testing.T) {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/settings/public")
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/settings/public"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/settings/public"),
"expected an explicitly public route to be allowed for any authenticated caller")
}
@@ -530,23 +530,23 @@ func TestProxyPermissionDeniedChecksVolumeWorkspaceManifestPermissions(t *testin
readOnly := authz.NewPermissionSet()
readOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead)
c, _ := newProxyVolumeWorkspaceContext(t, changes)
- require.True(t, m.proxyPermissionDenied(c, readOnly, proxyTestEnvID, "/volumes/data/workspace"))
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, readOnly, proxyTestEnvID, "/volumes/data/workspace"))
uploadOnly := authz.NewPermissionSet()
uploadOnly.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload)
c, _ = newProxyVolumeWorkspaceContext(t, changes)
- require.True(t, m.proxyPermissionDenied(c, uploadOnly, proxyTestEnvID, "/volumes/data/workspace"))
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, uploadOnly, proxyTestEnvID, "/volumes/data/workspace"))
permitted := authz.NewPermissionSet()
permitted.AddEnv(proxyTestEnvID, authz.PermVolumesRead, authz.PermVolumesUpload, authz.PermVolumesDelete)
c, rawBody := newProxyVolumeWorkspaceContext(t, changes)
- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))
replayed, err := io.ReadAll(c.Request().Body)
require.NoError(t, err)
require.Equal(t, rawBody, replayed)
c, rawBody = newProxyVolumeWorkspaceContext(t, changes, true)
- require.False(t, m.proxyPermissionDenied(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, permitted, proxyTestEnvID, "/volumes/data/workspace"))
replayed, err = io.ReadAll(c.Request().Body)
require.NoError(t, err)
require.Equal(t, rawBody, replayed)
@@ -568,13 +568,13 @@ func TestProxyPermissionDeniedChecksGitOpsImportLifecyclePermission(t *testing.T
createOnly := authz.NewPermissionSet()
createOnly.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate)
- require.True(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))
- require.False(t, m.proxyPermissionDenied(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(newProxyGitOpsImportContext(hookBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(newProxyGitOpsImportContext(plainBody), createOnly, proxyTestEnvID, "/gitops-syncs/import"))
withLifecycle := authz.NewPermissionSet()
withLifecycle.AddEnv(proxyTestEnvID, authz.PermGitOpsCreate, authz.PermGitOpsLifecycle)
c := newProxyGitOpsImportContext(hookBody)
- require.False(t, m.proxyPermissionDenied(c, withLifecycle, proxyTestEnvID, "/gitops-syncs/import"))
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, withLifecycle, proxyTestEnvID, "/gitops-syncs/import"))
replayed, err := io.ReadAll(c.Request().Body)
require.NoError(t, err)
require.Equal(t, hookBody, string(replayed))
@@ -592,13 +592,13 @@ func TestProxyPermissionDeniedWSTerminalRequiresExec(t *testing.T) {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/ws/containers/abc/terminal")
- require.True(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),
"expected WS terminal to be denied without containers:exec")
// Granting containers:exec allows the same stream.
ps.AddEnv(proxyTestEnvID, authz.PermContainersExec)
- require.False(t, m.proxyPermissionDenied(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, ps, proxyTestEnvID, "/ws/containers/abc/terminal"),
"expected WS terminal to be allowed with containers:exec")
}
@@ -610,22 +610,22 @@ func TestProxyPermissionDeniedResourceSortRequiresRead(t *testing.T) {
for _, sort := range []string{"cpuUsage", "memoryUsage"} {
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort="+sort)
- require.True(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),
+ require.Equal(t, http.StatusForbidden, m.proxyPermissionStatus(c, listOnly, proxyTestEnvID, "/containers"),
"expected resource sort %q to be denied with only containers:list", sort)
}
c := newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=name")
- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, listOnly, proxyTestEnvID, "/containers"),
"expected non-resource sorts to stay allowed with containers:list")
c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers")
- require.False(t, m.proxyPermissionDenied(c, listOnly, proxyTestEnvID, "/containers"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, listOnly, proxyTestEnvID, "/containers"),
"expected the plain list to stay allowed with containers:list")
withRead := authz.NewPermissionSet()
withRead.AddEnv(proxyTestEnvID, authz.PermContainersList, authz.PermContainersRead)
c = newProxyRequestContext(http.MethodGet, "/api/environments/"+proxyTestEnvID+"/containers?sort=memoryUsage&order=desc")
- require.False(t, m.proxyPermissionDenied(c, withRead, proxyTestEnvID, "/containers"),
+ require.Equal(t, http.StatusOK, m.proxyPermissionStatus(c, withRead, proxyTestEnvID, "/containers"),
"expected resource sort to be allowed with containers:list and containers:read")
}
diff --git a/backend/internal/middleware/ratelimit.go b/backend/internal/middleware/ratelimit.go
index 44f10e0f7..b59d83ba5 100644
--- a/backend/internal/middleware/ratelimit.go
+++ b/backend/internal/middleware/ratelimit.go
@@ -164,6 +164,7 @@ func PerTokenRateLimitForPaths(
paths []string,
perMinute int,
burst int,
+ tokenOf func(c *echo.Context) string,
isKnown func(token string) bool,
) echo.MiddlewareFunc {
if perMinute <= 0 {
@@ -186,7 +187,7 @@ func PerTokenRateLimitForPaths(
return next(c)
}
- token := c.Param("token")
+ token := tokenOf(c)
if token != "" && isKnown != nil && isKnown(token) {
if !tokenLimiter.allow(kit.SHA256Hex(token)) {
c.Response().Header().Set("Retry-After", "60")
diff --git a/backend/internal/middleware/ratelimit_test.go b/backend/internal/middleware/ratelimit_test.go
index 4545d59c7..9a123f732 100644
--- a/backend/internal/middleware/ratelimit_test.go
+++ b/backend/internal/middleware/ratelimit_test.go
@@ -4,6 +4,7 @@ import (
"net/http"
"net/http/httptest"
"strconv"
+ "strings"
"testing"
"time"
@@ -175,7 +176,7 @@ func newTokenRateLimitTestRouterInternal(
router := echo.New()
router.IPExtractor = echo.ExtractIPDirect()
- router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, perMinute, burst, isAuthentic))
+ router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, perMinute, burst, func(c *echo.Context) string { return c.Param("token") }, isAuthentic))
router.POST("/webhooks/trigger/:token", handler)
return func(token string) int {
@@ -273,7 +274,7 @@ func TestPerTokenRateLimitForPaths_AppliesOnlyToConfiguredPaths(t *testing.T) {
router := echo.New()
router.IPExtractor = echo.ExtractIPDirect()
- router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, 60, 1, nil))
+ router.Use(PerTokenRateLimitForPaths([]string{"/webhooks/trigger/:token"}, 60, 1, func(c *echo.Context) string { return c.Param("token") }, nil))
router.POST("/webhooks/trigger/:token", func(c *echo.Context) error {
return c.NoContent(http.StatusOK)
})
@@ -328,3 +329,29 @@ func TestIPRateLimiter_ProtectsCurrentKeyDuringSweep(t *testing.T) {
require.Contains(t, limiter.limiters, "current")
require.NotContains(t, limiter.limiters, "other")
}
+
+func TestPerTokenRateLimitForPaths_KeysByAgentTokenHeader(t *testing.T) {
+ router := echo.New()
+ router.IPExtractor = echo.ExtractIPDirect()
+ router.Use(PerTokenRateLimitForPaths(
+ []string{"/events"}, 60, 1,
+ func(c *echo.Context) string { return c.Request().Header.Get(HeaderAgentToken) },
+ func(token string) bool { return strings.HasPrefix(token, "known-") },
+ ))
+ router.POST("/events", okHandlerInternal)
+ doReq := func(token string) int {
+ req := httptest.NewRequest(http.MethodPost, "/events", http.NoBody)
+ req.RemoteAddr = "192.0.2.10:4000"
+ req.Header.Set(HeaderAgentToken, token)
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+ return rec.Code
+ }
+
+ require.Equal(t, http.StatusOK, doReq("known-a"))
+ require.Equal(t, http.StatusOK, doReq("known-b"))
+ require.Equal(t, http.StatusTooManyRequests, doReq("known-a"))
+ require.Equal(t, http.StatusOK, doReq("unknown-a"))
+ require.Equal(t, http.StatusTooManyRequests, doReq("unknown-b"))
+ require.Equal(t, http.StatusTooManyRequests, doReq(""))
+}
diff --git a/backend/internal/registry/handler.go b/backend/internal/registry/handler.go
index 6f1d9f60f..5b4f5eb32 100644
--- a/backend/internal/registry/handler.go
+++ b/backend/internal/registry/handler.go
@@ -18,7 +18,7 @@ import (
// ContainerRegistryHandler handles container registry management endpoints.
type ContainerRegistryHandler struct {
registryService *ContainerRegistryService
- syncRemoteRegistries func(context.Context) error
+ syncRemoteRegistries func(context.Context)
}
type ListContainerRegistriesInput struct {
@@ -55,7 +55,7 @@ type SyncContainerRegistriesInput struct {
}
// NewHandler builds the container registry HTTP handler.
-func NewHandler(registryService *ContainerRegistryService, syncRemoteRegistries func(context.Context) error) *ContainerRegistryHandler {
+func NewHandler(registryService *ContainerRegistryService, syncRemoteRegistries func(context.Context)) *ContainerRegistryHandler {
return &ContainerRegistryHandler{registryService: registryService, syncRemoteRegistries: syncRemoteRegistries}
}
@@ -207,9 +207,6 @@ func (h *ContainerRegistryHandler) triggerRemoteRegistrySync(ctx context.Context
return
}
- go func(syncCtx context.Context, syncReason string) {
- if err := h.syncRemoteRegistries(syncCtx); err != nil {
- slog.WarnContext(syncCtx, "Failed to fan out registry sync to remote environments", "reason", syncReason, "error", err.Error())
- }
- }(context.WithoutCancel(ctx), reason)
+ slog.DebugContext(ctx, "Queueing registry sync to remote environments", "reason", reason)
+ h.syncRemoteRegistries(ctx)
}
diff --git a/backend/internal/registry/module.go b/backend/internal/registry/module.go
index 5f0567b4c..ab9ddde95 100644
--- a/backend/internal/registry/module.go
+++ b/backend/internal/registry/module.go
@@ -17,7 +17,7 @@ type Module struct {
browseHandler *browse.Handler
}
-func New(service *ContainerRegistryService, syncRemoteRegistries func(context.Context) error) *Module {
+func New(service *ContainerRegistryService, syncRemoteRegistries func(context.Context)) *Module {
return &Module{
service: service,
handler: NewHandler(service, syncRemoteRegistries),
diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.go
index 1bde751ba..efcbb9b48 100644
--- a/backend/internal/s3/handler.go
+++ b/backend/internal/s3/handler.go
@@ -13,15 +13,14 @@ import (
)
const (
- s3DestinationPath = "/backups/s3"
- s3DestinationTag = "S3 Destinations"
- s3ConnectionTestSuccess = "S3 connection test succeeded"
- s3RemoteSyncFailureMessage = "Failed to fan out S3 destination sync to remote environments"
+ s3DestinationPath = "/backups/s3"
+ s3DestinationTag = "S3 Destinations"
+ s3ConnectionTestSuccess = "S3 connection test succeeded"
)
type s3DestinationHandler struct {
service *S3DestinationService
- syncRemoteDestinations func(context.Context) error
+ syncRemoteDestinations func(context.Context)
}
type listS3DestinationsInput struct {
@@ -168,11 +167,8 @@ func (h *s3DestinationHandler) test(ctx context.Context, input *testS3Destinatio
}
func (h *s3DestinationHandler) triggerRemoteSync(ctx context.Context, reason string) {
- go func(syncCtx context.Context, syncReason string) {
- if err := h.syncRemoteDestinations(syncCtx); err != nil {
- slog.WarnContext(syncCtx, s3RemoteSyncFailureMessage, "reason", syncReason, "error", err.Error())
- }
- }(context.WithoutCancel(ctx), reason)
+ slog.DebugContext(ctx, "Queueing S3 destination sync to remote environments", "reason", reason)
+ h.syncRemoteDestinations(ctx)
}
func (h *s3DestinationHandler) testConfiguration(ctx context.Context, input *testS3DestinationConfigurationInput) (*handlerutil.Out[base.MessageResponse], error) {
diff --git a/backend/internal/s3/module.go b/backend/internal/s3/module.go
index 827344e55..0b5544863 100644
--- a/backend/internal/s3/module.go
+++ b/backend/internal/s3/module.go
@@ -13,10 +13,10 @@ import (
type Module struct {
service *S3DestinationService
- syncRemoteDestinations func(context.Context) error
+ syncRemoteDestinations func(context.Context)
}
-func New(service *S3DestinationService, syncRemoteDestinations func(context.Context) error) *Module {
+func New(service *S3DestinationService, syncRemoteDestinations func(context.Context)) *Module {
return &Module{service: service, syncRemoteDestinations: syncRemoteDestinations}
}
@@ -35,7 +35,7 @@ func (m *Module) RegisterRoutes(api huma.API) {
RegisterS3Destinations(api, m.service, m.syncRemoteDestinations)
}
-func RegisterS3Destinations(api huma.API, service *S3DestinationService, syncRemoteDestinations func(context.Context) error) {
+func RegisterS3Destinations(api huma.API, service *S3DestinationService, syncRemoteDestinations func(context.Context)) {
handler := &s3DestinationHandler{service: service, syncRemoteDestinations: syncRemoteDestinations}
handlerutil.RegisterSecured(api,
diff --git a/backend/internal/variable/service.go b/backend/internal/variable/service.go
index c78d06364..fe5b7a74f 100644
--- a/backend/internal/variable/service.go
+++ b/backend/internal/variable/service.go
@@ -30,6 +30,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
@@ -50,11 +51,12 @@ const (
// .env.global file: locally via template.TemplateService, remotely via the existing
// agent variables endpoint.
type VariableService struct {
- db *database.DB
- proxyEnvironmentJSON func(context.Context, string, string, string, []byte, any) error
- listEnabledEnvironmentIDs func(context.Context) ([]string, error)
- settingsService *settings.SettingsService
- kvService *kv.KVService
+ db *database.DB
+ proxyEnvironmentJSON func(context.Context, string, string, string, []byte, any) error
+ listEnabledRemoteEnvironments func(context.Context) ([]environment.Environment, error)
+ settingsService *settings.SettingsService
+ kvService *kv.KVService
+ importLocks utils.KeyedMutex
statusMu sync.RWMutex
syncStatus map[string]env.EnvironmentSyncStatus
@@ -73,7 +75,7 @@ func NewVariableService(db *database.DB, environmentService *environment.Environ
}
if environmentService != nil {
service.proxyEnvironmentJSON = environmentService.ProxyJSONRequest
- service.listEnabledEnvironmentIDs = environmentService.ListEnabledEnvironmentIDs
+ service.listEnabledRemoteEnvironments = environmentService.ListEnabledRemoteEnvironments
}
return service
}
@@ -319,6 +321,9 @@ func (s *VariableService) SyncEnvironment(ctx context.Context, envID string) err
// ForgetSyncState makes the next SyncEnvironment resend regardless of change.
func (s *VariableService) ForgetSyncState(envID string) {
s.syncGate.Forget(envID)
+ s.statusMu.Lock()
+ delete(s.syncStatus, envID)
+ s.statusMu.Unlock()
}
func (s *VariableService) syncEnvironmentInternal(ctx context.Context, envID string) error {
@@ -359,16 +364,15 @@ func (s *VariableService) syncEnvironmentInternal(ctx context.Context, envID str
// syncTargetsInternal returns the local environment plus every enabled
// environment, deduplicated.
-func (s *VariableService) syncTargetsInternal(ctx context.Context) []string {
- envIDs, err := s.listEnabledEnvironmentIDs(ctx)
+func (s *VariableService) syncTargetsInternal(ctx context.Context) []environment.Environment {
+ remotes, err := s.listEnabledRemoteEnvironments(ctx)
if err != nil {
slog.ErrorContext(ctx, "Failed to list environments for global variable sync", "error", err)
- envIDs = nil
- }
- if !slices.Contains(envIDs, environment.LocalEnvironmentID) {
- envIDs = append([]string{environment.LocalEnvironmentID}, envIDs...)
+ remotes = nil
}
- return envIDs
+ return append([]environment.Environment{{ID: environment.LocalEnvironmentID}}, slices.DeleteFunc(remotes, func(remote environment.Environment) bool {
+ return strings.TrimSpace(kit.FromPtr(remote.AccessToken)) == ""
+ })...)
}
// SyncAll pushes the effective variable set to the local environment and every
@@ -378,8 +382,8 @@ func (s *VariableService) SyncAll(ctx context.Context) []env.EnvironmentSyncStat
targets := s.syncTargetsInternal(ctx)
// An explicit sync exists to repair agents, so every target is resent even
// when the manager's payload has not changed.
- for _, envID := range targets {
- s.ForgetSyncState(envID)
+ for _, target := range targets {
+ s.ForgetSyncState(target.ID)
}
s.syncEnvironmentsInternal(ctx, targets)
return s.SyncStatuses()
@@ -394,25 +398,33 @@ func (s *VariableService) SyncAllBackground(ctx context.Context) []env.Environme
slog.WarnContext(ctx, "Failed to sync global variables locally", "error", err)
}
- remoteIDs := slices.DeleteFunc(s.syncTargetsInternal(ctx), func(id string) bool { return id == environment.LocalEnvironmentID })
- for _, envID := range remoteIDs {
- s.recordSyncPendingInternal(envID)
+ remotes := slices.DeleteFunc(s.syncTargetsInternal(ctx), func(target environment.Environment) bool {
+ return target.ID == environment.LocalEnvironmentID
+ })
+ for _, target := range remotes {
+ s.recordSyncPendingInternal(target.ID)
}
bgCtx := context.WithoutCancel(ctx)
- go s.syncEnvironmentsInternal(bgCtx, remoteIDs)
+ go s.syncEnvironmentsInternal(bgCtx, remotes)
return s.SyncStatuses()
}
-func (s *VariableService) syncEnvironmentsInternal(ctx context.Context, envIDs []string) {
+func (s *VariableService) syncEnvironmentsInternal(ctx context.Context, targets []environment.Environment) {
var wg sync.WaitGroup
- for _, envID := range envIDs {
+ for _, target := range targets {
wg.Go(func() {
syncCtx, cancel := context.WithTimeout(ctx, variableSyncTimeout)
defer cancel()
- if err := s.SyncEnvironment(syncCtx, envID); err != nil {
- slog.WarnContext(syncCtx, "Failed to sync global variables to environment", "environmentId", envID, "error", err)
+ if target.IsEdge && !edge.HasActiveTunnel(target.ID) {
+ if _, ok := edge.RequestTunnelAndWait(syncCtx, target.ID, edge.DefaultTunnelDemandTTL, edge.DefaultTunnelAcquireTimeout()).Get(); !ok {
+ s.recordSyncPendingInternal(target.ID)
+ return
+ }
+ }
+ if err := s.SyncEnvironment(syncCtx, target.ID); err != nil {
+ slog.WarnContext(syncCtx, "Failed to sync global variables to environment", "environmentId", target.ID, "error", err)
}
})
}
@@ -611,6 +623,12 @@ func (s *VariableService) ImportLegacyLocalEnvFile(ctx context.Context) error {
// would not already produce for that environment as an env-scoped variable.
// An import failure aborts the push so existing agent data is never wiped.
func (s *VariableService) importRemoteLegacyVarsOnceInternal(ctx context.Context, envID string) error {
+ unlock, err := s.importLocks.LockContext(ctx, envID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
flag := globalVariablesImportedKVPrefix + envID
imported, err := s.kvService.GetBool(ctx, flag, false)
if err != nil {
diff --git a/backend/internal/variable/service_test.go b/backend/internal/variable/service_test.go
index 30c4a9fae..918c52201 100644
--- a/backend/internal/variable/service_test.go
+++ b/backend/internal/variable/service_test.go
@@ -301,7 +301,6 @@ func TestSyncEnvironment_DirectMaterializesSecretsThroughAgentRoute(t *testing.T
}).Error)
environmentService := environment.NewEnvironmentService(db, nil, nil, nil, service.settingsService, nil)
service.proxyEnvironmentJSON = environmentService.ProxyJSONRequest
- service.listEnabledEnvironmentIDs = environmentService.ListEnabledEnvironmentIDs
createMaterializedSecretVariableInternal(t, service)
require.NoError(t, service.SyncEnvironment(t.Context(), "env-direct"))
diff --git a/backend/internal/webhook/service_test.go b/backend/internal/webhook/service_test.go
index 94dd7fba9..7c323689c 100644
--- a/backend/internal/webhook/service_test.go
+++ b/backend/internal/webhook/service_test.go
@@ -179,7 +179,7 @@ func TestKnownTokenRateLimit_ExhaustedIPDoesNotBlockFirstUse(t *testing.T) {
require.NoError(t, sqlDB.Close())
router := echo.New()
router.IPExtractor = echo.ExtractIPDirect()
- router.Use(middleware.PerTokenRateLimitForPaths([]string{"/trigger/:token"}, 1, 1, svc.IsKnownToken))
+ router.Use(middleware.PerTokenRateLimitForPaths([]string{"/trigger/:token"}, 1, 1, func(c *echo.Context) string { return c.Param("token") }, svc.IsKnownToken))
router.POST("/trigger/:token", func(c *echo.Context) error { return c.NoContent(http.StatusAccepted) })
request := func(token string) int {
req := httptest.NewRequest(http.MethodPost, "/trigger/"+token, http.NoBody)
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.go
index b68a71cca..0638a111a 100644
--- a/backend/pkg/libarcane/edge/client.go
+++ b/backend/pkg/libarcane/edge/client.go
@@ -10,6 +10,7 @@ import (
"log/slog"
"net"
"net/http"
+ "net/url"
"slices"
"strings"
"sync"
@@ -124,6 +125,13 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)
if errCh != nil {
defer close(errCh)
}
+ if configured := strings.ToLower(strings.TrimSpace(c.cfg.EdgeTransport)); configured != "" && NormalizeEdgeTransport(configured) != configured {
+ slog.WarnContext(ctx, "Unknown EDGE_TRANSPORT value; using the default transport", "configured", c.cfg.EdgeTransport, "transport", NormalizeEdgeTransport(configured))
+ }
+ if parsed, err := url.Parse(httpx.ManagerBaseURL(c.cfg.ManagerApiUrl)); err == nil && strings.Trim(parsed.Path, "/") != "" && c.managerGRPCAddr != "" {
+ slog.WarnContext(ctx, "MANAGER_API_URL has a path prefix that gRPC cannot route; gRPC edge tunnel transport is unavailable", "pathPrefix", parsed.Path)
+ c.managerGRPCAddr = ""
+ }
// Log only the endpoints this agent will dial.
poll := UsePollEdgeTransport(c.cfg)
attrs := []any{
@@ -147,7 +155,15 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)
reconnectBackoff := backoff.NewExponentialBackOff()
reconnectBackoff.InitialInterval = c.reconnectInterval
reconnectBackoff.MaxInterval = maxReconnectInterval
- serve := kit.Ternary(UsePollEdgeTransport(c.cfg), c.connectAndServePoll, c.connectAndServeManagedTunnel)
+ serve := func(ctx context.Context) error {
+ if err := EnsureAgentMTLSAssets(ctx, c.cfg); err != nil {
+ return err
+ }
+ if err := ValidateAgentMTLSConfig(c.cfg); err != nil {
+ return err
+ }
+ return kit.Ternary(poll, c.connectAndServePoll, c.connectAndServeManagedTunnel)(ctx)
+ }
for {
if ctx.Err() != nil {
@@ -448,7 +464,7 @@ func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, w
if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {
c.beginCommandRequestTransfer(ctx, conn, transferID, msg)
} else {
- workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })
+ c.startCommandRequest(ctx, conn, msg, workers)
}
case MessageTypeFileChunk:
c.handleCommandRequestChunk(ctx, msg, workers)
@@ -464,7 +480,12 @@ func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, w
}
}
case MessageTypeCancelRequest:
- slog.DebugContext(ctx, "Ignoring edge cancel request on agent", "id", msg.ID)
+ if value, ok := c.commandCancels.LoadAndDelete(msg.ID); ok {
+ if cancel, isCancel := value.(context.CancelFunc); isCancel {
+ cancel()
+ slog.DebugContext(ctx, "Cancelled edge command on manager request", "id", msg.ID)
+ }
+ }
case MessageTypeCommandCredit:
if value, ok := c.commandRecorders.Load(msg.ID); ok {
if recorder, isRecorder := value.(*commandResponseRecorder); isRecorder {
@@ -544,6 +565,13 @@ func (c *TunnelClient) handleCommandRequestChunk(ctx context.Context, msg *Tunne
return
}
+ if transfer.body.Len()+len(msg.Body) > maxLegacyCommandBodyBytes {
+ if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
+ transfer.stop()
+ c.sendCommandComplete(transfer.conn, transfer.request.ID, http.StatusRequestEntityTooLarge, "command body is too large for a legacy edge tunnel; upgrade the manager and agent")
+ }
+ return
+ }
_, _ = transfer.body.Write(msg.Body)
transfer.nextSequence++
if !msg.EOF {
@@ -552,11 +580,21 @@ func (c *TunnelClient) handleCommandRequestChunk(ctx context.Context, msg *Tunne
if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
transfer.stop()
- transfer.request.Body = append([]byte(nil), transfer.body.Bytes()...)
- workers.Go(func() { c.handleCommandRequest(ctx, transfer.conn, transfer.request) })
+ transfer.request.Body = transfer.body.Bytes()
+ c.startCommandRequest(ctx, transfer.conn, transfer.request, workers)
}
}
+func (c *TunnelClient) startCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {
+ commandCtx, cancel := context.WithCancel(ctx)
+ c.commandCancels.Store(msg.ID, cancel)
+ workers.Go(func() {
+ defer c.commandCancels.Delete(msg.ID)
+ defer cancel()
+ c.handleCommandRequest(commandCtx, conn, msg)
+ })
+}
+
// Deprecated: legacy edge message protocol; remove in v3.
func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
if !ValidateEdgeCommand(msg.Command, msg.Method, msg.Path, false) {
@@ -571,7 +609,7 @@ func (c *TunnelClient) handleCommandRequest(ctx context.Context, conn TunnelConn
return
}
- reqCtx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+ reqCtx, cancel := context.WithTimeout(ctx, kit.Ternary(msg.TimeoutMillis > 0, min(time.Duration(msg.TimeoutMillis)*time.Millisecond, c.requestTimeout), c.requestTimeout))
defer cancel()
req, err := c.buildLocalHTTPRequest(reqCtx, msg)
@@ -1094,12 +1132,11 @@ func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (
if cfg.AgentToken == "" {
return nil, errors.New("AGENT_TOKEN is required")
}
-
- if err := EnsureAgentMTLSAssets(ctx, cfg); err != nil {
- return nil, err
+ if parsed, err := url.Parse(httpx.ManagerBaseURL(cfg.ManagerApiUrl)); NormalizeEdgeTransport(cfg.EdgeTransport) == EdgeTransportGRPC && err == nil && strings.Trim(parsed.Path, "/") != "" {
+ return nil, errors.New("gRPC transport cannot use a MANAGER_API_URL with a path prefix; use EDGE_TRANSPORT=websocket or auto")
}
- if err := ValidateAgentMTLSConfig(cfg); err != nil {
- return nil, err
+ if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeDisabled && !managerUsesTLS(cfg) {
+ return nil, errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https")
}
client := NewTunnelClient(cfg, handler)
@@ -1136,8 +1173,8 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {
if err != nil {
if resp != nil && resp.Body != nil {
defer func() { _ = resp.Body.Close() }()
- body, _ := io.ReadAll(resp.Body)
- return fmt.Errorf("failed to connect to manager websocket endpoint (status: %d, body: %s): %w", resp.StatusCode, string(body), err)
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+ return fmt.Errorf("failed to connect to manager websocket endpoint (status: %d, body: %s): %w", resp.StatusCode, strings.TrimSpace(string(body)), err)
}
return fmt.Errorf("failed to connect to manager websocket endpoint: %w", err)
}
diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.go
index 9273fa0ec..fa18a6b75 100644
--- a/backend/pkg/libarcane/edge/client_test.go
+++ b/backend/pkg/libarcane/edge/client_test.go
@@ -942,18 +942,36 @@ func TestStartTunnelClient_GRPCValidation(t *testing.T) {
require.Error(t, err)
assert.Contains(t, err.Error(), "MANAGER_API_URL to use https")
})
+}
- t.Run("required mtls auto-enrollment failure surfaces", func(t *testing.T) {
- _, err := StartTunnelClient(ctx, &Config{
- EdgeAgent: true,
- EdgeTransport: EdgeTransportGRPC,
- ManagerApiUrl: "https://127.0.0.1:1/api",
- AgentToken: "token",
- EdgeMTLSMode: EdgeMTLSModeRequired,
- }, http.NotFoundHandler())
- require.Error(t, err)
- assert.Contains(t, err.Error(), "edge mTLS enrollment request failed")
- })
+func TestTunnelClient_StartWithErrorChan_RetriesMTLSEnrollmentEachAttempt(t *testing.T) {
+ client := NewTunnelClient(&Config{
+ EdgeAgent: true,
+ EdgeTransport: EdgeTransportGRPC,
+ ManagerApiUrl: "https://127.0.0.1:1/api",
+ AgentToken: "token",
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSAssetsDir: t.TempDir(),
+ }, http.NotFoundHandler())
+ client.reconnectInterval = 10 * time.Millisecond
+
+ ctx, cancel := context.WithCancel(t.Context())
+ errCh := make(chan error, 1)
+ go client.StartWithErrorChan(ctx, errCh)
+
+ for range 2 {
+ select {
+ case err := <-errCh:
+ require.ErrorContains(t, err, "edge mTLS enrollment request failed")
+ case <-time.After(5 * time.Second):
+ t.Fatal("timed out waiting for an mTLS enrollment attempt")
+ }
+ }
+ cancel()
+ require.Eventually(t, func() bool {
+ _, open := <-errCh
+ return !open
+ }, 5*time.Second, 10*time.Millisecond)
}
func TestTunnelClient_connectAndServeGRPC_EmptyManagerAddress(t *testing.T) {
diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.go
index 4fe269774..7e11f91ba 100644
--- a/backend/pkg/libarcane/edge/client_transport_poll.go
+++ b/backend/pkg/libarcane/edge/client_transport_poll.go
@@ -213,8 +213,8 @@ func (c *TunnelClient) pollTunnelControl(ctx context.Context, httpClient *http.C
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
- respBody, _ := io.ReadAll(resp.Body)
- return nil, fmt.Errorf("poll request failed with status %d: %s", resp.StatusCode, string(respBody))
+ respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+ return nil, fmt.Errorf("poll request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))
}
var pollResp TunnelPollResponse
diff --git a/backend/pkg/libarcane/edge/command_client.go b/backend/pkg/libarcane/edge/command_client.go
index 4bbec4e66..41a570a81 100644
--- a/backend/pkg/libarcane/edge/command_client.go
+++ b/backend/pkg/libarcane/edge/command_client.go
@@ -161,6 +161,11 @@ func (c *CommandClient) Execute(ctx context.Context, tunnel *AgentTunnel, req *C
// Release an agent still waiting to send output that nobody will read.
credit(math.MaxInt32)
}
+ if ctx.Err() != nil && slices.Contains(tunnel.Capabilities, tunnelCapabilityProtoParity) {
+ if cancelErr := tunnel.Conn.Send(&TunnelMessage{ID: requestID, Type: MessageTypeCancelRequest}); cancelErr != nil {
+ slog.DebugContext(context.WithoutCancel(ctx), "Failed to send edge command cancel", "id", requestID, "error", cancelErr)
+ }
+ }
return nil, err
}
diff --git a/backend/pkg/libarcane/edge/commands.go b/backend/pkg/libarcane/edge/commands.go
index 6f5462b78..55fcfe575 100644
--- a/backend/pkg/libarcane/edge/commands.go
+++ b/backend/pkg/libarcane/edge/commands.go
@@ -284,23 +284,28 @@ var commandRoutesIndex = buildCommandRouteIndex(commandRoutes)
// ResolveEdgeCommandName returns the edge command for a request, or None when the allowlist does not cover it.
func ResolveEdgeCommandName(method, requestPath string, stream bool) mo.Option[string] {
requestPath, _, _ = strings.Cut(strings.TrimSpace(requestPath), "?")
- current := commandRoutesIndex[commandRouteKey{Method: kit.Ternary(stream, "", strings.ToUpper(strings.TrimSpace(method))), Stream: stream}]
- for part := range strings.SplitSeq(strings.Trim(requestPath, "/"), "/") {
- if current == nil {
- break
- }
- if next := current.static[part]; next != nil {
- current = next
- } else if part != "" {
- current = current.param
- } else {
- current = nil
- }
- }
- if current == nil || current.route == nil {
+ root := commandRoutesIndex[commandRouteKey{Method: kit.Ternary(stream, "", strings.ToUpper(strings.TrimSpace(method))), Stream: stream}]
+ route := root.match(strings.Split(strings.Trim(requestPath, "/"), "/"))
+ if route == nil {
return mo.None[string]()
}
- return mo.Some(current.route.CommandName)
+ return mo.Some(route.CommandName)
+}
+
+func (n *commandRouteNode) match(parts []string) *commandRoute {
+ if n == nil {
+ return nil
+ }
+ if len(parts) == 0 {
+ return n.route
+ }
+ if route := n.static[parts[0]].match(parts[1:]); route != nil {
+ return route
+ }
+ if parts[0] == "" {
+ return nil
+ }
+ return n.param.match(parts[1:])
}
// ValidateEdgeCommand reports whether commandName is the allowlisted command for the request.
diff --git a/backend/pkg/libarcane/edge/commands_test.go b/backend/pkg/libarcane/edge/commands_test.go
index 91968c3c8..4c5c580a3 100644
--- a/backend/pkg/libarcane/edge/commands_test.go
+++ b/backend/pkg/libarcane/edge/commands_test.go
@@ -41,6 +41,11 @@ func TestResolveEdgeCommandName(t *testing.T) {
shouldHit: true,
},
{name: "volume workspace list", method: "GET", path: "/api/environments/0/volumes/data/workspace", command: "volume.workspace.list", shouldHit: true},
+ {name: "volume named backups inspect", method: "GET", path: "/api/environments/0/volumes/backups", command: "volume.inspect", shouldHit: true},
+ {name: "volume named backups workspace", method: "GET", path: "/api/environments/0/volumes/backups/workspace", command: "volume.workspace.list", shouldHit: true},
+ {name: "volume named counts usage", method: "GET", path: "/api/environments/0/volumes/counts/usage", command: "volume.usage", shouldHit: true},
+ {name: "volume named sizes backups", method: "GET", path: "/api/environments/0/volumes/sizes/backups", command: "volume.backup.list", shouldHit: true},
+ {name: "volume backup download keeps static branch", method: "GET", path: "/api/environments/0/volumes/backups/b1/download", command: "volume.backup.download", shouldHit: true},
{
name: "volume workspace read file",
method: "GET",
diff --git a/backend/pkg/libarcane/edge/event_sync.go b/backend/pkg/libarcane/edge/event_sync.go
index 02c6b27de..5d9156dca 100644
--- a/backend/pkg/libarcane/edge/event_sync.go
+++ b/backend/pkg/libarcane/edge/event_sync.go
@@ -12,6 +12,8 @@ import (
"sync"
"time"
"uuid"
+
+ "go.getarcane.app/kit/pkg"
)
const (
@@ -69,7 +71,10 @@ func PublishEventToManager(ctx context.Context, event *TunnelEvent) error {
if sink.conn.IsClosed() {
return ErrNoActiveAgentTunnel
}
- return sink.conn.Send(&TunnelMessage{ID: uuid.New().String(), Type: MessageTypeEvent, Event: event})
+ if err := sink.conn.Send(&TunnelMessage{ID: uuid.New().String(), Type: MessageTypeEvent, Event: event}); err != nil {
+ return kit.Ternary(errors.Is(err, ErrTunnelConnectionClosed), fmt.Errorf("%w: %w", ErrNoActiveAgentTunnel, err), err)
+ }
+ return nil
}
body, err := json.Marshal(event)
@@ -83,7 +88,7 @@ func PublishEventToManager(ctx context.Context, event *TunnelEvent) error {
req.Header.Set("Content-Type", "application/json")
resp, err := sink.client.Do(req)
if err != nil {
- return fmt.Errorf("send edge event: %w", err)
+ return kit.Ternary(errors.Is(err, ErrTunnelConnectionClosed), fmt.Errorf("send edge event: %w: %w", ErrNoActiveAgentTunnel, err), fmt.Errorf("send edge event: %w", err))
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusNoContent {
diff --git a/backend/pkg/libarcane/edge/mux.go b/backend/pkg/libarcane/edge/mux.go
index 8a2d53ca4..fea17ac08 100644
--- a/backend/pkg/libarcane/edge/mux.go
+++ b/backend/pkg/libarcane/edge/mux.go
@@ -37,6 +37,8 @@ const (
muxWriteTimeout = 30 * time.Second
// muxDialTimeout bounds a stream open on its own, because http.Transport detaches dials from request cancellation.
muxDialTimeout = 30 * time.Second
+ // ProxyResponseHeaderTimeout is generous because proxied operations such as deploys can run for minutes before responding.
+ ProxyResponseHeaderTimeout = 30 * time.Minute
)
// errTunnelMuxActive rejects legacy messages once a connection carries a multiplexed session.
@@ -76,6 +78,7 @@ func newMuxHTTPClient(dial func(ctx context.Context) (net.Conn, error)) *http.Cl
IdleConnTimeout: 30 * time.Second,
MaxIdleConnsPerHost: 32,
ExpectContinueTimeout: time.Second,
+ ResponseHeaderTimeout: ProxyResponseHeaderTimeout,
ReadBufferSize: 64 << 10,
WriteBufferSize: 64 << 10,
})}
diff --git a/backend/pkg/libarcane/edge/mux_manager.go b/backend/pkg/libarcane/edge/mux_manager.go
index 94289ad11..e63560300 100644
--- a/backend/pkg/libarcane/edge/mux_manager.go
+++ b/backend/pkg/libarcane/edge/mux_manager.go
@@ -15,8 +15,11 @@ import (
"github.com/samber/mo"
)
-// maxMuxEventBytes bounds one agent event posted over a multiplexed session.
-const maxMuxEventBytes = 1 << 20
+const (
+ // maxMuxEventBytes bounds one agent event posted over a multiplexed session.
+ maxMuxEventBytes = 1 << 20
+ muxRegistrationPingTimeout = 3 * time.Second
+)
// dial opens a stream for the session client. Callers that found the tunnel during the register handshake
// wait for the session to start.
@@ -47,10 +50,31 @@ func (m *muxSession) close() {
if session != nil {
_ = session.Close()
}
- m.client.CloseIdleConnections()
})
}
+func (m *muxSession) alive(ctx context.Context, timeout time.Duration) bool {
+ m.mu.Lock()
+ session := m.session
+ m.mu.Unlock()
+ if session == nil {
+ return true
+ }
+ pinged := make(chan error, 1)
+ go func() {
+ _, err := session.Ping()
+ pinged <- err
+ }()
+ select {
+ case err := <-pinged:
+ return err == nil
+ case <-time.After(timeout):
+ return false
+ case <-ctx.Done():
+ return true
+ }
+}
+
// HTTPClient returns the client for a tunnel that negotiated the multiplexed data plane. Requests use
// MuxAgentBaseURL as their origin and are served by the agent's own router.
func (t *AgentTunnel) HTTPClient() mo.Option[*http.Client] {
diff --git a/backend/pkg/libarcane/edge/poll_control.go b/backend/pkg/libarcane/edge/poll_control.go
index 428602d64..8ee9a78c0 100644
--- a/backend/pkg/libarcane/edge/poll_control.go
+++ b/backend/pkg/libarcane/edge/poll_control.go
@@ -1,9 +1,6 @@
package edge
import (
- "encoding/json/v2"
- "errors"
- "io"
"log/slog"
"net/http"
"strings"
@@ -180,15 +177,6 @@ func (s *TunnelServer) HandlePoll(c *echo.Context) error {
req := c.Request()
ctx := req.Context()
- if req.Body != nil {
- var pollReq TunnelPollRequest
- err := json.UnmarshalRead(req.Body, &pollReq)
- _ = req.Body.Close()
- if err != nil && !errors.Is(err, http.ErrBodyReadAfterClose) && !errors.Is(err, io.EOF) {
- return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid poll payload"})
- }
- }
-
// Proxy-terminated mTLS consumes the client certificate, so the token remains the environment claim.
token, source := agentToken(req.Header.Values)
if token == "" {
diff --git a/backend/pkg/libarcane/edge/proxy.go b/backend/pkg/libarcane/edge/proxy.go
index db2465e54..a2a64e47d 100644
--- a/backend/pkg/libarcane/edge/proxy.go
+++ b/backend/pkg/libarcane/edge/proxy.go
@@ -20,7 +20,8 @@ const (
DefaultProxyTimeout = 5 * time.Minute
DefaultTunnelAcquirePollEvery = 100 * time.Millisecond
// Deprecated: legacy edge message protocol; remove in v3.
- tunnelStreamHeader = "X-Arcane-Tunnel-Stream"
+ tunnelStreamHeader = "X-Arcane-Tunnel-Stream"
+ maxLegacyCommandBodyBytes = 256 << 20
)
// browserSecurityHeaders must not cross a legacy tunnel: the agent's CORS check would reject the browser Origin,
@@ -153,7 +154,7 @@ func (s *grpcResponseState) handleTunnelMessage(method string, incoming *TunnelM
s.writeBody(incoming.Body)
if incoming.Error != "" && incoming.Status >= http.StatusBadRequest {
done, status, headers, body = true, incoming.Status, s.cleanHeaders(), s.respBody.Bytes()
- err = errors.New(incoming.Error)
+ err = &commandStatusError{status: incoming.Status, message: incoming.Error}
} else {
done = true
_, headers, body = s.finish()
@@ -268,7 +269,10 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) e
var body []byte
if req.Body != nil {
var err error
- body, err = io.ReadAll(req.Body)
+ body, err = io.ReadAll(http.MaxBytesReader(c.Response(), req.Body, maxLegacyCommandBodyBytes))
+ if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
+ return c.JSON(http.StatusRequestEntityTooLarge, map[string]any{"error": "request body is too large for a legacy edge tunnel; upgrade the agent"})
+ }
if err != nil {
slog.ErrorContext(ctx, "Failed to read request body for tunnel proxy", "error", err)
return c.JSON(http.StatusInternalServerError, map[string]any{"error": "failed to read request body"})
@@ -301,6 +305,9 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) e
if resp, unwrapErr := echo.UnwrapResponse(c.Response()); unwrapErr == nil && resp.Committed {
panic(http.ErrAbortHandler)
}
+ if statusErr, ok := errors.AsType[*commandStatusError](err); ok {
+ return c.JSON(statusErr.status, map[string]any{"error": statusErr.message})
+ }
if proxyCtx.Err() != nil {
return c.JSON(http.StatusGatewayTimeout, map[string]any{"error": "request timed out"})
}
@@ -311,6 +318,10 @@ func ProxyHTTPRequest(c *echo.Context, tunnel *AgentTunnel, targetPath string) e
return nil
}
+func (e *commandStatusError) Error() string {
+ return e.message
+}
+
// cleanHeaders returns the response headers without the internal stream marker.
func (s *grpcResponseState) cleanHeaders() map[string]string {
headers := maps.Clone(s.respHeaders)
diff --git a/backend/pkg/libarcane/edge/registry.go b/backend/pkg/libarcane/edge/registry.go
index 61a5f5b38..35f017f70 100644
--- a/backend/pkg/libarcane/edge/registry.go
+++ b/backend/pkg/libarcane/edge/registry.go
@@ -204,6 +204,10 @@ func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunne
if envID == "" {
return false, false, "environment ID is required", nil
}
+ if existing, ok := r.Get(envID).Get(); ok && existing != tunnel && existing.mux != nil && existing.connected() && !existing.mux.alive(ctx, muxRegistrationPingTimeout) {
+ slog.InfoContext(ctx, "Existing edge tunnel session failed a liveness ping; replacing it", "environmentId", envID)
+ _ = existing.CloseWithReason("edge tunnel liveness ping failed")
+ }
result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResult, bool, error) {
var result registerSessionResult
diff --git a/backend/pkg/libarcane/edge/tls.go b/backend/pkg/libarcane/edge/tls.go
index a194ba057..8c81eb478 100644
--- a/backend/pkg/libarcane/edge/tls.go
+++ b/backend/pkg/libarcane/edge/tls.go
@@ -62,6 +62,7 @@ const (
var (
generatedAssetNameSanitizer = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
managerCALocks utils.KeyedMutex
+ errEnrollUnreachable = errors.New("edge mTLS enrollment request failed")
// caKeyEncrypt is a variable so tests can force encryption failures.
caKeyEncrypt = libcrypto.Encrypt
)
@@ -185,7 +186,7 @@ func GenerateManagerClientMTLSAssetsWithContext(ctx context.Context, cfg *Config
uriSAN := certgen.BuildEdgeMTLSURISAN(appURL, safeEnvID)
// The URI SAN is the stable identity, so renaming an environment keeps its certificate.
- existing, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ existing, reason := loadClientCertificate(certPath, keyPath, time.Now(), agentMTLSRenewBefore)
certIssued := reason != "" || !supportedGeneratedKey(existing.Leaf.PublicKey) ||
existing.Leaf.CheckSignatureFrom(ca.Leaf) != nil || (uriSAN != nil && !certificateHasURISAN(existing.Leaf, uriSAN))
if certIssued {
@@ -282,8 +283,7 @@ func GeneratedManagerClientMTLSCertPath(cfg *Config, envID string) (string, erro
// EnsureAgentMTLSAssets downloads manager-generated client certificates when edge mTLS is
// enabled without configured client cert/key files, and renews them before they expire.
func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {
- if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled ||
- (strings.TrimSpace(cfg.EdgeMTLSCertFile) != "" && strings.TrimSpace(cfg.EdgeMTLSKeyFile) != "") {
+ if cfg == nil || NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeDisabled {
return nil
}
@@ -293,18 +293,27 @@ func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {
}
certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)
keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)
+ configuredCert, configuredKey := strings.TrimSpace(cfg.EdgeMTLSCertFile), strings.TrimSpace(cfg.EdgeMTLSKeyFile)
+ if configuredCert != "" && configuredKey != "" && (filepath.Clean(configuredCert) != certPath || filepath.Clean(configuredKey) != keyPath) {
+ return nil
+ }
- _, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ _, reason := loadClientCertificate(certPath, keyPath, time.Now(), agentMTLSRenewBefore)
if reason != "" {
if fileExists(certPath) {
slog.WarnContext(ctx, "Existing edge mTLS assets need renewal; enrolling new assets", "reason", reason, "certPath", certPath)
}
if enrollErr := enrollAgentMTLSAssets(ctx, cfg, assetsDir, certPath, keyPath); enrollErr != nil {
- if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired {
+ _, currentReason := loadClientCertificate(certPath, keyPath, time.Now(), 0)
+ switch {
+ case currentReason == "" && errors.Is(enrollErr, errEnrollUnreachable):
+ slog.WarnContext(ctx, "Edge mTLS renewal failed; keeping the current client certificate until it expires", "error", enrollErr, "certPath", certPath)
+ case NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) != EdgeMTLSModeRequired:
slog.WarnContext(ctx, "Edge mTLS enrollment failed; proceeding without client certificate", "error", enrollErr)
return nil
+ default:
+ return enrollErr
}
- return enrollErr
}
}
@@ -314,8 +323,10 @@ func EnsureAgentMTLSAssets(ctx context.Context, cfg *Config) error {
return fmt.Errorf("failed to write edge mTLS enrollment marker: %w", writeErr)
}
}
- cfg.EdgeMTLSCertFile = certPath
- cfg.EdgeMTLSKeyFile = keyPath
+ if configuredCert != certPath || configuredKey != keyPath {
+ cfg.EdgeMTLSCertFile = certPath
+ cfg.EdgeMTLSKeyFile = keyPath
+ }
if caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName); fileExists(caPath) && strings.TrimSpace(cfg.EdgeMTLSCAFile) == "" {
cfg.EdgeMTLSCAFile = caPath
}
@@ -333,7 +344,12 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPath
return errors.New("EDGE_MTLS_MODE requires MANAGER_API_URL to use https for certificate enrollment")
}
- httpClient, clientErr := NewManagerHTTPClient(cfg, 30*time.Second)
+ httpClient, clientErr := NewManagerHTTPClient(&Config{
+ EdgeMTLSMode: cfg.EdgeMTLSMode,
+ EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,
+ EdgeMTLSServerName: cfg.EdgeMTLSServerName,
+ ManagerApiUrl: cfg.ManagerApiUrl,
+ }, 30*time.Second)
if clientErr != nil {
return fmt.Errorf("failed to configure edge mTLS enrollment client: %w", clientErr)
}
@@ -347,7 +363,12 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPath
resp, doErr := httpClient.Do(req)
if doErr != nil {
- return fmt.Errorf("edge mTLS enrollment request failed: %w", doErr)
+ _, certRejected := errors.AsType[*tls.CertificateVerificationError](doErr)
+ _, alert := errors.AsType[tls.AlertError](doErr)
+ if certRejected || alert {
+ return fmt.Errorf("edge mTLS enrollment handshake failed: %w", doErr)
+ }
+ return fmt.Errorf("%w: %w", errEnrollUnreachable, doErr)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
@@ -373,7 +394,7 @@ func enrollAgentMTLSAssets(ctx context.Context, cfg *Config, assetsDir, certPath
return fmt.Errorf("failed to write edge mTLS asset %s: %w", file.Name, writeErr)
}
}
- if _, writtenReason := loadClientCertificate(certPath, keyPath, time.Now()); writtenReason != "" {
+ if _, writtenReason := loadClientCertificate(certPath, keyPath, time.Now(), 0); writtenReason != "" {
return fmt.Errorf("edge mTLS enrollment wrote unusable assets: %s", writtenReason)
}
return nil
@@ -404,7 +425,7 @@ func buildManagerClientTLSConfig(cfg *Config) (*tls.Config, error) {
if certPath == "" || keyPath == "" {
return tlsConfig, nil
}
- cert, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now(), 0)
if reason != "" {
if NormalizeEdgeMTLSMode(cfg.EdgeMTLSMode) == EdgeMTLSModeOptional {
slog.WarnContext(ctx, "Ignoring unusable optional edge mTLS client certificate; falling back to token auth", "certPath", certPath, "reason", reason)
@@ -534,12 +555,31 @@ func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, bo
// os rather than acfs: the assets dir may be configured anywhere on the host.
certPEM, certErr := os.ReadFile(certPath)
rawKey, keyErr := os.ReadFile(keyPath)
- keyPEM := ""
- if ciphertext, encrypted := strings.CutPrefix(strings.TrimSpace(string(rawKey)), caKeyEncryptedPrefix); certErr == nil && keyErr == nil && encrypted {
- keyPEM, _ = libcrypto.Decrypt(ciphertext)
+ for _, readErr := range []error{certErr, keyErr} {
+ if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
+ return tls.Certificate{}, false, fmt.Errorf("failed to read edge mTLS CA: %w", readErr)
+ }
}
- if ca, pairErr := tls.X509KeyPair(certPEM, []byte(keyPEM)); pairErr == nil && ca.Leaf.IsCA && supportedGeneratedKey(ca.Leaf.PublicKey) {
- return ca, false, nil
+ ciphertext, encrypted := strings.CutPrefix(strings.TrimSpace(string(rawKey)), caKeyEncryptedPrefix)
+ switch {
+ case certErr != nil && keyErr != nil:
+ case certErr != nil || keyErr != nil:
+ slog.WarnContext(ctx, "Edge mTLS CA is incomplete; regenerating it", "certPath", certPath, "keyPath", keyPath)
+ case !encrypted:
+ slog.WarnContext(ctx, "Edge mTLS CA key is not in the supported encrypted format; regenerating the CA", "keyPath", keyPath)
+ default:
+ keyPEM, decryptErr := libcrypto.Decrypt(ciphertext)
+ if decryptErr != nil {
+ return tls.Certificate{}, false, fmt.Errorf("failed to decrypt edge mTLS CA private key %s: %w", keyPath, decryptErr)
+ }
+ ca, pairErr := tls.X509KeyPair(certPEM, []byte(keyPEM))
+ if pairErr != nil {
+ return tls.Certificate{}, false, fmt.Errorf("failed to load edge mTLS CA key pair from %s: %w", assetsDir, pairErr)
+ }
+ if ca.Leaf.IsCA && supportedGeneratedKey(ca.Leaf.PublicKey) {
+ return ca, false, nil
+ }
+ slog.WarnContext(ctx, "Edge mTLS CA uses an unsupported algorithm; regenerating it", "certPath", certPath)
}
privateKey, err := certgen.GenerateMLDSA87PrivateKey()
@@ -566,14 +606,14 @@ func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, bo
if err != nil {
return tls.Certificate{}, false, fmt.Errorf("failed to marshal CA private key: %w", err)
}
- ciphertext, err := caKeyEncrypt(string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})))
+ encryptedKey, err := caKeyEncrypt(string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})))
if err != nil {
return tls.Certificate{}, false, fmt.Errorf("failed to encrypt edge mTLS CA private key: %w", err)
}
- if ciphertext == "" {
+ if encryptedKey == "" {
return tls.Certificate{}, false, errors.New("failed to encrypt edge mTLS CA private key: encrypted payload is empty")
}
- if writeErr := atomic.WriteFile(keyPath, []byte(caKeyEncryptedPrefix+ciphertext), 0o600); writeErr != nil {
+ if writeErr := atomic.WriteFile(keyPath, []byte(caKeyEncryptedPrefix+encryptedKey), 0o600); writeErr != nil {
return tls.Certificate{}, false, writeErr
}
@@ -582,8 +622,8 @@ func ensureManagerCA(ctx context.Context, assetsDir string) (tls.Certificate, bo
}
// loadClientCertificate loads a client key pair, or returns why it must be (re)issued: unreadable,
-// mismatched, outside its validity window, or within agentMTLSRenewBefore of expiry.
-func loadClientCertificate(certPath, keyPath string, now time.Time) (tls.Certificate, string) {
+// mismatched, outside its validity window, or within renewBefore of expiry.
+func loadClientCertificate(certPath, keyPath string, now time.Time, renewBefore time.Duration) (tls.Certificate, string) {
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
switch {
case err != nil:
@@ -592,7 +632,7 @@ func loadClientCertificate(certPath, keyPath string, now time.Time) (tls.Certifi
return tls.Certificate{}, "certificate is not valid before " + cert.Leaf.NotBefore.UTC().Format(time.RFC3339)
case !now.Before(cert.Leaf.NotAfter):
return tls.Certificate{}, "certificate expired at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)
- case now.Add(agentMTLSRenewBefore).After(cert.Leaf.NotAfter):
+ case renewBefore > 0 && now.Add(renewBefore).After(cert.Leaf.NotAfter):
return tls.Certificate{}, "certificate expires soon at " + cert.Leaf.NotAfter.UTC().Format(time.RFC3339)
}
return cert, ""
diff --git a/backend/pkg/libarcane/edge/tls_test.go b/backend/pkg/libarcane/edge/tls_test.go
index e7e6f8861..b301ac6fe 100644
--- a/backend/pkg/libarcane/edge/tls_test.go
+++ b/backend/pkg/libarcane/edge/tls_test.go
@@ -8,6 +8,7 @@ import (
"crypto/rand"
"crypto/tls"
"crypto/x509"
+ "encoding/json/v2"
"encoding/pem"
"errors"
"fmt"
@@ -192,6 +193,51 @@ func TestEnsureAgentMTLSAssets_LimitsEnrollmentErrorBody(t *testing.T) {
require.LessOrEqual(t, len(err.Error()), maxEnrollResponseBytes+128)
}
+func TestEnsureAgentMTLSAssets_EnrollsWithoutUnusableGeneratedCert(t *testing.T) {
+ generated, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSAssetsDir: t.TempDir(),
+ AppURL: "https://manager.example.com",
+ }, "env-renew", "Renew")
+ require.NoError(t, err)
+
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.TLS == nil || len(r.TLS.PeerCertificates) > 0 {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ _ = json.MarshalWrite(w, enrollMTLSResponse{Files: generated.Files})
+ }))
+ t.Cleanup(server.Close)
+
+ caPath := filepath.Join(t.TempDir(), "manager.crt")
+ require.NoError(t, os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{
+ Type: "CERTIFICATE",
+ Bytes: server.Certificate().Raw,
+ }), 0o600))
+
+ assetsDir := filepath.Join(t.TempDir(), "assets")
+ require.NoError(t, os.MkdirAll(assetsDir, 0o700))
+ certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)
+ keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)
+ require.NoError(t, os.WriteFile(certPath, []byte("not a certificate"), 0o600))
+ require.NoError(t, os.WriteFile(keyPath, []byte("not a key"), 0o600))
+
+ cfg := &Config{
+ ManagerApiUrl: server.URL + "/api",
+ AgentToken: "valid-token",
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSCAFile: caPath,
+ EdgeMTLSCertFile: certPath,
+ EdgeMTLSKeyFile: keyPath,
+ EdgeMTLSAssetsDir: assetsDir,
+ }
+
+ require.NoError(t, EnsureAgentMTLSAssets(t.Context(), cfg))
+ _, reason := loadClientCertificate(certPath, keyPath, time.Now(), 0)
+ require.Empty(t, reason)
+}
+
func TestEnsureAgentMTLSAssets_UsesDownloadedCAPathWhenPresent(t *testing.T) {
assetsDir := t.TempDir()
caPath := filepath.Join(assetsDir, generatedMTLSCACertFileName)
@@ -380,13 +426,41 @@ func TestLoadClientCertificate_RenewsExpiredCertificate(t *testing.T) {
certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)
keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)
- cert, reason := loadClientCertificate(certPath, keyPath, time.Now())
+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now(), agentMTLSRenewBefore)
require.Empty(t, reason)
- _, reason = loadClientCertificate(certPath, keyPath, cert.Leaf.NotAfter.Add(24*time.Hour))
+ _, reason = loadClientCertificate(certPath, keyPath, cert.Leaf.NotAfter.Add(24*time.Hour), agentMTLSRenewBefore)
require.Contains(t, reason, "expired")
}
+func TestLoadClientCertificate_RenewBeforeOnlyAppliesWhenRequested(t *testing.T) {
+ assetsDir := t.TempDir()
+ assets, err := GenerateManagerClientMTLSAssetsWithContext(t.Context(), &Config{
+ EdgeMTLSMode: EdgeMTLSModeRequired,
+ EdgeMTLSAssetsDir: t.TempDir(),
+ AppURL: "https://manager.example.com",
+ }, "env-window", "Window")
+ require.NoError(t, err)
+
+ for _, file := range assets.Files {
+ targetPath := filepath.Join(assetsDir, filepath.Base(file.Name))
+ perm := kit.Ternary(file.Permissions == "0600", 0o600, utils.FilePerm)
+ require.NoError(t, os.WriteFile(targetPath, []byte(file.Content), perm))
+ }
+
+ certPath := filepath.Join(assetsDir, generatedMTLSClientCertName)
+ keyPath := filepath.Join(assetsDir, generatedMTLSClientKeyName)
+ cert, reason := loadClientCertificate(certPath, keyPath, time.Now(), 0)
+ require.Empty(t, reason)
+
+ nearExpiry := cert.Leaf.NotAfter.Add(-24 * time.Hour)
+ _, reason = loadClientCertificate(certPath, keyPath, nearExpiry, 0)
+ require.Empty(t, reason, "a certificate inside the renewal window stays usable when no window is requested")
+
+ _, reason = loadClientCertificate(certPath, keyPath, nearExpiry, agentMTLSRenewBefore)
+ require.Contains(t, reason, "expires soon")
+}
+
func TestGenerateManagerClientMTLSAssets_ReissuesMismatchedKeyPair(t *testing.T) {
cfg := &Config{
EdgeMTLSMode: EdgeMTLSModeRequired,
diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.go
index bcd67bb36..fe47084b8 100644
--- a/backend/pkg/libarcane/edge/types.go
+++ b/backend/pkg/libarcane/edge/types.go
@@ -44,6 +44,11 @@ type commandRequestTransfer struct {
timer *time.Timer
}
+type commandStatusError struct {
+ status int
+ message string
+}
+
// TunnelClient represents the agent-side tunnel client
type TunnelClient struct {
cfg *Config
@@ -62,6 +67,7 @@ type TunnelClient struct {
activeStreams sync.Map // map[string]*activeWSStream
requestTransfers sync.Map // map[string]*commandRequestTransfer
commandRecorders sync.Map // map[string]*commandResponseRecorder awaiting credit
+ commandCancels sync.Map
transportPreferenceMu sync.RWMutex
preferWebSocketUntil time.Time
grpcFailureStreak int
diff --git a/backend/pkg/remenv/client.go b/backend/pkg/remenv/client.go
index c3c0fe059..24db6b578 100644
--- a/backend/pkg/remenv/client.go
+++ b/backend/pkg/remenv/client.go
@@ -10,8 +10,8 @@ import (
"net"
"net/http"
"strings"
- "time"
+ "go.getarcane.app/kit/pkg"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
@@ -92,16 +92,14 @@ type Client struct {
}
func NewClient(httpClient *http.Client, tunnel TunnelTransport) *Client {
- client := &http.Client{Timeout: 30 * time.Second}
- if httpClient != nil {
- copied := *httpClient
- client = &copied
- }
- client.Transport = otelhttp.NewTransport(client.Transport)
-
+ base := kit.FromPtr(httpClient)
return &Client{
- httpClient: client,
- tunnel: tunnel,
+ httpClient: &http.Client{
+ Transport: otelhttp.NewTransport(base.Transport),
+ CheckRedirect: base.CheckRedirect,
+ Jar: base.Jar,
+ },
+ tunnel: tunnel,
}
}
diff --git a/backend/pkg/utils/httpx/request.go b/backend/pkg/utils/httpx/request.go
index 2e64fd434..46cd34f39 100644
--- a/backend/pkg/utils/httpx/request.go
+++ b/backend/pkg/utils/httpx/request.go
@@ -153,6 +153,7 @@ func NewHTTPClient(options httpxtypes.ClientOptions) *http.Client {
MaxIdleConnsPerHost: 32,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: options.TLSHandshakeTimeout,
+ ResponseHeaderTimeout: options.ResponseHeaderTimeout,
ExpectContinueTimeout: 1 * time.Second,
ForceAttemptHTTP2: true,
}
diff --git a/frontend/src/lib/types/shared.ts b/frontend/src/lib/types/shared.ts
index 1648a972d..cad6eca9c 100644
--- a/frontend/src/lib/types/shared.ts
+++ b/frontend/src/lib/types/shared.ts
@@ -238,7 +238,7 @@ export interface DashboardSnapshot {
export type DashboardStreamEventType = 'snapshot' | 'error' | 'pending' | 'heartbeat';
-export type DashboardStreamErrorCode = 'agent_incompatible' | 'unreachable' | '';
+export type DashboardStreamErrorCode = 'agent_incompatible' | 'unreachable' | 'auth_failed' | '';
export interface DashboardStreamEvent {
type: DashboardStreamEventType;
diff --git a/tests/spec/cli.spec.ts b/tests/spec/cli.spec.ts
index 08a5c877c..fb30da61a 100644
--- a/tests/spec/cli.spec.ts
+++ b/tests/spec/cli.spec.ts
@@ -626,8 +626,8 @@ test.describe('arcane-cli e2e', () => {
// - a DENIED request returns 403 — the permission check runs before the tunnel
// lookup, so it short-circuits regardless of connectivity;
// - an AUTHORIZED request gets past the permission check and reaches the tunnel
-// stage, which returns 502 ("edge agent is not connected").
-// The 403-vs-502 split is exactly what proves the permission is enforced per
+// stage, which returns 503 ("Edge agent is not connected").
+// The 403-vs-503 split is exactly what proves the permission is enforced per
// environment and is not a blanket block. The local environment ("0") is served
// directly (never proxied) and must remain unaffected.
test.describe('arcane-cli remote environment RBAC', () => {
@@ -692,7 +692,7 @@ test.describe('arcane-cli remote environment RBAC', () => {
// Create a remote edge environment. No agent will ever connect to it; it
// exists only so the proxy treats requests to it as remote (edge://) and
- // reaches either the permission check (403) or the tunnel lookup (502).
+ // reaches either the permission check (403) or the tunnel lookup (503).
const envResponse = await adminFetch('/api/environments', {
method: 'POST',
body: JSON.stringify({
@@ -777,7 +777,7 @@ test.describe('arcane-cli remote environment RBAC', () => {
});
});
- test('authorizes a remote write for an admin key, reaching the disconnected agent (502, not 403)', async () => {
+ test('authorizes a remote write for an admin key, reaching the disconnected agent (503, not 403)', async () => {
await withAdminConfig(remoteEnvId, async (config) => {
const error = await runExpectingFailure(config.configPath, [
'volumes',
@@ -787,7 +787,7 @@ test.describe('arcane-cli remote environment RBAC', () => {
]);
// The admin key passes the permission check, so the request reaches the
// edge tunnel stage and fails only because no agent is connected.
- expect(error).toContain('status 502');
+ expect(error).toContain('status 503');
expect(error).not.toContain('status 403');
});
});
diff --git a/types/dashboard/dashboard.go b/types/dashboard/dashboard.go
index 35dcca89f..99263d30a 100644
--- a/types/dashboard/dashboard.go
+++ b/types/dashboard/dashboard.go
@@ -28,6 +28,7 @@ const (
StreamErrorCodeAgentIncompatible = "agent_incompatible"
StreamErrorCodeUnreachable = "unreachable"
+ StreamErrorCodeAuthFailed = "auth_failed"
)
type ActionItemSeverity string
diff --git a/types/httpx/client.go b/types/httpx/client.go
index 090f632c0..bfa54aec4 100644
--- a/types/httpx/client.go
+++ b/types/httpx/client.go
@@ -5,7 +5,8 @@ import "time"
// ClientOptions configures timeouts for an outbound HTTP client.
// Durations are used as provided; a zero Timeout disables the request timeout.
type ClientOptions struct {
- Timeout time.Duration
- TLSHandshakeTimeout time.Duration
- DialTimeout time.Duration
+ Timeout time.Duration
+ TLSHandshakeTimeout time.Duration
+ DialTimeout time.Duration
+ ResponseHeaderTimeout time.Duration
}