Skip to content

Modern Docker Management, Designed for Everyone

fix: reduce remote agent egress from manager polling and config sync (#4218)

Kyle Mendelld6f28f99e078
diff --git a/backend/internal/activity/handler.go b/backend/internal/activity/handler.go
index 865f4d09b..05a4d1ced 100644
--- a/backend/internal/activity/handler.go
+++ b/backend/internal/activity/handler.go
@@ -646,16 +646,21 @@ func (h *ActivityHandler) proxyListActivitiesForEnvironmentInternal(ctx context.
}
func (h *ActivityHandler) listRemoteActivitiesInternal(ctx context.Context, input *ListActivitiesInput, fetch func(string) (*base.Paginated[activitytypes.Activity], error)) (*handlerutil.Page[activitytypes.Activity], error) {
- all := *input
- all.Start, all.Limit = 0, -1
- remote, remoteErr := fetch("/api/environments/0/activities?" + activityListQueryInternal(&all).Encode())
- var remoteActivities []activitytypes.Activity
- if remoteErr == nil {
- remoteActivities = remote.Data
- }
+ params := normalizeRemoteActivityParamsInternal(activityListParamsInternal(input))
+ // The agent only ships the newest window the merged page can use, not its
+ // whole history; the stream poller repeats this every few seconds.
+ remoteActivities, remoteTotal, remoteErr := collectActivityWindowInternal(remoteActivityWindowInternal(params), func(start, limit int) ([]activitytypes.Activity, int64, error) {
+ page := *input
+ page.Start, page.Limit = start, limit
+ remote, err := fetch("/api/environments/0/activities?" + activityListQueryInternal(&page).Encode())
+ if err != nil {
+ return nil, 0, err
+ }
+ return remote.Data, max(remote.Pagination.TotalItems, 0), nil
+ })
// The proxy may have exhausted its timeout; manager summaries still need
// to be read so an offline environment's queued work remains visible.
- activities, page, err := h.activityService.ListRemoteActivities(ctx, input.EnvironmentID, remoteActivities, activityListParamsInternal(input))
+ activities, page, err := h.activityService.ListRemoteActivities(ctx, input.EnvironmentID, remoteActivities, remoteTotal, params)
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
diff --git a/backend/internal/activity/job_test.go b/backend/internal/activity/job_test.go
index 231042d13..8c75795d3 100644
--- a/backend/internal/activity/job_test.go
+++ b/backend/internal/activity/job_test.go
@@ -70,12 +70,12 @@ func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {
require.EqualValues(t, 1, page.TotalItems)
require.Equal(t, "remote", rows[0].EnvironmentID)
remoteItems := []activitytypes.Activity{{ID: "agent-work", EnvironmentID: "0", Type: activitytypes.TypeImagePull, Status: activitytypes.StatusRunning, CreatedAt: now}}
- rows, page, err = service.ListRemoteActivities(ctx, "remote", remoteItems, pagination.QueryParams{Start: 1, Limit: 1})
+ rows, page, err = service.ListRemoteActivities(ctx, "remote", remoteItems, int64(len(remoteItems)), pagination.QueryParams{Start: 1, Limit: 1})
require.NoError(t, err)
require.EqualValues(t, 2, page.TotalItems)
require.Len(t, rows, 1)
require.Equal(t, run.ActivityID, rows[0].ID)
- rows, _, err = service.ListRemoteActivities(ctx, "remote", remoteItems, pagination.QueryParams{Limit: 1})
+ rows, _, err = service.ListRemoteActivities(ctx, "remote", remoteItems, int64(len(remoteItems)), pagination.QueryParams{Limit: 1})
require.NoError(t, err)
require.Equal(t, "agent-work", rows[0].ID)
require.Equal(t, "remote", rows[0].EnvironmentID)
diff --git a/backend/internal/activity/remote.go b/backend/internal/activity/remote.go
index fb64bb8de..2e8459ff7 100644
--- a/backend/internal/activity/remote.go
+++ b/backend/internal/activity/remote.go
@@ -10,12 +10,70 @@ import (
"github.com/samber/lo"
)
+// remoteActivityPageCeiling mirrors the agent's page clamp in PaginateAndSortDB.
+const remoteActivityPageCeiling = 100
+
+// normalizeRemoteActivityParamsInternal applies the agent's paging bounds so both
+// sources are cut the same way; -1 stays the explicit "show all" sentinel.
+func normalizeRemoteActivityParamsInternal(params pagination.QueryParams) pagination.QueryParams {
+ params.Start = max(params.Start, 0)
+ switch {
+ case params.Limit == -1:
+ case params.Limit <= 0:
+ params.Limit = 20
+ case params.Limit > remoteActivityPageCeiling:
+ params.Limit = remoteActivityPageCeiling
+ }
+ return params
+}
+
+// remoteActivityWindowInternal is how many newest rows each source must supply
+// for the merged page to be exact: every row in the global top start+limit is
+// in its own source's top start+limit under the shared sort order.
+func remoteActivityWindowInternal(params pagination.QueryParams) int {
+ if params.Limit == -1 {
+ return -1
+ }
+ return params.Start + params.Limit
+}
+
+// collectActivityWindowInternal reads window rows from a source in pages the
+// agent accepts, stopping early when the source runs out. -1 reads everything.
+func collectActivityWindowInternal(window int, fetchPage func(start, limit int) ([]activitytypes.Activity, int64, error)) ([]activitytypes.Activity, int64, error) {
+ if window == -1 {
+ return fetchPage(0, -1)
+ }
+ var rows []activitytypes.Activity
+ var total int64
+ for len(rows) < window {
+ limit := min(remoteActivityPageCeiling, window-len(rows))
+ page, pageTotal, err := fetchPage(len(rows), limit)
+ if err != nil {
+ return nil, 0, err
+ }
+ rows = append(rows, page...)
+ total = pageTotal
+ if len(page) < limit {
+ break
+ }
+ }
+ return rows, total, nil
+}
+
// ListRemoteActivities combines manager-owned summaries with agent-owned work
-// before slicing the page. The remote records must already match the filters.
-func (s *ActivityService) ListRemoteActivities(ctx context.Context, environmentID string, remote []activitytypes.Activity, params pagination.QueryParams) ([]activitytypes.Activity, pagination.Response, error) {
- all := params
- all.Start, all.Limit = 0, -1
- activities, _, err := s.ListActivitiesPaginated(ctx, environmentID, all)
+// before slicing the page. The remote records must already match the filters
+// and be the agent's newest window; remoteTotal is the agent's filtered count.
+func (s *ActivityService) ListRemoteActivities(ctx context.Context, environmentID string, remote []activitytypes.Activity, remoteTotal int64, params pagination.QueryParams) ([]activitytypes.Activity, pagination.Response, error) {
+ params = normalizeRemoteActivityParamsInternal(params)
+ activities, localTotal, err := collectActivityWindowInternal(remoteActivityWindowInternal(params), func(start, limit int) ([]activitytypes.Activity, int64, error) {
+ local := params
+ local.Start, local.Limit = start, limit
+ rows, page, err := s.ListActivitiesPaginated(ctx, environmentID, local)
+ if err != nil {
+ return nil, 0, err
+ }
+ return rows, page.TotalItems, nil
+ })
if err != nil {
return nil, pagination.Response{}, err
}
@@ -23,14 +81,7 @@ func (s *ActivityService) ListRemoteActivities(ctx context.Context, environmentI
item.EnvironmentID = environmentID
activities = append(activities, item)
}
- params.Start = max(params.Start, 0)
- if params.Limit != -1 {
- if params.Limit <= 0 {
- params.Limit = 20
- }
- params.Limit = min(params.Limit, 100)
- }
- total := int64(len(activities))
+ total := remoteTotal + localTotal
config := pagination.Config[activitytypes.Activity]{SortBindings: []pagination.SortBinding[activitytypes.Activity]{{
Fn: func(a, b activitytypes.Activity) int { return compareActivitiesInternal(a, b, params) },
}}}
diff --git a/backend/internal/dashboard/handler.go b/backend/internal/dashboard/handler.go
index 309efa7cf..a7c31bb2a 100644
--- a/backend/internal/dashboard/handler.go
+++ b/backend/internal/dashboard/handler.go
@@ -4,6 +4,7 @@ import (
"context"
stderrors "errors"
"net/http"
+ "net/url"
"time"
"emperror.dev/errors"
@@ -36,6 +37,7 @@ type DashboardHandler struct {
type GetDashboardInput struct {
EnvironmentID string `path:"id" doc:"Environment ID"`
DebugAllGood bool `query:"debugAllGood" default:"false" doc:"Debug mode: force an empty action item list"`
+ IncludeTables bool `query:"includeTables" default:"true" doc:"Include the first-page container and image tables"`
}
const (
@@ -77,7 +79,7 @@ func (h *DashboardHandler) GetDashboard(ctx context.Context, input *GetDashboard
snapshot, err := h.dashboardService.GetSnapshot(ctx, DashboardActionItemsOptions{
DebugAllGood: input.DebugAllGood,
- }, true)
+ }, input.IncludeTables)
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
@@ -290,10 +292,13 @@ func (h *DashboardHandler) runRemoteDashboardStreamPollerInternal(ctx context.Co
// survives for classification (proxyRemoteJSONInternal would translate it
// into a huma error first).
func (h *DashboardHandler) fetchRemoteDashboardSnapshotInternal(ctx context.Context, environment environment.Environment, debugAllGood bool) (*dashboardtypes.Snapshot, error) {
- path := "/api/environments/0/dashboard"
+ // The all-environments dashboard only reads the aggregate counters, so the
+ // agent is asked to leave the container/image tables out of the payload.
+ query := url.Values{"includeTables": {"false"}}
if debugAllGood {
- path += "?debugAllGood=true"
+ query.Set("debugAllGood", "true")
}
+ path := "/api/environments/0/dashboard?" + query.Encode()
var out base.ApiResponse[dashboardtypes.Snapshot]
if err := h.environmentService.ProxyJSONRequestForEnvironment(ctx, environment, http.MethodGet, path, nil, &out); err != nil {
diff --git a/backend/internal/environment/environment.go b/backend/internal/environment/environment.go
index 49a12158a..646e1eff6 100644
--- a/backend/internal/environment/environment.go
+++ b/backend/internal/environment/environment.go
@@ -23,6 +23,7 @@ import (
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
httputils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
@@ -55,12 +56,31 @@ type EnvironmentService struct {
// runtimeWatchers receive a coalesced wake-up whenever an environment's
// liveness changes. See environment_runtime_notify.go.
runtimeWatchers runtimeWatchersInternal
+
+ // syncGate skips the periodic registry/S3/repository pushes to an agent
+ // while the payload it last accepted is unchanged.
+ syncGate utils.SyncGate
}
+// SyncDeliveryExpiry bounds how long an accepted config push is trusted. An
+// agent rebuilt with a fresh data volume while its status stayed online gets
+// everything again within this window without operator action.
+const SyncDeliveryExpiry = time.Hour
+
// VariableSyncer pushes the effective global-variable set to one environment.
// Implemented by variable.VariableService.
type VariableSyncer interface {
SyncEnvironment(ctx context.Context, envID string) error
+ ForgetSyncState(envID string)
+}
+
+// ForgetSyncState makes the next sync of every resource group resend to the
+// environment regardless of whether the payload changed.
+func (s *EnvironmentService) ForgetSyncState(environmentID string) {
+ s.syncGate.Forget(environmentID)
+ if s.variableSyncer != nil {
+ s.variableSyncer.ForgetSyncState(environmentID)
+ }
}
const (
@@ -73,6 +93,7 @@ func NewEnvironmentService(db *database.DB, httpClient *http.Client, dockerServi
httpClient = http.DefaultClient
}
return &EnvironmentService{
+ syncGate: utils.SyncGate{Expiry: SyncDeliveryExpiry},
db: db,
httpClient: httpClient,
dockerService: dockerService,
@@ -375,6 +396,7 @@ func (s *EnvironmentService) DeleteEnvironment(ctx context.Context, id string, u
}
s.edgeTokens.invalidate(id)
+ s.ForgetSyncState(id)
s.remoteEnvs.remove(id)
// Create event in background
@@ -510,6 +532,7 @@ func (s *EnvironmentService) SyncResourcesToEnvironment(ctx context.Context, env
}, func(ctx context.Context) error {
var failedGroups []string
+ s.ForgetSyncState(environmentID)
if err := s.SyncRegistriesToEnvironment(ctx, environmentID); err != nil {
slog.WarnContext(ctx, "Failed to sync registries", "environmentID", environmentID, "error", err.Error())
failedGroups = append(failedGroups, "container registries")
diff --git a/backend/internal/environment/environment_health.go b/backend/internal/environment/environment_health.go
index 6e6e48432..d68089cd4 100644
--- a/backend/internal/environment/environment_health.go
+++ b/backend/internal/environment/environment_health.go
@@ -143,6 +143,7 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s
if !environment.Enabled {
return schedulertypes.Outcome{Status: schedulertypes.Canceled, Message: "Environment disabled"}, nil
}
+ wasOnline := environment.Status == string(EnvironmentStatusOnline)
status, err := s.TestConnection(ctx, envID, nil)
switch {
case err != nil:
@@ -157,6 +158,12 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s
return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
+ // An agent that just came back may have restarted with different state;
+ // resend everything once instead of trusting the last delivered payload.
+ if !wasOnline {
+ s.ForgetSyncState(envID)
+ }
+
var syncErrors []error
syncCtx, cancel := context.WithTimeout(ctx, environmentHealthCheckTimeout)
defer cancel()
diff --git a/backend/internal/environment/environment_proxy.go b/backend/internal/environment/environment_proxy.go
index c3c7b1990..6d068bbb6 100644
--- a/backend/internal/environment/environment_proxy.go
+++ b/backend/internal/environment/environment_proxy.go
@@ -479,8 +479,6 @@ func (s *EnvironmentService) fanOutSyncToEnvironment[Model any, Item any, Reques
return err
}
- slog.InfoContext(ctx, "Starting sync to environment", "kind", kind, "environmentID", environmentID, "environmentName", target.Name, "apiUrl", target.TargetURL)
-
var records []Model
if err := s.db.WithContext(ctx).Find(&records).Error; err != nil {
return errors.WrapIff(err, "failed to get %s", kind)
@@ -501,6 +499,18 @@ func (s *EnvironmentService) fanOutSyncToEnvironment[Model any, Item any, Reques
if err != nil {
return errors.WrapIf(err, "failed to marshal sync request")
}
+ unchanged, finishDelivery, err := s.syncGate.Begin(ctx, environmentID, path, reqBody)
+ if err != nil {
+ return errors.WrapIf(err, "sync cancelled while waiting for an in-flight delivery")
+ }
+ if unchanged {
+ slog.DebugContext(ctx, "Skipping sync; payload unchanged since last delivery", "kind", kind, "environmentID", environmentID, "environmentName", target.Name)
+ return nil
+ }
+ delivered := false
+ defer func() { finishDelivery(delivered) }()
+
+ slog.InfoContext(ctx, "Starting sync to environment", "kind", kind, "environmentID", environmentID, "environmentName", target.Name, "apiUrl", target.TargetURL)
reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
@@ -520,6 +530,7 @@ func (s *EnvironmentService) fanOutSyncToEnvironment[Model any, Item any, Reques
if !result.Success {
return errors.Errorf("sync failed: %s", result.Data.Message)
}
+ delivered = true
slog.InfoContext(ctx, "Successfully synced to environment", "kind", kind, "environmentID", environmentID, "environmentName", target.Name)
diff --git a/backend/internal/environment/environment_sync_test.go b/backend/internal/environment/environment_sync_test.go
index 3613f71c0..46065f6c3 100644
--- a/backend/internal/environment/environment_sync_test.go
+++ b/backend/internal/environment/environment_sync_test.go
@@ -170,3 +170,51 @@ func TestSyncRepositoriesPreservesEmptyCredentials(t *testing.T) {
service := environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)
require.NoError(t, service.SyncRepositoriesToEnvironment(t.Context(), "remote"))
}
+
+func TestSyncSkipsUnchangedPayloadUntilForgotten(t *testing.T) {
+ db := setupSyncDBInternal(t)
+ var calls atomic.Int32
+ var reject atomic.Bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ if reject.Load() {
+ http.Error(w, "agent unavailable", http.StatusInternalServerError)
+ return
+ }
+ // A write failure only means the manager already gave up on this
+ // request, which the call-count assertions below would surface.
+ _, _ = w.Write([]byte(`{"success":true}`))
+ }))
+ defer server.Close()
+ token, err := crypto.Encrypt("secret")
+ require.NoError(t, err)
+ require.NoError(t, db.Create(&registry.ContainerRegistry{BaseModel: database.BaseModel{ID: "ghcr"}, URL: "ghcr.io", RegistryType: registry.RegistryTypeGeneric, Username: "user", Token: token}).Error)
+ require.NoError(t, db.Create(&environment.Environment{BaseModel: database.BaseModel{ID: "remote"}, ApiUrl: server.URL, AccessToken: new("agent-token")}).Error)
+ service := environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)
+ sync := func() error { return service.SyncRegistriesToEnvironment(t.Context(), "remote") }
+ changeRegistry := func(url string) {
+ require.NoError(t, db.Model(&registry.ContainerRegistry{}).Where("id = ?", "ghcr").Update("url", url).Error)
+ }
+
+ require.NoError(t, sync())
+ require.NoError(t, sync())
+ require.EqualValues(t, 1, calls.Load(), "an unchanged payload is not resent")
+
+ changeRegistry("ghcr.io/v2")
+ require.NoError(t, sync())
+ require.EqualValues(t, 2, calls.Load(), "a changed payload is resent")
+
+ service.ForgetSyncState("remote")
+ require.NoError(t, sync())
+ require.EqualValues(t, 3, calls.Load(), "forgetting forces a resend")
+
+ reject.Store(true)
+ changeRegistry("ghcr.io/v3")
+ require.Error(t, sync())
+ require.EqualValues(t, 4, calls.Load())
+ reject.Store(false)
+ require.NoError(t, sync())
+ require.EqualValues(t, 5, calls.Load(), "a rejected payload is retried")
+ require.NoError(t, sync())
+ require.EqualValues(t, 5, calls.Load(), "an accepted payload is remembered")
+}
diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.go
index cd9ba7bbe..db9f6c363 100644
--- a/backend/internal/environment/handler.go
+++ b/backend/internal/environment/handler.go
@@ -942,6 +942,7 @@ func (h *EnvironmentHandler) triggerPostUpdateTasksInternal(ctx context.Context,
}
func (h *EnvironmentHandler) triggerEnvironmentResourceSyncInternal(ctx context.Context, environmentID string, environmentName string, reason string) {
+ h.environmentService.ForgetSyncState(environmentID)
detachedCtx := context.WithoutCancel(ctx)
go func(syncCtx context.Context, envID string, envName string, syncReason string) {
diff --git a/backend/internal/gitrepo/service.go b/backend/internal/gitrepo/service.go
index 05cfb859c..47e54d432 100644
--- a/backend/internal/gitrepo/service.go
+++ b/backend/internal/gitrepo/service.go
@@ -654,7 +654,7 @@ func (s *GitRepositoryService) checkRepositoryNeedsUpdate(item gitops.Repository
{&existing.SigningKey, signingKey},
{&existing.SigningKeyPassphrase, passphrase},
} {
- changed, err := applyEncryptedInternal(credential.field, credential.plaintext)
+ changed, err := utils.ApplyEncrypted(credential.field, credential.plaintext)
if err != nil {
return false, err
}
@@ -663,32 +663,6 @@ func (s *GitRepositoryService) checkRepositoryNeedsUpdate(item gitops.Repository
return needsUpdate, nil
}
-// applyEncryptedInternal stores plaintext encrypted in field only when it differs from the stored value.
-func applyEncryptedInternal(field *string, plaintext string) (bool, error) {
- if plaintext == "" {
- if *field == "" {
- return false, nil
- }
- *field = ""
- return true, nil
- }
- if *field != "" {
- current, err := crypto.Decrypt(*field)
- if err != nil {
- return false, errors.WrapIf(err, "failed to decrypt stored credential")
- }
- if current == plaintext {
- return false, nil
- }
- }
- encrypted, err := crypto.Encrypt(plaintext)
- if err != nil {
- return false, errors.WrapIf(err, "failed to encrypt credential")
- }
- *field = encrypted
- return true, nil
-}
-
func (s *GitRepositoryService) createNewRepository(ctx context.Context, item gitops.RepositorySync) error {
var encryptedToken, encryptedSSHKey string
var err error
diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.go
index c769da482..4939b48bf 100644
--- a/backend/internal/job/service_test.go
+++ b/backend/internal/job/service_test.go
@@ -56,7 +56,7 @@ func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testin
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
cfg := jobSvc.GetJobSchedules(ctx)
- require.Equal(t, "*/30 * * * * *", cfg.DockerClientRefreshInterval)
+ require.Equal(t, "0 */5 * * * *", cfg.DockerClientRefreshInterval)
require.Equal(t, "0 0 * * * *", cfg.PollingInterval)
}
@@ -112,7 +112,7 @@ func TestJobService_ListJobs_IncludesDockerClientRefreshJob(t *testing.T) {
require.True(t, refreshJob.CanRunManually)
require.Equal(t, "monitoring", refreshJob.Category)
require.Equal(t, "dockerClientRefreshInterval", refreshJob.SettingsKey)
- require.Equal(t, "*/30 * * * * *", refreshJob.Schedule)
+ require.Equal(t, "0 */5 * * * *", refreshJob.Schedule)
}
func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {
diff --git a/backend/internal/registry/service.go b/backend/internal/registry/service.go
index 7a11935ef..72bbd716b 100644
--- a/backend/internal/registry/service.go
+++ b/backend/internal/registry/service.go
@@ -1191,27 +1191,23 @@ func (s *ContainerRegistryService) checkRegistryNeedsUpdateInternal(item contain
if newType == RegistryTypeGeneric {
needsUpdate = utils.ApplyChanged(&existing.Username, mo.Some(item.Username)) || needsUpdate
- encryptedToken, err := crypto.Encrypt(item.Token)
+ tokenChanged, err := utils.ApplyEncrypted(&existing.Token, item.Token)
if err != nil {
- slog.Warn("failed to encrypt token during sync, skipping field", "registry", existing.ID, "error", err)
- } else {
- needsUpdate = utils.ApplyChanged(&existing.Token, mo.Some(encryptedToken)) || needsUpdate
+ return false, errors.WrapIff(err, "failed to apply token for registry %s", existing.ID)
}
-
- return needsUpdate, nil
+ return tokenChanged || needsUpdate, nil
}
credChanged := utils.ApplyChanged(&existing.AWSAccessKeyID, mo.Some(item.AWSAccessKeyID))
credChanged = utils.ApplyChanged(&existing.AWSRegion, mo.Some(item.AWSRegion)) || credChanged
- // Encrypt and update AWS secret if provided
+ // Update the AWS secret only when the manager sent one that differs.
if item.AWSSecretAccessKey != "" {
- encryptedSecret, err := crypto.Encrypt(item.AWSSecretAccessKey)
+ secretChanged, err := utils.ApplyEncrypted(&existing.AWSSecretAccessKey, item.AWSSecretAccessKey)
if err != nil {
- slog.Warn("failed to encrypt AWS secret during sync, skipping field", "registry", existing.ID, "error", err)
- } else {
- credChanged = utils.ApplyChanged(&existing.AWSSecretAccessKey, mo.Some(encryptedSecret)) || credChanged
+ return false, errors.WrapIff(err, "failed to apply AWS secret for registry %s", existing.ID)
}
+ credChanged = secretChanged || credChanged
}
// Invalidate cached ECR token when credentials change
diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.go
index edc80f0dd..d744bb7fa 100644
--- a/backend/internal/s3/service.go
+++ b/backend/internal/s3/service.go
@@ -11,8 +11,10 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
s3config "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ "github.com/samber/mo"
"go.getarcane.app/sys/crypto"
"gorm.io/gorm"
)
@@ -115,16 +117,19 @@ func (s *S3DestinationService) GetS3Destination(ctx context.Context, id string)
return &dto, nil
}
-func applyS3ConfigurationInternal(destination *S3Destination, configuration s3config.Configuration, encryptedSecret string) {
- destination.Name = configuration.Name
- destination.Endpoint = configuration.Endpoint
- destination.Bucket = configuration.Bucket
- destination.Region = configuration.Region
- destination.AccessKeyID = configuration.AccessKeyID
- destination.SecretAccessKey = encryptedSecret
- destination.Prefix = configuration.Prefix
- destination.UseSSL = configuration.UseSSL
- destination.ForcePathStyle = configuration.ForcePathStyle
+// applyS3ConfigurationInternal copies configuration onto destination and
+// reports whether any field actually changed.
+func applyS3ConfigurationInternal(destination *S3Destination, configuration s3config.Configuration, encryptedSecret string) bool {
+ changed := utils.ApplyChanged(&destination.Name, mo.Some(configuration.Name))
+ changed = utils.ApplyChanged(&destination.Endpoint, mo.Some(configuration.Endpoint)) || changed
+ changed = utils.ApplyChanged(&destination.Bucket, mo.Some(configuration.Bucket)) || changed
+ changed = utils.ApplyChanged(&destination.Region, mo.Some(configuration.Region)) || changed
+ changed = utils.ApplyChanged(&destination.AccessKeyID, mo.Some(configuration.AccessKeyID)) || changed
+ changed = utils.ApplyChanged(&destination.SecretAccessKey, mo.Some(encryptedSecret)) || changed
+ changed = utils.ApplyChanged(&destination.Prefix, mo.Some(configuration.Prefix)) || changed
+ changed = utils.ApplyChanged(&destination.UseSSL, mo.Some(configuration.UseSSL)) || changed
+ changed = utils.ApplyChanged(&destination.ForcePathStyle, mo.Some(configuration.ForcePathStyle)) || changed
+ return changed
}
func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input backuptypes.CreateS3Destination) (*backuptypes.S3Destination, error) {
@@ -270,23 +275,29 @@ func (s *S3DestinationService) SyncS3Destinations(ctx context.Context, destinati
if err := configuration.Validate(true); err != nil {
return fmt.Errorf("invalid S3 destination %s: %w", configuration.ID, err)
}
- encryptedSecret, err := crypto.Encrypt(configuration.SecretAccessKey)
- if err != nil {
- return fmt.Errorf("failed to encrypt S3 secret access key for %s: %w", configuration.ID, err)
- }
destination, exists := existingByID[configuration.ID]
if !exists {
destination = &S3Destination{ID: configuration.ID}
}
- applyS3ConfigurationInternal(destination, configuration, encryptedSecret)
- if !item.CreatedAt.IsZero() {
+ // The manager resends the full set every health tick; unchanged rows
+ // keep their ciphertext and skip the write.
+ secretChanged, err := utils.ApplyEncrypted(&destination.SecretAccessKey, configuration.SecretAccessKey)
+ if err != nil {
+ return fmt.Errorf("failed to apply S3 secret access key for %s: %w", configuration.ID, err)
+ }
+ changed := applyS3ConfigurationInternal(destination, configuration, destination.SecretAccessKey) || secretChanged || !exists
+ if !item.CreatedAt.IsZero() && !destination.CreatedAt.Equal(item.CreatedAt) {
destination.CreatedAt = item.CreatedAt
+ changed = true
}
- if item.UpdatedAt != nil {
+ if item.UpdatedAt != nil && (destination.UpdatedAt == nil || !destination.UpdatedAt.Equal(*item.UpdatedAt)) {
destination.UpdatedAt = item.UpdatedAt
+ changed = true
}
- if err := s.db.WithContext(ctx).Save(destination).Error; err != nil {
- return fmt.Errorf("failed to sync S3 destination %s: %w", configuration.ID, err)
+ if changed {
+ if err := s.db.WithContext(ctx).Save(destination).Error; err != nil {
+ return fmt.Errorf("failed to sync S3 destination %s: %w", configuration.ID, err)
+ }
}
syncedIDs[configuration.ID] = struct{}{}
}
diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.go
index 678ebc009..a3c76e8b6 100644
--- a/backend/internal/settings/service.go
+++ b/backend/internal/settings/service.go
@@ -209,7 +209,7 @@ func DefaultSettingsConfig() *Settings {
PollingEnabled: SettingVariable{Value: "true"},
PollingInterval: SettingVariable{Value: "0 0 * * * *"},
ImageEventWatcherEnabled: SettingVariable{Value: "false"},
- DockerClientRefreshInterval: SettingVariable{Value: "*/30 * * * * *"},
+ DockerClientRefreshInterval: SettingVariable{Value: "0 */5 * * * *"},
EventCleanupInterval: SettingVariable{Value: "0 0 */6 * * *"},
ExpiredSessionsCleanupInterval: SettingVariable{Value: "0 0 0 * * *"},
ActivityHistoryRetentionDays: SettingVariable{Value: "30"},
@@ -231,7 +231,7 @@ func DefaultSettingsConfig() *Settings {
PruneBuildCacheMode: SettingVariable{Value: "none"},
PruneBuildCacheUntil: SettingVariable{Value: ""},
AutoHealEnabled: SettingVariable{Value: "false"},
- AutoHealInterval: SettingVariable{Value: "*/30 * * * * *"},
+ AutoHealInterval: SettingVariable{Value: "0 */5 * * * *"},
AutoHealExcludedContainers: SettingVariable{Value: ""},
AutoHealMaxRestarts: SettingVariable{Value: "5"},
AutoHealRestartWindow: SettingVariable{Value: "30"},
@@ -764,6 +764,13 @@ func (s *SettingsService) persistSettings(ctx context.Context, values []SettingV
})
}
+// retiredSettingValuesInternal lists former defaults that startup replaces
+// with the current default when an install still holds them verbatim.
+var retiredSettingValuesInternal = map[string][]string{
+ "dockerClientRefreshInterval": {"*/30 * * * * *"},
+ "autoHealInterval": {"*/30 * * * * *"},
+}
+
func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {
_, err := s.writes.Execute(ctx, "ensure default settings", func(writeCtx context.Context) (actors.NoPayload, error) {
defaultSettings := s.getDefaultSettings()
@@ -781,6 +788,10 @@ func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {
}
case err != nil:
return errors.WrapIff(err, "failed to check for existing setting %s", defaultSetting.Key)
+ case slices.Contains(retiredSettingValuesInternal[defaultSetting.Key], existing.Value):
+ if err := tx.Model(&SettingVariable{}).Where("key = ?", defaultSetting.Key).Update("value", defaultSetting.Value).Error; err != nil {
+ return errors.WrapIff(err, "failed to replace retired default for setting %s", defaultSetting.Key)
+ }
}
}
return nil
diff --git a/backend/internal/settings/service_test.go b/backend/internal/settings/service_test.go
index ff5a81d52..a9a1b99e8 100644
--- a/backend/internal/settings/service_test.go
+++ b/backend/internal/settings/service_test.go
@@ -78,6 +78,23 @@ func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {
require.NoError(t, err)
}
+func TestSettingsService_EnsureDefaultSettings_ReplacesRetiredDefaults(t *testing.T) {
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, db.Create(&SettingVariable{Key: "dockerClientRefreshInterval", Value: "*/30 * * * * *"}).Error)
+ require.NoError(t, db.Create(&SettingVariable{Key: "autoHealInterval", Value: "*/10 * * * * *"}).Error)
+
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+
+ var retired, custom SettingVariable
+ require.NoError(t, db.First(&retired, "key = ?", "dockerClientRefreshInterval").Error)
+ require.Equal(t, "0 */5 * * * *", retired.Value, "the retired default moves to the current default")
+ require.NoError(t, db.First(&custom, "key = ?", "autoHealInterval").Error)
+ require.Equal(t, "*/10 * * * * *", custom.Value, "a customised value is left alone")
+}
+
func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
diff --git a/backend/internal/variable/variable.go b/backend/internal/variable/variable.go
index 770697ca2..6cba5fef7 100644
--- a/backend/internal/variable/variable.go
+++ b/backend/internal/variable/variable.go
@@ -56,10 +56,14 @@ type VariableService struct {
statusMu sync.RWMutex
syncStatus map[string]env.EnvironmentSyncStatus
+
+ // syncGate skips the agent PUT when the effective variable set is unchanged.
+ syncGate utils.SyncGate
}
func NewVariableService(db *database.DB, environmentService *environment.EnvironmentService, settingsService *settings.SettingsService, kvService *kv.KVService) *VariableService {
service := &VariableService{
+ syncGate: utils.SyncGate{Expiry: environment.SyncDeliveryExpiry},
db: db,
settingsService: settingsService,
kvService: kvService,
@@ -317,6 +321,11 @@ func (s *VariableService) SyncEnvironment(ctx context.Context, envID string) err
return err
}
+// ForgetSyncState makes the next SyncEnvironment resend regardless of change.
+func (s *VariableService) ForgetSyncState(envID string) {
+ s.syncGate.Forget(envID)
+}
+
func (s *VariableService) syncEnvironmentInternal(ctx context.Context, envID string) error {
if envID != environment.LocalEnvironmentID {
if err := s.importRemoteLegacyVarsOnceInternal(ctx, envID); err != nil {
@@ -337,11 +346,20 @@ func (s *VariableService) syncEnvironmentInternal(ctx context.Context, envID str
if err != nil {
return errors.WrapIf(err, "failed to marshal variables for sync")
}
+ unchanged, finishDelivery, err := s.syncGate.Begin(ctx, envID, agentVariablesPath, body)
+ if err != nil {
+ return errors.WrapIf(err, "variable sync cancelled while waiting for an in-flight delivery")
+ }
+ if unchanged {
+ return nil
+ }
var out struct {
Success bool `json:"success"`
}
- return s.proxyEnvironmentJSON(ctx, envID, http.MethodPut, agentVariablesPath, body, &out)
+ err = s.proxyEnvironmentJSON(ctx, envID, http.MethodPut, agentVariablesPath, body, &out)
+ finishDelivery(err == nil)
+ return err
}
// syncTargetsInternal returns the local environment plus every enabled
@@ -362,7 +380,13 @@ func (s *VariableService) syncTargetsInternal(ctx context.Context) []string {
// enabled remote environment in parallel, waiting for all of them. Failures
// are recorded per environment and never abort the other pushes.
func (s *VariableService) SyncAll(ctx context.Context) []env.EnvironmentSyncStatus {
- s.syncEnvironmentsInternal(ctx, s.syncTargetsInternal(ctx))
+ targets := s.syncTargetsInternal(ctx)
+ // An explicit sync exists to repair agents, so every target is resent even
+ // when the manager's payload has not changed.
+ for _, envID := range targets {
+ s.ForgetSyncState(envID)
+ }
+ s.syncEnvironmentsInternal(ctx, targets)
return s.SyncStatuses()
}
@@ -531,10 +555,7 @@ func (s *VariableService) WriteLocalEnvFile(ctx context.Context, vars []env.Vari
var builder strings.Builder
builder.WriteString("# Global Environment Variables\n")
- builder.WriteString("# These variables are available to all projects\n")
- builder.WriteString("# Last updated: ")
- builder.WriteString(time.Now().Format(time.RFC3339))
- builder.WriteString("\n\n")
+ builder.WriteString("# These variables are available to all projects\n\n")
for _, v := range vars {
if strings.TrimSpace(v.Key) == "" {
diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.go
index 09c855830..077f0763a 100644
--- a/backend/internal/volume/handler.go
+++ b/backend/internal/volume/handler.go
@@ -554,6 +554,7 @@ func (h *VolumeHandler) UpdateBackupPolicy(ctx context.Context, input *UpdateVol
// Destinations are reconciled before every policy write so the agent
// can validate S3 references. Only batches that actually need S3
// fail on a sync error; local-only edits proceed.
+ h.environmentService.ForgetSyncState(input.EnvironmentID)
if syncErr := h.environmentService.SyncS3DestinationsToEnvironment(activityCtx, input.EnvironmentID); syncErr != nil {
if hasS3 {
errorStatus = http.StatusBadGateway
diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.go
index a1e0d2601..6efe44b86 100644
--- a/backend/pkg/fswatch/watcher.go
+++ b/backend/pkg/fswatch/watcher.go
@@ -253,7 +253,7 @@ func (fw *Watcher) drainPendingPathsInternal() []string {
// It does NOT decide whether the event should trigger a sync — that is
// shouldHandleEventInternal's job.
func (fw *Watcher) handleEventInternal(ctx context.Context, event fsnotify.Event) {
- if event.Has(fsnotify.Create) {
+ if event.Has(fsnotify.Create) && !projects.IsIgnoredProjectDirName(filepath.Base(event.Name)) {
logicalName := fw.logicalPathForWatchEventInternal(event.Name)
if fw.isWatchableDirectory(event.Name) {
if fw.shouldWatchDir(logicalName) {
@@ -283,6 +283,12 @@ func (fw *Watcher) shouldHandleEventInternal(event fsnotify.Event) bool {
if !event.Has(fsnotify.Write) && !event.Has(fsnotify.Create) && !event.Has(fsnotify.Rename) && !event.Has(fsnotify.Remove) && !event.Has(fsnotify.Chmod) {
return false
}
+ // Arcane's own scratch directories (update backups, GitOps staging) churn
+ // compose files that must not re-sync the live projects.
+ rel, err := filepath.Rel(fw.watchedPath, fw.logicalPathForWatchEventInternal(event.Name))
+ if err == nil && projects.PathContainsIgnoredDirectory(rel) {
+ return false
+ }
return projects.IsProjectFile(filepath.Base(event.Name))
}
@@ -332,6 +338,9 @@ func (fw *Watcher) addExistingDirectoriesRecursiveInternal(path string, logicalP
}
for _, entry := range entries {
+ if projects.IsIgnoredProjectDirName(entry.Name()) {
+ continue
+ }
childPath := filepath.Join(path, entry.Name())
if !projects.IsProjectDirectoryEntry(entry, childPath, fw.followSymlinks) {
continue
diff --git a/backend/pkg/projects/discovery.go b/backend/pkg/projects/discovery.go
index 658b5d418..f22601835 100644
--- a/backend/pkg/projects/discovery.go
+++ b/backend/pkg/projects/discovery.go
@@ -105,7 +105,7 @@ func DiscoverProjectDirectories(ctx context.Context, root string, followSymlinks
func walkProjectDirectoriesInternal(ctx context.Context, root, path string, isRoot bool, currentDepth int, maxDepth int, followSymlinks bool, ancestors map[string]struct{}, discovered *[]DiscoveredProjectDir) error {
if !isRoot {
name := filepath.Base(path)
- if IsInternalScratchDirName(name) || IsFilesystemSnapshotDirName(name) {
+ if IsIgnoredProjectDirName(name) {
return nil
}
}
@@ -229,8 +229,18 @@ func IsFilesystemSnapshotDirName(name string) bool {
// filesystem snapshot/trash directory name, i.e. the path points into a
// point-in-time copy rather than a live project directory.
func PathContainsSnapshotDirectory(relPath string) bool {
+ return pathHasSegmentInternal(relPath, IsFilesystemSnapshotDirName)
+}
+
+// PathContainsIgnoredDirectory reports whether any segment of relPath is a
+// directory that project discovery skips (see IsIgnoredProjectDirName).
+func PathContainsIgnoredDirectory(relPath string) bool {
+ return pathHasSegmentInternal(relPath, IsIgnoredProjectDirName)
+}
+
+func pathHasSegmentInternal(relPath string, match func(string) bool) bool {
for segment := range strings.SplitSeq(filepath.ToSlash(relPath), "/") {
- if IsFilesystemSnapshotDirName(segment) {
+ if match(segment) {
return true
}
}
@@ -262,6 +272,12 @@ func IsInternalScratchDirName(name string) bool {
return IsGitOpsScratchDirName(name)
}
+// IsIgnoredProjectDirName reports whether discovery and the filesystem watcher
+// must skip name: Arcane scratch directories and filesystem snapshot directories.
+func IsIgnoredProjectDirName(name string) bool {
+ return IsInternalScratchDirName(name) || IsFilesystemSnapshotDirName(name)
+}
+
// ComposeContentProjectName returns the normalized top-level `name:` from
// compose YAML content, or "" when the key is absent or unusable. Interpolated
// names (containing `${`) are treated as absent so the backend and the
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index aa8c5c703..66c319735 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -31,6 +31,7 @@ import (
)
const AutoHealJobName = "auto-heal"
+const autoHealDefaultSchedule = "0 */5 * * * *"
const autoHealInspectConcurrency = 4
const autoHealAdmissionScopeInternal = "auto-heal"
@@ -88,15 +89,15 @@ func (j *AutoHealJob) ShouldSchedule(ctx context.Context) bool {
}
func (j *AutoHealJob) Schedule(ctx context.Context) string {
- schedule := j.settingsService.GetStringSetting(ctx, "autoHealInterval", "*/30 * * * * *")
+ schedule := j.settingsService.GetStringSetting(ctx, "autoHealInterval", autoHealDefaultSchedule)
if schedule == "" {
- schedule = "*/30 * * * * *"
+ schedule = autoHealDefaultSchedule
}
parser := scheduleutil.Parser()
if _, err := parser.Parse(schedule); err != nil {
slog.WarnContext(ctx, "Invalid cron expression for auto-heal, using default", "invalid_schedule", schedule, "error", err)
- return "*/30 * * * * *"
+ return autoHealDefaultSchedule
}
return schedule
diff --git a/backend/pkg/scheduler/docker_client_refresh_job.go b/backend/pkg/scheduler/docker_client_refresh_job.go
index 5dc5d924d..c65b4334e 100644
--- a/backend/pkg/scheduler/docker_client_refresh_job.go
+++ b/backend/pkg/scheduler/docker_client_refresh_job.go
@@ -13,7 +13,7 @@ import (
)
const DockerClientRefreshJobName = "docker-client-refresh"
-const dockerClientRefreshDefaultSchedule = "*/30 * * * * *"
+const dockerClientRefreshDefaultSchedule = "0 */5 * * * *"
// DockerClientRefreshJob keeps the cached Docker client aligned with the daemon
// API version after daemon restarts or upgrades.
diff --git a/backend/pkg/utils/fingerprint.go b/backend/pkg/utils/fingerprint.go
index 123646f49..c1fff4bdb 100644
--- a/backend/pkg/utils/fingerprint.go
+++ b/backend/pkg/utils/fingerprint.go
@@ -1,9 +1,12 @@
package utils
import (
+ "context"
+ "crypto/sha256"
"encoding/binary"
"hash"
"hash/fnv"
+ "sync"
"time"
)
@@ -115,3 +118,73 @@ func (f *Fingerprint) Slice[T any](items []T, write func(*Fingerprint, *T)) *Fin
}
return f
}
+
+// SyncGate remembers the SHA-256 of the last payload delivered per scope and
+// key so periodic re-syncs skip the network when nothing changed, and
+// serializes deliveries for one key so overlapping pushes reach the target in
+// the order they are remembered. Only the digest is retained. Expiry bounds
+// how long a delivery is trusted (zero means forever) so a target rebuilt
+// behind an unchanged status still converges. The zero value is ready to use.
+type SyncGate struct {
+ Expiry time.Duration
+
+ mu sync.Mutex
+ locks KeyedMutex
+ sent map[string]map[string]syncGateEntryInternal
+ epochs map[string]uint64
+}
+
+type syncGateEntryInternal struct {
+ digest [sha256.Size]byte
+ sentAt time.Time
+}
+
+// Begin reports whether body is already delivered for scope and key. When it
+// is not, the key stays locked until finish runs; finish(true) remembers the
+// delivery unless the scope was forgotten in between. Callers must invoke
+// finish exactly once. Waiting for an in-flight delivery ends with ctx.
+func (g *SyncGate) Begin(ctx context.Context, scope, key string, body []byte) (unchanged bool, finish func(delivered bool), err error) {
+ unlock, err := g.locks.LockContext(ctx, scope+"\x00"+key)
+ if err != nil {
+ return false, nil, err
+ }
+ g.mu.Lock()
+ epoch := g.epochs[scope]
+ entry, ok := g.sent[scope][key]
+ unchanged = ok && entry.digest == sha256.Sum256(body) && (g.Expiry <= 0 || time.Since(entry.sentAt) < g.Expiry)
+ g.mu.Unlock()
+ if unchanged {
+ unlock()
+ return true, func(bool) {}, nil
+ }
+ return false, func(delivered bool) {
+ defer unlock()
+ if !delivered {
+ return
+ }
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if g.epochs[scope] != epoch {
+ return
+ }
+ if g.sent == nil {
+ g.sent = make(map[string]map[string]syncGateEntryInternal)
+ }
+ if g.sent[scope] == nil {
+ g.sent[scope] = make(map[string]syncGateEntryInternal)
+ }
+ g.sent[scope][key] = syncGateEntryInternal{digest: sha256.Sum256(body), sentAt: time.Now()}
+ }, nil
+}
+
+// Forget drops every remembered payload for scope so the next sync resends,
+// and discards the outcome of deliveries still in flight for it.
+func (g *SyncGate) Forget(scope string) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ delete(g.sent, scope)
+ if g.epochs == nil {
+ g.epochs = make(map[string]uint64)
+ }
+ g.epochs[scope]++
+}
diff --git a/backend/pkg/utils/fingerprint_test.go b/backend/pkg/utils/fingerprint_test.go
new file mode 100644
index 000000000..c2b43a115
--- /dev/null
+++ b/backend/pkg/utils/fingerprint_test.go
@@ -0,0 +1,110 @@
+package utils
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func deliverInternal(gate *SyncGate, scope, key string, body []byte) bool {
+ unchanged, finish, err := gate.Begin(context.Background(), scope, key, body)
+ if err != nil {
+ panic(err)
+ }
+ finish(!unchanged)
+ return unchanged
+}
+
+func TestSyncGate_TracksLastDeliveredPayloadPerScopeAndKey(t *testing.T) {
+ var gate SyncGate
+ body := []byte(`{"registries":[]}`)
+
+ require.False(t, deliverInternal(&gate, "env", "/registries", body), "nothing delivered yet")
+ require.True(t, deliverInternal(&gate, "env", "/registries", body))
+ require.False(t, deliverInternal(&gate, "env", "/registries", []byte(`{"registries":[{"id":"a"}]}`)))
+ require.False(t, deliverInternal(&gate, "env", "/s3", body), "keys are independent")
+ require.False(t, deliverInternal(&gate, "other", "/registries", body), "scopes are independent")
+
+ gate.Forget("env")
+ require.False(t, deliverInternal(&gate, "env", "/registries", body), "forget drops every key in the scope")
+ gate.Forget("missing")
+}
+
+func TestSyncGate_FailedDeliveryIsNotRemembered(t *testing.T) {
+ var gate SyncGate
+ body := []byte(`{"registries":[]}`)
+ unchanged, finish, err := gate.Begin(context.Background(), "env", "/registries", body)
+ require.NoError(t, err)
+ require.False(t, unchanged)
+ finish(false)
+ require.False(t, deliverInternal(&gate, "env", "/registries", body))
+}
+
+func TestSyncGate_ForgetDiscardsInFlightDelivery(t *testing.T) {
+ var gate SyncGate
+ body := []byte(`{"registries":[]}`)
+
+ _, finish, err := gate.Begin(context.Background(), "env", "/registries", body)
+ require.NoError(t, err)
+ gate.Forget("env")
+ finish(true)
+ require.False(t, deliverInternal(&gate, "env", "/registries", body), "a delivery that started before Forget must not count")
+ require.True(t, deliverInternal(&gate, "env", "/registries", body))
+}
+
+func TestSyncGate_SerializesDeliveriesPerKey(t *testing.T) {
+ var gate SyncGate
+ older, newer := []byte(`{"v":1}`), []byte(`{"v":2}`)
+ _, finishOlder, err := gate.Begin(context.Background(), "env", "/registries", older)
+ require.NoError(t, err)
+
+ started := make(chan struct{})
+ done := make(chan bool)
+ go func() {
+ close(started)
+ unchanged, finish, err := gate.Begin(context.Background(), "env", "/registries", newer)
+ if err != nil {
+ t.Error(err)
+ done <- false
+ return
+ }
+ finish(true)
+ done <- unchanged
+ }()
+ <-started
+ select {
+ case <-done:
+ t.Fatal("the second delivery must wait for the first to finish")
+ case <-time.After(50 * time.Millisecond):
+ }
+ finishOlder(true)
+ require.False(t, <-done)
+ require.True(t, deliverInternal(&gate, "env", "/registries", newer), "the last delivery wins")
+}
+
+func TestSyncGate_BeginStopsWaitingWhenContextEnds(t *testing.T) {
+ var gate SyncGate
+ body := []byte(`{"v":1}`)
+ _, finish, err := gate.Begin(context.Background(), "env", "/registries", body)
+ require.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ _, _, err = gate.Begin(ctx, "env", "/registries", body)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+
+ finish(true)
+ require.True(t, deliverInternal(&gate, "env", "/registries", body), "the key is usable after an abandoned wait")
+}
+
+func TestSyncGate_ExpiryForcesPeriodicResend(t *testing.T) {
+ gate := SyncGate{Expiry: time.Hour}
+ body := []byte(`{"registries":[]}`)
+ require.False(t, deliverInternal(&gate, "env", "/registries", body))
+ require.True(t, deliverInternal(&gate, "env", "/registries", body))
+
+ gate.sent["env"]["/registries"] = syncGateEntryInternal{digest: gate.sent["env"]["/registries"].digest, sentAt: time.Now().Add(-2 * time.Hour)}
+ require.False(t, deliverInternal(&gate, "env", "/registries", body), "an old delivery is no longer trusted")
+}
diff --git a/backend/pkg/utils/keyedmutex.go b/backend/pkg/utils/keyedmutex.go
index 1858866f7..1edc4fea1 100644
--- a/backend/pkg/utils/keyedmutex.go
+++ b/backend/pkg/utils/keyedmutex.go
@@ -1,6 +1,10 @@
package utils
-import "sync"
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+)
// KeyedMutex hands out one mutex per key, so unrelated keys proceed
// concurrently while work on the same key is serialized.
@@ -65,6 +69,40 @@ func (k *KeyedMutex) Lock(key string) func() {
}
}
+// LockContext is Lock bounded by ctx. When ctx ends first the key is left
+// untouched and the waiter is released once the lock would have been taken.
+func (k *KeyedMutex) LockContext(ctx context.Context, key string) (func(), error) {
+ entry := k.reserve(key)
+ unlock := func() {
+ entry.mu.Unlock()
+ k.release(key, entry)
+ }
+ // Exactly one side claims the outcome: the worker once it holds the lock,
+ // or the caller once ctx ends. The loser is responsible for nothing, so
+ // the lock can never be left held by an exited goroutine.
+ var claimed atomic.Bool
+ acquired := make(chan struct{})
+ go func() {
+ entry.mu.Lock()
+ if !claimed.CompareAndSwap(false, true) {
+ unlock()
+ return
+ }
+ close(acquired)
+ }()
+ select {
+ case <-acquired:
+ return unlock, nil
+ case <-ctx.Done():
+ if !claimed.CompareAndSwap(false, true) {
+ // The worker claimed first; wait for the handoff and give it back.
+ <-acquired
+ unlock()
+ }
+ return nil, ctx.Err()
+ }
+}
+
// RLock blocks until the key's mutex is held for reading and returns its
// unlock function. Multiple readers for the same key may proceed concurrently.
func (k *KeyedMutex) RLock(key string) func() {
diff --git a/backend/pkg/utils/update_value_util.go b/backend/pkg/utils/update_value_util.go
index 5fd7b1725..9826ca5cb 100644
--- a/backend/pkg/utils/update_value_util.go
+++ b/backend/pkg/utils/update_value_util.go
@@ -3,7 +3,9 @@ package utils
import (
"slices"
+ "emperror.dev/errors"
"github.com/samber/mo"
+ "go.getarcane.app/sys/crypto"
)
// ApplyChanged updates target when value is present and differs from the current value.
@@ -38,3 +40,28 @@ func ApplyNullable[T comparable](target **T, value mo.Option[T]) bool {
*target = value.ToPointer()
return true
}
+
+// ApplyEncrypted stores plaintext encrypted in field only when it differs from
+// the stored value, so unchanged credentials keep their ciphertext. An empty
+// plaintext clears the field. A stored value that no longer decrypts (for
+// example after a key change) is replaced so the source of truth wins.
+func ApplyEncrypted(field *string, plaintext string) (bool, error) {
+ if plaintext == "" {
+ if *field == "" {
+ return false, nil
+ }
+ *field = ""
+ return true, nil
+ }
+ if *field != "" {
+ if current, err := crypto.Decrypt(*field); err == nil && current == plaintext {
+ return false, nil
+ }
+ }
+ encrypted, err := crypto.Encrypt(plaintext)
+ if err != nil {
+ return false, errors.WrapIf(err, "failed to encrypt credential")
+ }
+ *field = encrypted
+ return true, nil
+}
diff --git a/frontend/src/lib/stores/json-line-stream.svelte.ts b/frontend/src/lib/stores/json-line-stream.svelte.ts
index 3c98e97cf..5c20d30bd 100644
--- a/frontend/src/lib/stores/json-line-stream.svelte.ts
+++ b/frontend/src/lib/stores/json-line-stream.svelte.ts
@@ -64,6 +64,8 @@ export function createJSONLineStream<TEvent extends JSONLineEventBase>(config: J
// Closing a tab leaves teardown to whenever the browser gets around to
// dropping the socket. Until it does, the server keeps writing heartbeats
// into a connection nobody is reading — so tear down explicitly instead.
+ // A hidden tab is torn down the same way: every open stream makes the
+ // manager poll each remote agent, which nobody is looking at.
function watchPageLifecycle() {
if (!browser || removePageLifecycleListeners) {
return;
@@ -76,12 +78,23 @@ export function createJSONLineStream<TEvent extends JSONLineEventBase>(config: J
void connectStream(nextGeneration());
}
};
+ const onVisibilityChange = () => {
+ if (document.hidden) {
+ abortStream();
+ return;
+ }
+ if (started && !streamAbortController && !_streamFailed) {
+ void connectStream(nextGeneration());
+ }
+ };
window.addEventListener('pagehide', onPageHide);
window.addEventListener('pageshow', onPageShow);
+ document.addEventListener('visibilitychange', onVisibilityChange);
removePageLifecycleListeners = () => {
window.removeEventListener('pagehide', onPageHide);
window.removeEventListener('pageshow', onPageShow);
+ document.removeEventListener('visibilitychange', onVisibilityChange);
};
}
@@ -89,6 +102,10 @@ export function createJSONLineStream<TEvent extends JSONLineEventBase>(config: J
if (!browser || !isCurrentGeneration(generation)) {
return;
}
+ // Reconnects and restarts in a background tab wait for visibilitychange.
+ if (document.hidden) {
+ return;
+ }
// Overlapping connects would otherwise strand the previous controller:
// abortStream() only ever reaches the newest one, so the older request