Skip to content

Modern Docker Management, Designed for Everyone

fix: prevent image update checks from getting stuck in a running state (#3327)

Kyle Mendell6617ad6fd5af
diff --git a/AGENTS.md b/AGENTS.md
index fa107e395..cbd89fcc3 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -14,7 +14,7 @@ Arcane is a Docker management UI: **Go backend** (Echo + Huma v2), **SvelteKit v
## ⚠️ Golden Rules — READ FIRST
1. **ALWAYS update existing code in place.** Never create a new function, service, helper, or API wrapper when one already exists that does the same thing. Find the existing code and modify it.
-2. **NEVER create stub, shim, or pass-through helper functions.** If a `pkg/` helper or service method exists, call it directly. Do not write a thin wrapper that just forwards to it.
+2. **NEVER create stub, shim, or pass-through helper functions.** If a `pkg/` helper or service method exists, call it directly. Do not write a thin wrapper that just forwards to it. This includes refactors: never hollow out an existing exported function into a one-line forwarder to a new `*Internal` variant with a mode flag — keep the real body in the existing function and let the other callsite handle its own difference in behavior.
3. **NEVER duplicate functionality.** Before writing anything, search `pkg/`, `internal/services/`, and `frontend/src/lib/services/` for existing implementations.
4. **NEVER create a new API standard.** If the codebase uses Huma v2 typed handlers on Echo, do not introduce Gin, raw `http.HandlerFunc`, or untyped patterns. Extend the existing standard.
5. If you need a paragraph comment in order to justify a function, it is wrong. Fix the code.
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 387a01fd2..b5d745591 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -67,6 +67,7 @@ type registerJobsParams struct {
FilesystemWatcher *scheduler.FilesystemWatcherJob
VulnerabilityScan *scheduler.VulnerabilityScanJob
AutoHeal *scheduler.AutoHealJob
+ ActivitySweep *scheduler.ActivitySweepJob
}
func registerJobs(params registerJobsParams) {
@@ -107,6 +108,9 @@ func registerJobs(params registerJobsParams) {
// and is only rebound on settings changes below.
params.Scheduler.RegisterJob(params.VulnerabilityScan)
params.Scheduler.RegisterJob(params.AutoHeal)
+ // Internal self-healing sweep (managers and agents alike); intentionally
+ // absent from job_metadata so it stays out of the Jobs UI.
+ params.Scheduler.RegisterJob(params.ActivitySweep)
// GitOps sync and environment health are no longer single global jobs; each
// entity registers its own dynamic job.
diff --git a/backend/internal/common/errors.go b/backend/internal/common/errors.go
index b969eb120..528473682 100644
--- a/backend/internal/common/errors.go
+++ b/backend/internal/common/errors.go
@@ -2086,3 +2086,20 @@ type DashboardSnapshotUnavailableError struct{}
func (e *DashboardSnapshotUnavailableError) Error() string {
return "dashboard snapshot not available"
}
+
+// ActivitySlotWaitTimeoutError is returned when a queued activity gave up
+// waiting for a per-environment concurrency slot, typically because
+// long-running work (e.g. an auto-update run) is holding every slot.
+type ActivitySlotWaitTimeoutError struct{}
+
+func (e *ActivitySlotWaitTimeoutError) Error() string {
+ return "timed out waiting for a free activity slot; other long-running activities are holding all slots"
+}
+
+// ImageScanInProgressError is returned when a manual image update check is
+// requested while another scan is already running.
+type ImageScanInProgressError struct{}
+
+func (e *ImageScanInProgressError) Error() string {
+ return "an image update check is already in progress"
+}
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index 3d9c7053f..b73851b7f 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -77,5 +77,6 @@ var JobOptions = fx.Options(
provideFilesystemWatcherJobInternal,
scheduler.NewVulnerabilityScanJob,
scheduler.NewAutoHealJob,
+ scheduler.NewActivitySweepJob,
),
)
diff --git a/backend/internal/services/activity_service.go b/backend/internal/services/activity_service.go
index c1fa4d1e4..8d5d81333 100644
--- a/backend/internal/services/activity_service.go
+++ b/backend/internal/services/activity_service.go
@@ -11,9 +11,11 @@ import (
"sync"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
@@ -26,6 +28,10 @@ const (
defaultActivityHistoryLimit = 1000
defaultActivityMessages = 500
staleImageUpdateCheckAge = 6 * time.Hour
+ // abandonedActivityGrace is how old an untracked queued/running activity
+ // must be before the periodic sweep fails it. It covers the window between
+ // StartActivity's row creation and the worker's Track call.
+ abandonedActivityGrace = 2 * time.Minute
)
type ActivityService struct {
@@ -397,6 +403,22 @@ func (s *ActivityService) AwaitActivitySlot(ctx context.Context, activityID, env
return nil
}
+// AwaitActivitySlotBounded waits for a concurrency slot like AwaitActivitySlot
+// but gives up after timeouts.DefaultActivitySlotWait, returning
+// ActivitySlotWaitTimeoutError so the caller fails the queued activity loudly
+// instead of parking forever behind long-running slot holders.
+func (s *ActivityService) AwaitActivitySlotBounded(ctx context.Context, activityID, environmentID string) error {
+ slotCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultActivitySlotWait)
+ defer cancel()
+ if err := s.AwaitActivitySlot(slotCtx, activityID, environmentID); err != nil {
+ if ctx.Err() == nil && errors.Is(err, context.DeadlineExceeded) {
+ return &common.ActivitySlotWaitTimeoutError{}
+ }
+ return err
+ }
+ return nil
+}
+
func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string, req UpdateActivityRequest) (*activitytypes.Activity, error) {
if err := s.checkInitInternal(); err != nil {
return nil, err
@@ -548,21 +570,37 @@ func (s *ActivityService) CompleteActivity(ctx context.Context, activityID strin
now := time.Now()
var model models.Activity
- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
- return fmt.Errorf("failed to load activity: %w", err)
- }
+ // A lost terminal write leaves the activity stuck in running forever, so
+ // retry transient DB contention (SQLITE_BUSY under bulk activity writes)
+ // instead of surfacing it once and giving up.
+ const completeWriteAttempts = 3
+ var writeErr error
+ for attempt := 1; attempt <= completeWriteAttempts; attempt++ {
+ writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
+ return fmt.Errorf("failed to load activity: %w", err)
+ }
- updates := completeActivityUpdatesInternal(model.StartedAt, status, finalMessage, errMessage, finalStep, now)
- if err := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {
- return fmt.Errorf("failed to complete activity: %w", err)
+ updates := completeActivityUpdatesInternal(model.StartedAt, status, finalMessage, errMessage, finalStep, now)
+ if err := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {
+ return fmt.Errorf("failed to complete activity: %w", err)
+ }
+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
+ return fmt.Errorf("failed to load completed activity: %w", err)
+ }
+ return nil
+ })
+ if writeErr == nil {
+ break
}
- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
- return fmt.Errorf("failed to load completed activity: %w", err)
+ if attempt == completeWriteAttempts || !isRetryableDBWriteErrorInternal(writeErr) {
+ return nil, writeErr
}
- return nil
- }); err != nil {
- return nil, err
+ slog.WarnContext(ctx, "retrying activity terminal status write", "activityId", activityID, "attempt", attempt, "error", writeErr)
+ time.Sleep(250 * time.Millisecond * time.Duration(attempt))
+ }
+ if writeErr != nil {
+ return nil, writeErr
}
if strings.TrimSpace(finalMessage) != "" {
@@ -705,6 +743,86 @@ func (s *ActivityService) FailStaleImageUpdateChecks(ctx context.Context) (int64
return failed, errors.Join(failErrs...)
}
+// isTrackedInternal reports whether activityID has a live work registration in
+// this process (i.e. a worker goroutine called Track and has not completed yet).
+func (s *ActivityService) isTrackedInternal(activityID string) bool {
+ s.runningMu.Lock()
+ defer s.runningMu.Unlock()
+ _, ok := s.running[activityID]
+ return ok
+}
+
+// FailAbandonedActivities marks queued/running activities whose worker is no
+// longer alive in this process as failed, releasing any concurrency slot they
+// still hold. Liveness comes from the running map, not age: every creation
+// path Tracks its activity right after StartActivity, and the short grace
+// period covers that create→Track window. This assumes exactly one Arcane
+// process owns the database (managers and agents each own theirs); running
+// multiple replicas against one database would make each replica sweep the
+// other's live work.
+//
+// The terminal write is status-guarded like CancelActivity's untracked
+// fallback: a worker completing concurrently wins the race and the row is
+// skipped here.
+func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, error) {
+ if s == nil || s.db == nil {
+ return 0, nil
+ }
+
+ cutoff := time.Now().Add(-abandonedActivityGrace)
+ activeStatuses := []models.ActivityStatus{models.ActivityStatusQueued, models.ActivityStatusRunning}
+ var candidates []models.Activity
+ if err := s.db.WithContext(ctx).
+ Where("status IN ? AND started_at < ?", activeStatuses, cutoff).
+ Find(&candidates).Error; err != nil {
+ return 0, fmt.Errorf("find abandoned activities: %w", err)
+ }
+
+ const message = "Activity was marked failed because its worker is no longer running"
+ errMessage := message
+ var swept int64
+ var sweepErrs []error
+ for i := range candidates {
+ activityID := candidates[i].ID
+ if s.isTrackedInternal(activityID) {
+ continue
+ }
+
+ now := time.Now()
+ var finalized models.Activity
+ lostRace := false
+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ updates := completeActivityUpdatesInternal(candidates[i].StartedAt, models.ActivityStatusFailed, message, &errMessage, nil, now)
+ result := tx.Model(&models.Activity{}).
+ Where("id = ? AND status IN ?", activityID, activeStatuses).
+ Updates(updates)
+ if result.Error != nil {
+ return fmt.Errorf("fail abandoned activity: %w", result.Error)
+ }
+ if result.RowsAffected == 0 {
+ lostRace = true
+ return nil
+ }
+ if err := tx.First(&finalized, "id = ?", activityID).Error; err != nil {
+ return fmt.Errorf("load failed abandoned activity: %w", err)
+ }
+ return nil
+ }); err != nil {
+ sweepErrs = append(sweepErrs, fmt.Errorf("sweep activity %s: %w", activityID, err))
+ continue
+ }
+ if lostRace {
+ continue
+ }
+
+ s.releaseSlotInternal(activityID)
+ s.publishActivityInternal(activityToDTOInternal(&finalized))
+ swept++
+ }
+
+ return swept, errors.Join(sweepErrs...)
+}
+
// ResolveOrphanedQueuedActivities fails any activity still queued at startup.
// Queued state is owned by a live goroutine blocked on AwaitActivitySlot, so a
// queued row after a restart can never start running.
diff --git a/backend/internal/services/activity_service_test.go b/backend/internal/services/activity_service_test.go
index 83a6d8222..20efca6df 100644
--- a/backend/internal/services/activity_service_test.go
+++ b/backend/internal/services/activity_service_test.go
@@ -602,6 +602,55 @@ func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {
require.Equal(t, models.ActivityStatusSuccess, completed.Status)
}
+func TestActivityServiceFailAbandonedActivitiesInternal(t *testing.T) {
+ ctx := context.Background()
+ db := setupActivityServiceTestDBInternal(t)
+ service := NewActivityService(db, nil)
+
+ abandoned, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImageUpdateCheck,
+ LatestMessage: "checking",
+ })
+ require.NoError(t, err)
+ tracked, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeAutoUpdate,
+ LatestMessage: "updating",
+ })
+ require.NoError(t, err)
+ fresh, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImageUpdateCheck,
+ LatestMessage: "checking",
+ })
+ require.NoError(t, err)
+
+ backdated := time.Now().Add(-10 * time.Minute)
+ require.NoError(t, db.Model(&models.Activity{}).
+ Where("id IN ?", []string{abandoned.ID, tracked.ID}).
+ Update("started_at", backdated).Error)
+ _ = service.Track(ctx, tracked.ID)
+
+ swept, err := service.FailAbandonedActivities(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, 1, swept)
+
+ var abandonedRow models.Activity
+ require.NoError(t, db.First(&abandonedRow, "id = ?", abandoned.ID).Error)
+ require.Equal(t, models.ActivityStatusFailed, abandonedRow.Status)
+ require.NotNil(t, abandonedRow.EndedAt)
+ require.Contains(t, abandonedRow.LatestMessage, "worker is no longer running")
+
+ var trackedRow models.Activity
+ require.NoError(t, db.First(&trackedRow, "id = ?", tracked.ID).Error)
+ require.Equal(t, models.ActivityStatusRunning, trackedRow.Status)
+
+ var freshRow models.Activity
+ require.NoError(t, db.First(&freshRow, "id = ?", fresh.ID).Error)
+ require.Equal(t, models.ActivityStatusRunning, freshRow.Status)
+}
+
func TestActivityServiceResolveStaleAutoUpdateActivitiesInternal(t *testing.T) {
ctx := context.Background()
db := setupActivityServiceTestDBInternal(t)
diff --git a/backend/internal/services/image_update_service.go b/backend/internal/services/image_update_service.go
index 3cfb4b2eb..a738f5391 100644
--- a/backend/internal/services/image_update_service.go
+++ b/backend/internal/services/image_update_service.go
@@ -6,6 +6,7 @@ import (
"fmt"
"log/slog"
"maps"
+ "runtime/debug"
"strings"
"sync"
"time"
@@ -191,7 +192,9 @@ func (s *ImageUpdateService) completeImageUpdateActivityInternal(ctx context.Con
}
step := "Image update check complete"
if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage, step); err != nil {
- slog.DebugContext(ctx, "failed to complete image update activity", "activityId", activityID, "error", err)
+ // A lost terminal write strands the activity in running forever, so it
+ // must be loud enough to correlate with a stuck activity panel entry.
+ slog.ErrorContext(ctx, "failed to complete image update activity", "activityId", activityID, "error", err)
}
}
@@ -204,10 +207,7 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef stri
if result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {
result.ResponseTimeMs = int(time.Since(startTime).Milliseconds())
result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, imageRef+" — digest pinned, skipped", 100, "Skipping image update check")
- if saveErr := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); saveErr != nil {
- slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", saveErr.Error())
- }
+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 100)
if s.eventService != nil {
metadata := models.JSON{
"action": "check_update",
@@ -1310,21 +1310,19 @@ func (s *ImageUpdateService) recordBatchImageCheckInternal(ctx context.Context,
}
}
-func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs []string, externalCreds []containerregistry.Credential) (map[string]*imageupdate.Response, error) {
- startBatch := time.Now()
- results := make(map[string]*imageupdate.Response, len(imageRefs))
- if len(imageRefs) == 0 {
- return results, nil
+// recordDigestPinnedSkipInternal notes a digest-pinned ref as skipped on the
+// activity stream and persists its unchanged result.
+func (s *ImageUpdateService) recordDigestPinnedSkipInternal(ctx context.Context, activityID, imageRef string, result *imageupdate.Response, progress int) {
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, imageRef+" — digest pinned, skipped", progress, "Skipping image update check")
+ if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {
+ slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())
}
+}
- activityID := s.startImageUpdateActivityInternal(ctx, fmt.Sprintf("%d images", len(imageRefs)), len(imageRefs))
- ctx = s.activityService.Track(ctx, activityID)
- activitylib.AwaitHandlerActivitySlot(ctx, s.activityService, activityID, "0")
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), 5, "Preparing image update check")
- slog.DebugContext(ctx, "Starting batch image update check", "imageCount", len(imageRefs), "externalCredCount", len(externalCreds))
-
- regRepos, initialResults, images := s.parseAndGroupImagesInternal(imageRefs)
- maps.Copy(results, initialResults)
+// recordInitialBatchResultsInternal stamps the activity ID on the parse-stage
+// results and persists digest-pinned refs as skipped, since they never reach
+// the registry check stage.
+func (s *ImageUpdateService) recordInitialBatchResultsInternal(ctx context.Context, activityID string, initialResults map[string]*imageupdate.Response) {
for imageRef, result := range initialResults {
if result == nil {
continue
@@ -1337,20 +1335,68 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs
continue
}
- s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, imageRef+" — digest pinned, skipped", 5, "Skipping image update check")
- if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {
- slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())
+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 5)
+ }
+}
+
+func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs []string, externalCreds []containerregistry.Credential) (results map[string]*imageupdate.Response, err error) {
+ startBatch := time.Now()
+ results = make(map[string]*imageupdate.Response, len(imageRefs))
+ if len(imageRefs) == 0 {
+ return results, nil
+ }
+
+ activityID := s.startImageUpdateActivityInternal(ctx, fmt.Sprintf("%d images", len(imageRefs)), len(imageRefs))
+ ctx = s.activityService.Track(ctx, activityID)
+
+ // A single deferred finalizer owns the terminal activity write so it runs
+ // on success, error, and panic alike — any early return that skipped
+ // completion would strand the row in running forever. It reads the Track
+ // ctx so a user cancellation still records a cancelled status.
+ defer func() {
+ if r := recover(); r != nil {
+ // Don't re-panic: the caller is the long-lived watcher goroutine.
+ err = fmt.Errorf("image update check panicked: %v", r)
+ slog.ErrorContext(ctx, "image update check panicked", "activityId", activityID, "panic", r, "stack", string(debug.Stack()))
+ }
+ if err != nil {
+ s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+err.Error())
+ return
+ }
+ successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)
+ finalMessage := fmt.Sprintf("Image update check completed: %d checked, %d errors", successCount, errorCount)
+ s.completeImageUpdateActivityInternal(ctx, activityID, errorCount == 0, finalMessage)
+ }()
+
+ if activityID != "" {
+ // Bounded slot wait: update-all runs share these slots and can hold
+ // them for a long time; parking here forever would wedge every future
+ // scan behind the watcher's single-flight gate. On timeout the
+ // finalizer above flips the queued row to failed.
+ if slotErr := s.activityService.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {
+ return results, slotErr
}
}
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), 5, "Preparing image update check")
+ slog.DebugContext(ctx, "Starting batch image update check", "imageCount", len(imageRefs), "externalCredCount", len(externalCreds))
- composeBuildRefs, err := s.composeBuildImageRefsInternal(ctx)
- if err != nil {
- wrappedErr := fmt.Errorf("prepare compose build image references: %w", err)
- s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+wrappedErr.Error())
- return results, wrappedErr
+ regRepos, initialResults, images := s.parseAndGroupImagesInternal(imageRefs)
+ maps.Copy(results, initialResults)
+ s.recordInitialBatchResultsInternal(ctx, activityID, initialResults)
+
+ // The aggregate scan gets its own deadline: individual registry RPCs are
+ // bounded, but the batch as a whole (including registry limiter waits)
+ // was not, and a wedged scan holds its activity slot indefinitely.
+ scanCtx, cancelScan := context.WithTimeout(ctx, timeouts.DefaultImageUpdateScan)
+ defer cancelScan()
+
+ composeBuildRefs, composeErr := s.composeBuildImageRefsInternal(scanCtx)
+ if composeErr != nil {
+ err = fmt.Errorf("prepare compose build image references: %w", composeErr)
+ return results, err
}
- resolvedCreds := s.resolveBatchCredentialsInternal(ctx, externalCreds)
+ resolvedCreds := s.resolveBatchCredentialsInternal(scanCtx, externalCreds)
slog.DebugContext(ctx, "Resolved batch registry credentials", "credentialCount", len(resolvedCreds), "registryCount", len(regRepos))
@@ -1358,25 +1404,33 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs
results: results,
total: len(images),
}
- g, groupCtx := errgroup.WithContext(ctx)
+ g, groupCtx := errgroup.WithContext(scanCtx)
g.SetLimit(10) // Limit concurrency
for _, img := range images {
- g.Go(func() error {
+ g.Go(func() (checkErr error) {
+ // x/sync's errgroup does not recover goroutine panics (they crash
+ // the process), so convert them to errors here; the deferred
+ // finalizer then records the failed terminal status.
+ defer func() {
+ if r := recover(); r != nil {
+ checkErr = fmt.Errorf("image update check panicked: %v", r)
+ slog.ErrorContext(groupCtx, "image update check worker panicked", "activityId", activityID, "imageRef", img.canonicalRef, "panic", r, "stack", string(debug.Stack()))
+ }
+ }()
return s.checkBatchImageInternal(groupCtx, activityID, resolvedCreds, img, composeBuildRefs, recorder)
})
}
- if err := g.Wait(); err != nil {
+ if err = g.Wait(); err != nil {
+ if ctx.Err() == nil && errors.Is(scanCtx.Err(), context.DeadlineExceeded) {
+ err = fmt.Errorf("image update check timed out after %s: %w", timeouts.DefaultImageUpdateScan, err)
+ }
slog.ErrorContext(ctx, "Batch check error", "error", err)
- s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+err.Error())
return results, err
}
successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)
- finalSuccess := errorCount == 0
- finalMessage := fmt.Sprintf("Image update check completed: %d checked, %d errors", successCount, errorCount)
- s.completeImageUpdateActivityInternal(ctx, activityID, finalSuccess, finalMessage)
slog.InfoContext(ctx, "Batch image update check completed",
"totalImages", len(imageRefs),
"successCount", successCount,
diff --git a/backend/internal/services/image_update_service_test.go b/backend/internal/services/image_update_service_test.go
index 547d4a067..3835c959e 100644
--- a/backend/internal/services/image_update_service_test.go
+++ b/backend/internal/services/image_update_service_test.go
@@ -849,6 +849,36 @@ func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInter
require.Contains(t, activity.LatestMessage, "0 checked, 1 errors")
}
+func TestImageUpdateService_CheckMultipleImagesPanicMarksActivityFailedInternal(t *testing.T) {
+ db := setupImageUpdateTestDB(t)
+ require.NoError(t, db.AutoMigrate(&models.Activity{}, &models.ActivityMessage{}))
+
+ settingsService := newImageUpdateTestSettingsServiceInternal("30", "30")
+ activityService := NewActivityService(db, nil)
+ dockerServer := newImageUpdateRegistryOnlyServer(t, "team/app:1.2.3", digest.FromString("unused").String())
+ defer dockerServer.Close()
+ registryService := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ panic("registry check exploded")
+ },
+ }, nil
+ }, nil)
+
+ svc := NewImageUpdateService(db, settingsService, registryService, &DockerClientService{client: newTestDockerClient(t, dockerServer)}, nil, nil, activityService)
+
+ _, err := svc.CheckMultipleImages(context.Background(), []string{"registry.example.com/team/app:1.2.3"}, nil)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "image update check panicked")
+ require.Contains(t, err.Error(), "registry check exploded")
+
+ var activity models.Activity
+ require.NoError(t, db.Where("type = ?", models.ActivityTypeImageUpdateCheck).First(&activity).Error)
+ require.Equal(t, models.ActivityStatusFailed, activity.Status)
+ require.NotNil(t, activity.EndedAt)
+ require.Contains(t, activity.LatestMessage, "Image update check failed")
+}
+
func TestImageUpdateService_GetAllImageRefsUsesDockerAPITimeoutInternal(t *testing.T) {
db := setupImageUpdateTestDB(t)
settingsService := newImageUpdateTestSettingsServiceInternal("30", "1")
diff --git a/backend/internal/services/system_service.go b/backend/internal/services/system_service.go
index d08fb17cf..a4419e06d 100644
--- a/backend/internal/services/system_service.go
+++ b/backend/internal/services/system_service.go
@@ -372,6 +372,7 @@ type startMatchingContainersOptionsInternal struct {
func (s *SystemService) startMatchingContainersInternal(ctx context.Context, environmentID string, opts startMatchingContainersOptionsInternal) (*containertypes.ActionResult, error) {
activityID := s.startSystemContainerActivityInternal(ctx, environmentID, models.ActivityTypeContainerStart, opts.ResourceName, opts.StartMessage)
+ ctx = s.activityService.Track(ctx, activityID)
containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)
if err != nil {
result := &containertypes.ActionResult{
@@ -393,6 +394,7 @@ func (s *SystemService) startMatchingContainersInternal(ctx context.Context, env
func (s *SystemService) StopAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {
activityID := s.startSystemContainerActivityInternal(ctx, environmentID, models.ActivityTypeContainerStop, "All containers", "Stopping all containers")
+ ctx = s.activityService.Track(ctx, activityID)
containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)
if err != nil {
result := &containertypes.ActionResult{
diff --git a/backend/internal/services/updater_service.go b/backend/internal/services/updater_service.go
index 8d26f5705..0a68f0caf 100644
--- a/backend/internal/services/updater_service.go
+++ b/backend/internal/services/updater_service.go
@@ -22,6 +22,7 @@ import (
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
projectspkg "github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/types/v2/updater"
@@ -152,7 +153,6 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optio
out = &updater.Result{Items: []updater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
ctx = s.trackActivityInternal(ctx, activityID)
ctx = contextWithActivityIDInternal(ctx, activityID)
- activitylib.AwaitHandlerActivitySlot(ctx, s.deps.Activity, activityID, "0")
defer func() {
if out == nil {
@@ -165,6 +165,21 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optio
s.completeAutoUpdateActivityInternal(ctx, activityID, out, err)
}()
+ if activityID != "" && s.deps.Activity != nil {
+ // Bounded slot wait: an unbounded wait behind other long-running runs
+ // would strand the queued activity row (the completion defer above
+ // flips it to failed on timeout instead).
+ if err = s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); err != nil {
+ return out, err
+ }
+ }
+
+ // The engine's per-container docker operations carry no timeouts, so cap
+ // the whole run; the Track ctx stays unbounded for the deferred completion
+ // so user cancellation is still detected there.
+ runCtx, cancelRun := context.WithTimeout(ctx, timeouts.DefaultAutoUpdateApply)
+ defer cancelRun()
+
s.recordAutoUpdateEventInternal(ctx, models.EventSeverityInfo, models.JSON{
"phase": "start",
"dryRun": options.DryRun,
@@ -176,11 +191,11 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optio
s.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Planning pending updates", "Planning updates", 5)
if len(options.ResourceIds) > 0 {
- if err = s.applyScopedUpdatesInternal(ctx, options, out); err != nil {
+ if err = s.applyScopedUpdatesInternal(runCtx, options, out); err != nil {
return out, err
}
} else {
- moduleResult, engineErr := s.engineInternal().ApplyPending(ctx, moduleOptionsFromUpdaterOptionsInternal(options))
+ moduleResult, engineErr := s.engineInternal().ApplyPending(runCtx, moduleOptionsFromUpdaterOptionsInternal(options))
if moduleResult != nil {
out = resultFromModuleInternal(moduleResult)
out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
@@ -194,7 +209,7 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optio
if !options.DryRun && s.deps.ImageUpdates != nil {
s.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Cleaning up update records", "Cleaning up", 95)
- if cleanupErr := s.deps.ImageUpdates.CleanupOrphanedRecords(ctx); cleanupErr != nil {
+ if cleanupErr := s.deps.ImageUpdates.CleanupOrphanedRecords(runCtx); cleanupErr != nil {
s.loggerInternal().WarnContext(ctx, "cleanup orphaned update records failed", "error", cleanupErr)
}
}
@@ -524,7 +539,10 @@ func (s *UpdaterService) DockerClient(ctx context.Context) (*client.Client, erro
return s.deps.Docker.GetClient(ctx)
}
-// PullImage pulls an image through Arcane's image service.
+// PullImage pulls an image through Arcane's image service. The pull is
+// bounded by the dockerImagePullTimeout setting — ImageService.PullImage does
+// not bound itself, and an unbounded engine pull would hold the auto-update
+// run (and its activity slot) indefinitely.
func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progress io.Writer) error {
if s == nil || s.deps.ImagePuller == nil {
return &common.UpdaterImageServiceUnavailableError{}
@@ -533,15 +551,22 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres
writer := activitylib.NewWriter(ctx, s.deps.Activity, activityID, progress, "Pulling updated images")
defer activitylib.FlushWriter(writer)
+ pullTimeoutSeconds := 0
+ if s.deps.Settings != nil {
+ pullTimeoutSeconds = s.deps.Settings.GetSettingsConfig().DockerImagePullTimeout.AsInt()
+ }
+ pullCtx, cancelPull := timeouts.WithTimeout(ctx, pullTimeoutSeconds, timeouts.DefaultDockerImagePull)
+ defer cancelPull()
+
if s.deps.Projects != nil {
- resolved, err := s.deps.Projects.resolveRegistryCredentialsInternal(ctx)
+ resolved, err := s.deps.Projects.resolveRegistryCredentialsInternal(pullCtx)
if err != nil {
return fmt.Errorf("resolve registry credentials: %w", err)
}
- return s.deps.ImagePuller.PullImage(ctx, imageRef, writer, s.deps.SystemUser, resolved)
+ return s.deps.ImagePuller.PullImage(pullCtx, imageRef, writer, s.deps.SystemUser, resolved)
}
- return s.deps.ImagePuller.PullImage(ctx, imageRef, writer, s.deps.SystemUser, nil)
+ return s.deps.ImagePuller.PullImage(pullCtx, imageRef, writer, s.deps.SystemUser, nil)
}
// PendingImageUpdates returns pending image update records from Arcane's database.
@@ -841,7 +866,9 @@ func (s *UpdaterService) completeAutoUpdateActivityInternal(ctx context.Context,
}
if _, err := s.deps.Activity.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {
- slog.DebugContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)
+ // A lost terminal write strands the activity in running forever, so it
+ // must be loud enough to correlate with a stuck activity panel entry.
+ slog.ErrorContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)
}
}
diff --git a/backend/internal/services/vulnerability_service.go b/backend/internal/services/vulnerability_service.go
index 3a574ec55..736e00400 100644
--- a/backend/internal/services/vulnerability_service.go
+++ b/backend/internal/services/vulnerability_service.go
@@ -3465,7 +3465,7 @@ func (s *VulnerabilityService) saveScanResultWithRetryInternal(
}
if attempt < maxAttempts {
- if !isRetryableScanSaveErrorInternal(lastErr) {
+ if !isRetryableDBWriteErrorInternal(lastErr) {
break
}
@@ -3491,7 +3491,7 @@ func (s *VulnerabilityService) saveScanResultWithRetryInternal(
return lastErr
}
-func isRetryableScanSaveErrorInternal(err error) bool {
+func isRetryableDBWriteErrorInternal(err error) bool {
if err == nil {
return false
}
diff --git a/backend/pkg/libarcane/activity/handler.go b/backend/pkg/libarcane/activity/handler.go
index 1afb66635..ecfe61ec4 100644
--- a/backend/pkg/libarcane/activity/handler.go
+++ b/backend/pkg/libarcane/activity/handler.go
@@ -175,7 +175,7 @@ func CompleteHandlerActivity(ctx context.Context, activityService Service, activ
activityCtx := utils.ActivityRuntimeContext(ctx, nil)
if _, completeErr := activityService.CompleteActivity(activityCtx, activityID, status, finalMessage, errMessage); completeErr != nil {
- slog.DebugContext(activityCtx, "failed to complete background activity", "activityId", activityID, "error", completeErr)
+ slog.WarnContext(activityCtx, "failed to complete background activity", "activityId", activityID, "error", completeErr)
}
}
diff --git a/backend/pkg/libarcane/timeouts/timeouts.go b/backend/pkg/libarcane/timeouts/timeouts.go
index 1fc74bbef..7afdb38f0 100644
--- a/backend/pkg/libarcane/timeouts/timeouts.go
+++ b/backend/pkg/libarcane/timeouts/timeouts.go
@@ -14,6 +14,20 @@ const (
DefaultRegistry = 30 * time.Second
DefaultProxyRequest = 60 * time.Second
DefaultBuildTimeout = 30 * time.Minute
+ // DefaultImageUpdateScan bounds an entire batch image update check
+ // end-to-end; individual registry RPCs are separately bounded, but the
+ // aggregate scan needs its own ceiling so a wedged batch cannot hold its
+ // activity slot forever.
+ DefaultImageUpdateScan = 15 * time.Minute
+ // DefaultActivitySlotWait bounds how long a queued activity waits for a
+ // per-environment concurrency slot before failing loudly instead of
+ // parking forever behind hung work.
+ DefaultActivitySlotWait = 2 * time.Minute
+ // DefaultAutoUpdateApply bounds an entire auto-update (update-all) run.
+ // The updater engine's per-container docker operations carry no timeouts
+ // of their own, so this is the Arcane-side backstop that keeps a hung
+ // engine op from holding an activity slot indefinitely.
+ DefaultAutoUpdateApply = 30 * time.Minute
)
func GetDuration(settingSeconds int, defaultDuration time.Duration) time.Duration {
diff --git a/backend/pkg/scheduler/activity_sweep_job.go b/backend/pkg/scheduler/activity_sweep_job.go
new file mode 100644
index 000000000..011346796
--- /dev/null
+++ b/backend/pkg/scheduler/activity_sweep_job.go
@@ -0,0 +1,42 @@
+package scheduler
+
+import (
+ "context"
+ "log/slog"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"
+)
+
+const ActivitySweepJobName = "activity-sweep"
+
+// ActivitySweepJob periodically fails queued/running activities whose worker
+// is no longer alive in this process, so a lost terminal write (crash, panic,
+// dropped DB write) cannot leave an activity stuck in running forever. It is
+// an internal job: it has no job_metadata entry and is invisible in the Jobs
+// UI.
+type ActivitySweepJob struct {
+ activityService *services.ActivityService
+}
+
+func NewActivitySweepJob(activityService *services.ActivityService) *ActivitySweepJob {
+ return &ActivitySweepJob{activityService: activityService}
+}
+
+func (j *ActivitySweepJob) Name() string {
+ return ActivitySweepJobName
+}
+
+func (j *ActivitySweepJob) Schedule(_ context.Context) string {
+ return "0 */5 * * * *"
+}
+
+func (j *ActivitySweepJob) Run(ctx context.Context) {
+ swept, err := j.activityService.FailAbandonedActivities(ctx)
+ if err != nil {
+ slog.WarnContext(ctx, "activity sweep failed", "jobName", ActivitySweepJobName, "swept", swept, "error", err)
+ return
+ }
+ if swept > 0 {
+ slog.InfoContext(ctx, "marked abandoned activities as failed", "jobName", ActivitySweepJobName, "count", swept)
+ }
+}
diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.go
index cb7f782f2..c1d5c498c 100644
--- a/backend/pkg/scheduler/auto_update_job.go
+++ b/backend/pkg/scheduler/auto_update_job.go
@@ -3,14 +3,25 @@ package scheduler
import (
"context"
"log/slog"
+ "sync/atomic"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/types/v2/updater"
)
+// pendingUpdateApplierInternal is the slice of UpdaterService the job needs,
+// kept as an interface so the overlap guard is testable with a fake.
+type pendingUpdateApplierInternal interface {
+ ApplyPending(ctx context.Context, options updater.Options) (*updater.Result, error)
+}
+
type AutoUpdateJob struct {
- updaterService *services.UpdaterService
+ updaterService pendingUpdateApplierInternal
settingsService *services.SettingsService
+ // running guards against overlapping cron ticks: an update-all run can
+ // outlast its schedule interval, and stacked runs exhaust the shared
+ // activity slots.
+ running atomic.Bool
}
func NewAutoUpdateJob(updaterService *services.UpdaterService, settingsService *services.SettingsService) *AutoUpdateJob {
@@ -47,6 +58,12 @@ func (j *AutoUpdateJob) Run(ctx context.Context) {
return
}
+ if !j.running.CompareAndSwap(false, true) {
+ slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")
+ return
+ }
+ defer j.running.Store(false)
+
slog.InfoContext(ctx, "auto-update run started")
result, err := j.updaterService.ApplyPending(ctx, updater.Options{})
diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.go
index 06a2c78c5..f370aa85e 100644
--- a/backend/pkg/scheduler/auto_update_job_test.go
+++ b/backend/pkg/scheduler/auto_update_job_test.go
@@ -2,8 +2,11 @@ package scheduler
import (
"context"
+ "sync/atomic"
"testing"
+ "time"
+ "github.com/getarcaneapp/arcane/types/v2/updater"
"github.com/stretchr/testify/require"
)
@@ -23,3 +26,53 @@ func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T)
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", false))
require.False(t, job.ShouldSchedule(ctx))
}
+
+type blockingApplierFakeInternal struct {
+ calls atomic.Int32
+ started chan struct{}
+ release chan struct{}
+}
+
+func (f *blockingApplierFakeInternal) ApplyPending(context.Context, updater.Options) (*updater.Result, error) {
+ f.calls.Add(1)
+ select {
+ case f.started <- struct{}{}:
+ default:
+ }
+ <-f.release
+ return &updater.Result{}, nil
+}
+
+func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {
+ ctx := context.Background()
+ _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
+ require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", true))
+
+ applier := &blockingApplierFakeInternal{
+ started: make(chan struct{}, 1),
+ release: make(chan struct{}),
+ }
+ job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc}
+
+ firstDone := make(chan struct{})
+ go func() {
+ job.Run(ctx)
+ close(firstDone)
+ }()
+ <-applier.started
+
+ // Overlapping tick returns immediately without a second ApplyPending.
+ job.Run(ctx)
+ require.Equal(t, int32(1), applier.calls.Load())
+
+ close(applier.release)
+ select {
+ case <-firstDone:
+ case <-time.After(time.Second):
+ t.Fatal("first auto-update run did not finish")
+ }
+
+ // The guard resets once the run finishes.
+ job.Run(ctx)
+ require.Equal(t, int32(2), applier.calls.Load())
+}
diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.go
index 17aa1dafc..4221e4080 100644
--- a/backend/pkg/scheduler/image_update_watcher.go
+++ b/backend/pkg/scheduler/image_update_watcher.go
@@ -9,6 +9,7 @@ import (
"sync/atomic"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
@@ -167,7 +168,11 @@ func (w *ImageUpdateWatcher) RefreshSchedule() {
}
}
-// RunNow performs the same full-host image scan used by automatic watcher triggers.
+// RunNow performs the same full-host image scan used by automatic watcher
+// triggers. It does not wait behind an active scan: a request during one
+// returns ImageScanInProgressError immediately instead of parking
+// (potentially forever, given the detached run-now context) on the
+// single-flight gate. The triggered-scan loop waits and retries itself.
func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil || w.metadataReady == nil {
return errors.New("image update watcher is not initialized")
@@ -179,20 +184,11 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
}
run := &imageUpdateScanRunInternal{done: make(chan struct{})}
- for {
- activeRun := w.activeRun.Load()
- if activeRun == nil && w.activeRun.CompareAndSwap(nil, run) {
- break
- }
- if activeRun == nil {
- continue
- }
-
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-activeRun.done:
+ for !w.activeRun.CompareAndSwap(nil, run) {
+ if w.activeRun.Load() != nil {
+ return &common.ImageScanInProgressError{}
}
+ // Lost the CAS to a run that has since been released; retry.
}
defer func() {
w.activeRun.Store(nil)
@@ -285,7 +281,18 @@ func (w *ImageUpdateWatcher) runTriggeredScansInternal(ctx context.Context) {
if !w.waitForDebounceInternal(ctx) {
return
}
- if err := w.RunNow(ctx); err != nil && !errors.Is(err, context.Canceled) {
+ err := w.RunNow(ctx)
+ var inProgress *common.ImageScanInProgressError
+ if errors.As(err, &inProgress) {
+ // A manual scan holds the gate; wait it out and re-queue the
+ // trigger so this scan attempt is not lost.
+ if !w.waitForActiveScanInternal(ctx) {
+ return
+ }
+ w.Trigger()
+ continue
+ }
+ if err != nil && !errors.Is(err, context.Canceled) {
slog.ErrorContext(ctx, "image update watcher scan failed", "error", err)
}
}
@@ -332,6 +339,21 @@ func (w *ImageUpdateWatcher) runScheduledPollsInternal(ctx context.Context) {
}
}
+// waitForActiveScanInternal blocks until any in-flight scan releases the
+// single-flight gate. It returns false when ctx is cancelled first.
+func (w *ImageUpdateWatcher) waitForActiveScanInternal(ctx context.Context) bool {
+ activeRun := w.activeRun.Load()
+ if activeRun == nil {
+ return true
+ }
+ select {
+ case <-ctx.Done():
+ return false
+ case <-activeRun.done:
+ return true
+ }
+}
+
func (w *ImageUpdateWatcher) waitForDebounceInternal(ctx context.Context) bool {
timer := time.NewTimer(w.debounce)
defer timer.Stop()
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index 6df45a569..06b9430f5 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -10,6 +10,7 @@ import (
"testing"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
"github.com/moby/moby/api/types/events"
@@ -162,7 +163,7 @@ func (b *lockedBufferInternal) Write(p []byte) (int, error) {
func (b *lockedBufferInternal) stringInternal() string {
b.mu.Lock()
defer b.mu.Unlock()
- return b.Buffer.String()
+ return b.String()
}
func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {
@@ -288,51 +289,64 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {
require.Eventually(t, func() bool { return scanner.countInternal() == 2 }, time.Second, 5*time.Millisecond)
}
-func TestImageUpdateWatcher_RunNowSerializesConcurrentCalls(t *testing.T) {
+func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *testing.T) {
releaseCh := make(chan struct{})
scanner := &imageUpdateScannerFakeInternal{
- startedCh: make(chan int, 2),
+ startedCh: make(chan int, 1),
releaseCh: releaseCh,
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
markImageUpdateWatcherMetadataReadyForTestInternal(watcher)
- errCh := make(chan error, 2)
- go func() { errCh <- watcher.RunNow(context.Background()) }()
+ firstErrCh := make(chan error, 1)
+ go func() { firstErrCh <- watcher.RunNow(context.Background()) }()
require.Equal(t, 1, <-scanner.startedCh)
- go func() { errCh <- watcher.RunNow(context.Background()) }()
- time.Sleep(20 * time.Millisecond)
+ var inProgress *common.ImageScanInProgressError
+ require.ErrorAs(t, watcher.RunNow(context.Background()), &inProgress)
require.Equal(t, 1, scanner.countInternal())
+
close(releaseCh)
- require.Equal(t, 2, <-scanner.startedCh)
- require.NoError(t, <-errCh)
- require.NoError(t, <-errCh)
+ require.NoError(t, <-firstErrCh)
require.Equal(t, 1, scanner.maxActiveInternal())
+
+ // The gate is free again after the first scan finishes.
+ require.NoError(t, watcher.RunNow(context.Background()))
+ require.Equal(t, 2, scanner.countInternal())
}
-func TestImageUpdateWatcher_RunNowWaiterHonorsCancellation(t *testing.T) {
+func TestImageUpdateWatcher_TriggeredScanRetriesAfterManualScanFinishes(t *testing.T) {
releaseCh := make(chan struct{})
scanner := &imageUpdateScannerFakeInternal{
- startedCh: make(chan int, 1),
+ startedCh: make(chan int, 4),
releaseCh: releaseCh,
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
- markImageUpdateWatcherMetadataReadyForTestInternal(watcher)
+ eventBus := bus.NewDockerEventBus()
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
+ startImageUpdateWatcherForTestInternal(t, watcher)
- firstErrCh := make(chan error, 1)
- go func() { firstErrCh <- watcher.RunNow(context.Background()) }()
+ // Let the startup scan finish so the gate is free.
require.Equal(t, 1, <-scanner.startedCh)
+ releaseCh <- struct{}{}
+ require.Eventually(t, func() bool { return watcher.activeRun.Load() == nil }, time.Second, time.Millisecond)
- waitingCtx, cancel := context.WithCancel(context.Background())
- cancel()
- require.ErrorIs(t, watcher.RunNow(waitingCtx), context.Canceled)
- require.Equal(t, 1, scanner.countInternal())
+ // A manual scan takes the gate, then a Docker event fires: the triggered
+ // loop must wait out the manual scan and still run its own scan after.
+ manualErrCh := make(chan error, 1)
+ go func() { manualErrCh <- watcher.RunNow(context.Background()) }()
+ require.Equal(t, 2, <-scanner.startedCh)
+ eventBus.Publish(events.Message{Type: events.ImageEventType, Action: events.ActionPull})
- close(releaseCh)
- require.NoError(t, <-firstErrCh)
+ time.Sleep(30 * time.Millisecond)
+ require.Equal(t, 2, scanner.countInternal())
+ releaseCh <- struct{}{}
+ require.NoError(t, <-manualErrCh)
+
+ require.Equal(t, 3, <-scanner.startedCh)
+ releaseCh <- struct{}{}
+ require.Equal(t, 1, scanner.maxActiveInternal())
}
func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *testing.T) {