Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: prevent image update checks from getting stuck in a running state (#3327)
Kyle Mendell6617ad6fd5af
diff --git a/AGENTS.md b/AGENTS.mdindex fa107e395..cbd89fcc3 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -14,7 +14,7 @@ Arcane is a Docker management UI: **Go backend** (Echo + Huma v2), **SvelteKit v## ⚠️ Golden Rules — READ FIRST1. **ALWAYS update existing code in place.** Never create a new function, service, helper, or API wrapper when one already exists that does the same thing. Find the existing code and modify it.-2. **NEVER create stub, shim, or pass-through helper functions.** If a `pkg/` helper or service method exists, call it directly. Do not write a thin wrapper that just forwards to it.+2. **NEVER create stub, shim, or pass-through helper functions.** If a `pkg/` helper or service method exists, call it directly. Do not write a thin wrapper that just forwards to it. This includes refactors: never hollow out an existing exported function into a one-line forwarder to a new `*Internal` variant with a mode flag — keep the real body in the existing function and let the other callsite handle its own difference in behavior.3. **NEVER duplicate functionality.** Before writing anything, search `pkg/`, `internal/services/`, and `frontend/src/lib/services/` for existing implementations.4. **NEVER create a new API standard.** If the codebase uses Huma v2 typed handlers on Echo, do not introduce Gin, raw `http.HandlerFunc`, or untyped patterns. Extend the existing standard.5. If you need a paragraph comment in order to justify a function, it is wrong. Fix the code.diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 387a01fd2..b5d745591 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -67,6 +67,7 @@ type registerJobsParams struct {FilesystemWatcher *scheduler.FilesystemWatcherJobVulnerabilityScan *scheduler.VulnerabilityScanJobAutoHeal *scheduler.AutoHealJob+ ActivitySweep *scheduler.ActivitySweepJob}func registerJobs(params registerJobsParams) {@@ -107,6 +108,9 @@ func registerJobs(params registerJobsParams) {// and is only rebound on settings changes below.params.Scheduler.RegisterJob(params.VulnerabilityScan)params.Scheduler.RegisterJob(params.AutoHeal)+ // Internal self-healing sweep (managers and agents alike); intentionally+ // absent from job_metadata so it stays out of the Jobs UI.+ params.Scheduler.RegisterJob(params.ActivitySweep)// GitOps sync and environment health are no longer single global jobs; each// entity registers its own dynamic job.diff --git a/backend/internal/common/errors.go b/backend/internal/common/errors.goindex b969eb120..528473682 100644--- a/backend/internal/common/errors.go+++ b/backend/internal/common/errors.go@@ -2086,3 +2086,20 @@ type DashboardSnapshotUnavailableError struct{}func (e *DashboardSnapshotUnavailableError) Error() string {return "dashboard snapshot not available"}++// ActivitySlotWaitTimeoutError is returned when a queued activity gave up+// waiting for a per-environment concurrency slot, typically because+// long-running work (e.g. an auto-update run) is holding every slot.+type ActivitySlotWaitTimeoutError struct{}++func (e *ActivitySlotWaitTimeoutError) Error() string {+ return "timed out waiting for a free activity slot; other long-running activities are holding all slots"+}++// ImageScanInProgressError is returned when a manual image update check is+// requested while another scan is already running.+type ImageScanInProgressError struct{}++func (e *ImageScanInProgressError) Error() string {+ return "an image update check is already in progress"+}diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex 3d9c7053f..b73851b7f 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -77,5 +77,6 @@ var JobOptions = fx.Options(provideFilesystemWatcherJobInternal,scheduler.NewVulnerabilityScanJob,scheduler.NewAutoHealJob,+ scheduler.NewActivitySweepJob,),)diff --git a/backend/internal/services/activity_service.go b/backend/internal/services/activity_service.goindex c1fa4d1e4..8d5d81333 100644--- a/backend/internal/services/activity_service.go+++ b/backend/internal/services/activity_service.go@@ -11,9 +11,11 @@ import ("sync""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"@@ -26,6 +28,10 @@ const (defaultActivityHistoryLimit = 1000defaultActivityMessages = 500staleImageUpdateCheckAge = 6 * time.Hour+ // abandonedActivityGrace is how old an untracked queued/running activity+ // must be before the periodic sweep fails it. It covers the window between+ // StartActivity's row creation and the worker's Track call.+ abandonedActivityGrace = 2 * time.Minute)type ActivityService struct {@@ -397,6 +403,22 @@ func (s *ActivityService) AwaitActivitySlot(ctx context.Context, activityID, envreturn nil}+// AwaitActivitySlotBounded waits for a concurrency slot like AwaitActivitySlot+// but gives up after timeouts.DefaultActivitySlotWait, returning+// ActivitySlotWaitTimeoutError so the caller fails the queued activity loudly+// instead of parking forever behind long-running slot holders.+func (s *ActivityService) AwaitActivitySlotBounded(ctx context.Context, activityID, environmentID string) error {+ slotCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultActivitySlotWait)+ defer cancel()+ if err := s.AwaitActivitySlot(slotCtx, activityID, environmentID); err != nil {+ if ctx.Err() == nil && errors.Is(err, context.DeadlineExceeded) {+ return &common.ActivitySlotWaitTimeoutError{}+ }+ return err+ }+ return nil+}+func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string, req UpdateActivityRequest) (*activitytypes.Activity, error) {if err := s.checkInitInternal(); err != nil {return nil, err@@ -548,21 +570,37 @@ func (s *ActivityService) CompleteActivity(ctx context.Context, activityID strinnow := time.Now()var model models.Activity- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {- return fmt.Errorf("failed to load activity: %w", err)- }+ // A lost terminal write leaves the activity stuck in running forever, so+ // retry transient DB contention (SQLITE_BUSY under bulk activity writes)+ // instead of surfacing it once and giving up.+ const completeWriteAttempts = 3+ var writeErr error+ for attempt := 1; attempt <= completeWriteAttempts; attempt++ {+ writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {+ return fmt.Errorf("failed to load activity: %w", err)+ }- updates := completeActivityUpdatesInternal(model.StartedAt, status, finalMessage, errMessage, finalStep, now)- if err := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {- return fmt.Errorf("failed to complete activity: %w", err)+ updates := completeActivityUpdatesInternal(model.StartedAt, status, finalMessage, errMessage, finalStep, now)+ if err := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {+ return fmt.Errorf("failed to complete activity: %w", err)+ }+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {+ return fmt.Errorf("failed to load completed activity: %w", err)+ }+ return nil+ })+ if writeErr == nil {+ break}- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {- return fmt.Errorf("failed to load completed activity: %w", err)+ if attempt == completeWriteAttempts || !isRetryableDBWriteErrorInternal(writeErr) {+ return nil, writeErr}- return nil- }); err != nil {- return nil, err+ slog.WarnContext(ctx, "retrying activity terminal status write", "activityId", activityID, "attempt", attempt, "error", writeErr)+ time.Sleep(250 * time.Millisecond * time.Duration(attempt))+ }+ if writeErr != nil {+ return nil, writeErr}if strings.TrimSpace(finalMessage) != "" {@@ -705,6 +743,86 @@ func (s *ActivityService) FailStaleImageUpdateChecks(ctx context.Context) (int64return failed, errors.Join(failErrs...)}+// isTrackedInternal reports whether activityID has a live work registration in+// this process (i.e. a worker goroutine called Track and has not completed yet).+func (s *ActivityService) isTrackedInternal(activityID string) bool {+ s.runningMu.Lock()+ defer s.runningMu.Unlock()+ _, ok := s.running[activityID]+ return ok+}++// FailAbandonedActivities marks queued/running activities whose worker is no+// longer alive in this process as failed, releasing any concurrency slot they+// still hold. Liveness comes from the running map, not age: every creation+// path Tracks its activity right after StartActivity, and the short grace+// period covers that create→Track window. This assumes exactly one Arcane+// process owns the database (managers and agents each own theirs); running+// multiple replicas against one database would make each replica sweep the+// other's live work.+//+// The terminal write is status-guarded like CancelActivity's untracked+// fallback: a worker completing concurrently wins the race and the row is+// skipped here.+func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, error) {+ if s == nil || s.db == nil {+ return 0, nil+ }++ cutoff := time.Now().Add(-abandonedActivityGrace)+ activeStatuses := []models.ActivityStatus{models.ActivityStatusQueued, models.ActivityStatusRunning}+ var candidates []models.Activity+ if err := s.db.WithContext(ctx).+ Where("status IN ? AND started_at < ?", activeStatuses, cutoff).+ Find(&candidates).Error; err != nil {+ return 0, fmt.Errorf("find abandoned activities: %w", err)+ }++ const message = "Activity was marked failed because its worker is no longer running"+ errMessage := message+ var swept int64+ var sweepErrs []error+ for i := range candidates {+ activityID := candidates[i].ID+ if s.isTrackedInternal(activityID) {+ continue+ }++ now := time.Now()+ var finalized models.Activity+ lostRace := false+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ updates := completeActivityUpdatesInternal(candidates[i].StartedAt, models.ActivityStatusFailed, message, &errMessage, nil, now)+ result := tx.Model(&models.Activity{}).+ Where("id = ? AND status IN ?", activityID, activeStatuses).+ Updates(updates)+ if result.Error != nil {+ return fmt.Errorf("fail abandoned activity: %w", result.Error)+ }+ if result.RowsAffected == 0 {+ lostRace = true+ return nil+ }+ if err := tx.First(&finalized, "id = ?", activityID).Error; err != nil {+ return fmt.Errorf("load failed abandoned activity: %w", err)+ }+ return nil+ }); err != nil {+ sweepErrs = append(sweepErrs, fmt.Errorf("sweep activity %s: %w", activityID, err))+ continue+ }+ if lostRace {+ continue+ }++ s.releaseSlotInternal(activityID)+ s.publishActivityInternal(activityToDTOInternal(&finalized))+ swept+++ }++ return swept, errors.Join(sweepErrs...)+}+// ResolveOrphanedQueuedActivities fails any activity still queued at startup.// Queued state is owned by a live goroutine blocked on AwaitActivitySlot, so a// queued row after a restart can never start running.diff --git a/backend/internal/services/activity_service_test.go b/backend/internal/services/activity_service_test.goindex 83a6d8222..20efca6df 100644--- a/backend/internal/services/activity_service_test.go+++ b/backend/internal/services/activity_service_test.go@@ -602,6 +602,55 @@ func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {require.Equal(t, models.ActivityStatusSuccess, completed.Status)}+func TestActivityServiceFailAbandonedActivitiesInternal(t *testing.T) {+ ctx := context.Background()+ db := setupActivityServiceTestDBInternal(t)+ service := NewActivityService(db, nil)++ abandoned, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeImageUpdateCheck,+ LatestMessage: "checking",+ })+ require.NoError(t, err)+ tracked, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeAutoUpdate,+ LatestMessage: "updating",+ })+ require.NoError(t, err)+ fresh, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeImageUpdateCheck,+ LatestMessage: "checking",+ })+ require.NoError(t, err)++ backdated := time.Now().Add(-10 * time.Minute)+ require.NoError(t, db.Model(&models.Activity{}).+ Where("id IN ?", []string{abandoned.ID, tracked.ID}).+ Update("started_at", backdated).Error)+ _ = service.Track(ctx, tracked.ID)++ swept, err := service.FailAbandonedActivities(ctx)+ require.NoError(t, err)+ require.EqualValues(t, 1, swept)++ var abandonedRow models.Activity+ require.NoError(t, db.First(&abandonedRow, "id = ?", abandoned.ID).Error)+ require.Equal(t, models.ActivityStatusFailed, abandonedRow.Status)+ require.NotNil(t, abandonedRow.EndedAt)+ require.Contains(t, abandonedRow.LatestMessage, "worker is no longer running")++ var trackedRow models.Activity+ require.NoError(t, db.First(&trackedRow, "id = ?", tracked.ID).Error)+ require.Equal(t, models.ActivityStatusRunning, trackedRow.Status)++ var freshRow models.Activity+ require.NoError(t, db.First(&freshRow, "id = ?", fresh.ID).Error)+ require.Equal(t, models.ActivityStatusRunning, freshRow.Status)+}+func TestActivityServiceResolveStaleAutoUpdateActivitiesInternal(t *testing.T) {ctx := context.Background()db := setupActivityServiceTestDBInternal(t)diff --git a/backend/internal/services/image_update_service.go b/backend/internal/services/image_update_service.goindex 3cfb4b2eb..a738f5391 100644--- a/backend/internal/services/image_update_service.go+++ b/backend/internal/services/image_update_service.go@@ -6,6 +6,7 @@ import ("fmt""log/slog""maps"+ "runtime/debug""strings""sync""time"@@ -191,7 +192,9 @@ func (s *ImageUpdateService) completeImageUpdateActivityInternal(ctx context.Con}step := "Image update check complete"if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage, step); err != nil {- slog.DebugContext(ctx, "failed to complete image update activity", "activityId", activityID, "error", err)+ // A lost terminal write strands the activity in running forever, so it+ // must be loud enough to correlate with a stuck activity panel entry.+ slog.ErrorContext(ctx, "failed to complete image update activity", "activityId", activityID, "error", err)}}@@ -204,10 +207,7 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef striif result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {result.ResponseTimeMs = int(time.Since(startTime).Milliseconds())result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, imageRef+" — digest pinned, skipped", 100, "Skipping image update check")- if saveErr := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); saveErr != nil {- slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", saveErr.Error())- }+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 100)if s.eventService != nil {metadata := models.JSON{"action": "check_update",@@ -1310,21 +1310,19 @@ func (s *ImageUpdateService) recordBatchImageCheckInternal(ctx context.Context,}}-func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs []string, externalCreds []containerregistry.Credential) (map[string]*imageupdate.Response, error) {- startBatch := time.Now()- results := make(map[string]*imageupdate.Response, len(imageRefs))- if len(imageRefs) == 0 {- return results, nil+// recordDigestPinnedSkipInternal notes a digest-pinned ref as skipped on the+// activity stream and persists its unchanged result.+func (s *ImageUpdateService) recordDigestPinnedSkipInternal(ctx context.Context, activityID, imageRef string, result *imageupdate.Response, progress int) {+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, imageRef+" — digest pinned, skipped", progress, "Skipping image update check")+ if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {+ slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())}+}- activityID := s.startImageUpdateActivityInternal(ctx, fmt.Sprintf("%d images", len(imageRefs)), len(imageRefs))- ctx = s.activityService.Track(ctx, activityID)- activitylib.AwaitHandlerActivitySlot(ctx, s.activityService, activityID, "0")- s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), 5, "Preparing image update check")- slog.DebugContext(ctx, "Starting batch image update check", "imageCount", len(imageRefs), "externalCredCount", len(externalCreds))-- regRepos, initialResults, images := s.parseAndGroupImagesInternal(imageRefs)- maps.Copy(results, initialResults)+// recordInitialBatchResultsInternal stamps the activity ID on the parse-stage+// results and persists digest-pinned refs as skipped, since they never reach+// the registry check stage.+func (s *ImageUpdateService) recordInitialBatchResultsInternal(ctx context.Context, activityID string, initialResults map[string]*imageupdate.Response) {for imageRef, result := range initialResults {if result == nil {continue@@ -1337,20 +1335,68 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefscontinue}- s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, imageRef+" — digest pinned, skipped", 5, "Skipping image update check")- if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {- slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 5)+ }+}++func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs []string, externalCreds []containerregistry.Credential) (results map[string]*imageupdate.Response, err error) {+ startBatch := time.Now()+ results = make(map[string]*imageupdate.Response, len(imageRefs))+ if len(imageRefs) == 0 {+ return results, nil+ }++ activityID := s.startImageUpdateActivityInternal(ctx, fmt.Sprintf("%d images", len(imageRefs)), len(imageRefs))+ ctx = s.activityService.Track(ctx, activityID)++ // A single deferred finalizer owns the terminal activity write so it runs+ // on success, error, and panic alike — any early return that skipped+ // completion would strand the row in running forever. It reads the Track+ // ctx so a user cancellation still records a cancelled status.+ defer func() {+ if r := recover(); r != nil {+ // Don't re-panic: the caller is the long-lived watcher goroutine.+ err = fmt.Errorf("image update check panicked: %v", r)+ slog.ErrorContext(ctx, "image update check panicked", "activityId", activityID, "panic", r, "stack", string(debug.Stack()))+ }+ if err != nil {+ s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+err.Error())+ return+ }+ successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)+ finalMessage := fmt.Sprintf("Image update check completed: %d checked, %d errors", successCount, errorCount)+ s.completeImageUpdateActivityInternal(ctx, activityID, errorCount == 0, finalMessage)+ }()++ if activityID != "" {+ // Bounded slot wait: update-all runs share these slots and can hold+ // them for a long time; parking here forever would wedge every future+ // scan behind the watcher's single-flight gate. On timeout the+ // finalizer above flips the queued row to failed.+ if slotErr := s.activityService.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {+ return results, slotErr}}+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, models.ActivityMessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), 5, "Preparing image update check")+ slog.DebugContext(ctx, "Starting batch image update check", "imageCount", len(imageRefs), "externalCredCount", len(externalCreds))- composeBuildRefs, err := s.composeBuildImageRefsInternal(ctx)- if err != nil {- wrappedErr := fmt.Errorf("prepare compose build image references: %w", err)- s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+wrappedErr.Error())- return results, wrappedErr+ regRepos, initialResults, images := s.parseAndGroupImagesInternal(imageRefs)+ maps.Copy(results, initialResults)+ s.recordInitialBatchResultsInternal(ctx, activityID, initialResults)++ // The aggregate scan gets its own deadline: individual registry RPCs are+ // bounded, but the batch as a whole (including registry limiter waits)+ // was not, and a wedged scan holds its activity slot indefinitely.+ scanCtx, cancelScan := context.WithTimeout(ctx, timeouts.DefaultImageUpdateScan)+ defer cancelScan()++ composeBuildRefs, composeErr := s.composeBuildImageRefsInternal(scanCtx)+ if composeErr != nil {+ err = fmt.Errorf("prepare compose build image references: %w", composeErr)+ return results, err}- resolvedCreds := s.resolveBatchCredentialsInternal(ctx, externalCreds)+ resolvedCreds := s.resolveBatchCredentialsInternal(scanCtx, externalCreds)slog.DebugContext(ctx, "Resolved batch registry credentials", "credentialCount", len(resolvedCreds), "registryCount", len(regRepos))@@ -1358,25 +1404,33 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefsresults: results,total: len(images),}- g, groupCtx := errgroup.WithContext(ctx)+ g, groupCtx := errgroup.WithContext(scanCtx)g.SetLimit(10) // Limit concurrencyfor _, img := range images {- g.Go(func() error {+ g.Go(func() (checkErr error) {+ // x/sync's errgroup does not recover goroutine panics (they crash+ // the process), so convert them to errors here; the deferred+ // finalizer then records the failed terminal status.+ defer func() {+ if r := recover(); r != nil {+ checkErr = fmt.Errorf("image update check panicked: %v", r)+ slog.ErrorContext(groupCtx, "image update check worker panicked", "activityId", activityID, "imageRef", img.canonicalRef, "panic", r, "stack", string(debug.Stack()))+ }+ }()return s.checkBatchImageInternal(groupCtx, activityID, resolvedCreds, img, composeBuildRefs, recorder)})}- if err := g.Wait(); err != nil {+ if err = g.Wait(); err != nil {+ if ctx.Err() == nil && errors.Is(scanCtx.Err(), context.DeadlineExceeded) {+ err = fmt.Errorf("image update check timed out after %s: %w", timeouts.DefaultImageUpdateScan, err)+ }slog.ErrorContext(ctx, "Batch check error", "error", err)- s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+err.Error())return results, err}successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)- finalSuccess := errorCount == 0- finalMessage := fmt.Sprintf("Image update check completed: %d checked, %d errors", successCount, errorCount)- s.completeImageUpdateActivityInternal(ctx, activityID, finalSuccess, finalMessage)slog.InfoContext(ctx, "Batch image update check completed","totalImages", len(imageRefs),"successCount", successCount,diff --git a/backend/internal/services/image_update_service_test.go b/backend/internal/services/image_update_service_test.goindex 547d4a067..3835c959e 100644--- a/backend/internal/services/image_update_service_test.go+++ b/backend/internal/services/image_update_service_test.go@@ -849,6 +849,36 @@ func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInterrequire.Contains(t, activity.LatestMessage, "0 checked, 1 errors")}+func TestImageUpdateService_CheckMultipleImagesPanicMarksActivityFailedInternal(t *testing.T) {+ db := setupImageUpdateTestDB(t)+ require.NoError(t, db.AutoMigrate(&models.Activity{}, &models.ActivityMessage{}))++ settingsService := newImageUpdateTestSettingsServiceInternal("30", "30")+ activityService := NewActivityService(db, nil)+ dockerServer := newImageUpdateRegistryOnlyServer(t, "team/app:1.2.3", digest.FromString("unused").String())+ defer dockerServer.Close()+ registryService := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {+ return &fakeRegistryDaemonClient{+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {+ panic("registry check exploded")+ },+ }, nil+ }, nil)++ svc := NewImageUpdateService(db, settingsService, registryService, &DockerClientService{client: newTestDockerClient(t, dockerServer)}, nil, nil, activityService)++ _, err := svc.CheckMultipleImages(context.Background(), []string{"registry.example.com/team/app:1.2.3"}, nil)+ require.Error(t, err)+ require.Contains(t, err.Error(), "image update check panicked")+ require.Contains(t, err.Error(), "registry check exploded")++ var activity models.Activity+ require.NoError(t, db.Where("type = ?", models.ActivityTypeImageUpdateCheck).First(&activity).Error)+ require.Equal(t, models.ActivityStatusFailed, activity.Status)+ require.NotNil(t, activity.EndedAt)+ require.Contains(t, activity.LatestMessage, "Image update check failed")+}+func TestImageUpdateService_GetAllImageRefsUsesDockerAPITimeoutInternal(t *testing.T) {db := setupImageUpdateTestDB(t)settingsService := newImageUpdateTestSettingsServiceInternal("30", "1")diff --git a/backend/internal/services/system_service.go b/backend/internal/services/system_service.goindex d08fb17cf..a4419e06d 100644--- a/backend/internal/services/system_service.go+++ b/backend/internal/services/system_service.go@@ -372,6 +372,7 @@ type startMatchingContainersOptionsInternal struct {func (s *SystemService) startMatchingContainersInternal(ctx context.Context, environmentID string, opts startMatchingContainersOptionsInternal) (*containertypes.ActionResult, error) {activityID := s.startSystemContainerActivityInternal(ctx, environmentID, models.ActivityTypeContainerStart, opts.ResourceName, opts.StartMessage)+ ctx = s.activityService.Track(ctx, activityID)containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)if err != nil {result := &containertypes.ActionResult{@@ -393,6 +394,7 @@ func (s *SystemService) startMatchingContainersInternal(ctx context.Context, envfunc (s *SystemService) StopAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {activityID := s.startSystemContainerActivityInternal(ctx, environmentID, models.ActivityTypeContainerStop, "All containers", "Stopping all containers")+ ctx = s.activityService.Track(ctx, activityID)containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)if err != nil {result := &containertypes.ActionResult{diff --git a/backend/internal/services/updater_service.go b/backend/internal/services/updater_service.goindex 8d26f5705..0a68f0caf 100644--- a/backend/internal/services/updater_service.go+++ b/backend/internal/services/updater_service.go@@ -22,6 +22,7 @@ import (dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"projectspkg "github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/types/v2/updater"@@ -152,7 +153,6 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optioout = &updater.Result{Items: []updater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}ctx = s.trackActivityInternal(ctx, activityID)ctx = contextWithActivityIDInternal(ctx, activityID)- activitylib.AwaitHandlerActivitySlot(ctx, s.deps.Activity, activityID, "0")defer func() {if out == nil {@@ -165,6 +165,21 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optios.completeAutoUpdateActivityInternal(ctx, activityID, out, err)}()+ if activityID != "" && s.deps.Activity != nil {+ // Bounded slot wait: an unbounded wait behind other long-running runs+ // would strand the queued activity row (the completion defer above+ // flips it to failed on timeout instead).+ if err = s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); err != nil {+ return out, err+ }+ }++ // The engine's per-container docker operations carry no timeouts, so cap+ // the whole run; the Track ctx stays unbounded for the deferred completion+ // so user cancellation is still detected there.+ runCtx, cancelRun := context.WithTimeout(ctx, timeouts.DefaultAutoUpdateApply)+ defer cancelRun()+s.recordAutoUpdateEventInternal(ctx, models.EventSeverityInfo, models.JSON{"phase": "start","dryRun": options.DryRun,@@ -176,11 +191,11 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optios.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Planning pending updates", "Planning updates", 5)if len(options.ResourceIds) > 0 {- if err = s.applyScopedUpdatesInternal(ctx, options, out); err != nil {+ if err = s.applyScopedUpdatesInternal(runCtx, options, out); err != nil {return out, err}} else {- moduleResult, engineErr := s.engineInternal().ApplyPending(ctx, moduleOptionsFromUpdaterOptionsInternal(options))+ moduleResult, engineErr := s.engineInternal().ApplyPending(runCtx, moduleOptionsFromUpdaterOptionsInternal(options))if moduleResult != nil {out = resultFromModuleInternal(moduleResult)out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()@@ -194,7 +209,7 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options updater.Optioif !options.DryRun && s.deps.ImageUpdates != nil {s.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Cleaning up update records", "Cleaning up", 95)- if cleanupErr := s.deps.ImageUpdates.CleanupOrphanedRecords(ctx); cleanupErr != nil {+ if cleanupErr := s.deps.ImageUpdates.CleanupOrphanedRecords(runCtx); cleanupErr != nil {s.loggerInternal().WarnContext(ctx, "cleanup orphaned update records failed", "error", cleanupErr)}}@@ -524,7 +539,10 @@ func (s *UpdaterService) DockerClient(ctx context.Context) (*client.Client, erroreturn s.deps.Docker.GetClient(ctx)}-// PullImage pulls an image through Arcane's image service.+// PullImage pulls an image through Arcane's image service. The pull is+// bounded by the dockerImagePullTimeout setting — ImageService.PullImage does+// not bound itself, and an unbounded engine pull would hold the auto-update+// run (and its activity slot) indefinitely.func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progress io.Writer) error {if s == nil || s.deps.ImagePuller == nil {return &common.UpdaterImageServiceUnavailableError{}@@ -533,15 +551,22 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progreswriter := activitylib.NewWriter(ctx, s.deps.Activity, activityID, progress, "Pulling updated images")defer activitylib.FlushWriter(writer)+ pullTimeoutSeconds := 0+ if s.deps.Settings != nil {+ pullTimeoutSeconds = s.deps.Settings.GetSettingsConfig().DockerImagePullTimeout.AsInt()+ }+ pullCtx, cancelPull := timeouts.WithTimeout(ctx, pullTimeoutSeconds, timeouts.DefaultDockerImagePull)+ defer cancelPull()+if s.deps.Projects != nil {- resolved, err := s.deps.Projects.resolveRegistryCredentialsInternal(ctx)+ resolved, err := s.deps.Projects.resolveRegistryCredentialsInternal(pullCtx)if err != nil {return fmt.Errorf("resolve registry credentials: %w", err)}- return s.deps.ImagePuller.PullImage(ctx, imageRef, writer, s.deps.SystemUser, resolved)+ return s.deps.ImagePuller.PullImage(pullCtx, imageRef, writer, s.deps.SystemUser, resolved)}- return s.deps.ImagePuller.PullImage(ctx, imageRef, writer, s.deps.SystemUser, nil)+ return s.deps.ImagePuller.PullImage(pullCtx, imageRef, writer, s.deps.SystemUser, nil)}// PendingImageUpdates returns pending image update records from Arcane's database.@@ -841,7 +866,9 @@ func (s *UpdaterService) completeAutoUpdateActivityInternal(ctx context.Context,}if _, err := s.deps.Activity.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {- slog.DebugContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)+ // A lost terminal write strands the activity in running forever, so it+ // must be loud enough to correlate with a stuck activity panel entry.+ slog.ErrorContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)}}diff --git a/backend/internal/services/vulnerability_service.go b/backend/internal/services/vulnerability_service.goindex 3a574ec55..736e00400 100644--- a/backend/internal/services/vulnerability_service.go+++ b/backend/internal/services/vulnerability_service.go@@ -3465,7 +3465,7 @@ func (s *VulnerabilityService) saveScanResultWithRetryInternal(}if attempt < maxAttempts {- if !isRetryableScanSaveErrorInternal(lastErr) {+ if !isRetryableDBWriteErrorInternal(lastErr) {break}@@ -3491,7 +3491,7 @@ func (s *VulnerabilityService) saveScanResultWithRetryInternal(return lastErr}-func isRetryableScanSaveErrorInternal(err error) bool {+func isRetryableDBWriteErrorInternal(err error) bool {if err == nil {return false}diff --git a/backend/pkg/libarcane/activity/handler.go b/backend/pkg/libarcane/activity/handler.goindex 1afb66635..ecfe61ec4 100644--- a/backend/pkg/libarcane/activity/handler.go+++ b/backend/pkg/libarcane/activity/handler.go@@ -175,7 +175,7 @@ func CompleteHandlerActivity(ctx context.Context, activityService Service, activactivityCtx := utils.ActivityRuntimeContext(ctx, nil)if _, completeErr := activityService.CompleteActivity(activityCtx, activityID, status, finalMessage, errMessage); completeErr != nil {- slog.DebugContext(activityCtx, "failed to complete background activity", "activityId", activityID, "error", completeErr)+ slog.WarnContext(activityCtx, "failed to complete background activity", "activityId", activityID, "error", completeErr)}}diff --git a/backend/pkg/libarcane/timeouts/timeouts.go b/backend/pkg/libarcane/timeouts/timeouts.goindex 1fc74bbef..7afdb38f0 100644--- a/backend/pkg/libarcane/timeouts/timeouts.go+++ b/backend/pkg/libarcane/timeouts/timeouts.go@@ -14,6 +14,20 @@ const (DefaultRegistry = 30 * time.SecondDefaultProxyRequest = 60 * time.SecondDefaultBuildTimeout = 30 * time.Minute+ // DefaultImageUpdateScan bounds an entire batch image update check+ // end-to-end; individual registry RPCs are separately bounded, but the+ // aggregate scan needs its own ceiling so a wedged batch cannot hold its+ // activity slot forever.+ DefaultImageUpdateScan = 15 * time.Minute+ // DefaultActivitySlotWait bounds how long a queued activity waits for a+ // per-environment concurrency slot before failing loudly instead of+ // parking forever behind hung work.+ DefaultActivitySlotWait = 2 * time.Minute+ // DefaultAutoUpdateApply bounds an entire auto-update (update-all) run.+ // The updater engine's per-container docker operations carry no timeouts+ // of their own, so this is the Arcane-side backstop that keeps a hung+ // engine op from holding an activity slot indefinitely.+ DefaultAutoUpdateApply = 30 * time.Minute)func GetDuration(settingSeconds int, defaultDuration time.Duration) time.Duration {diff --git a/backend/pkg/scheduler/activity_sweep_job.go b/backend/pkg/scheduler/activity_sweep_job.gonew file mode 100644index 000000000..011346796--- /dev/null+++ b/backend/pkg/scheduler/activity_sweep_job.go@@ -0,0 +1,42 @@+package scheduler++import (+ "context"+ "log/slog"++ "github.com/getarcaneapp/arcane/backend/v2/internal/services"+)++const ActivitySweepJobName = "activity-sweep"++// ActivitySweepJob periodically fails queued/running activities whose worker+// is no longer alive in this process, so a lost terminal write (crash, panic,+// dropped DB write) cannot leave an activity stuck in running forever. It is+// an internal job: it has no job_metadata entry and is invisible in the Jobs+// UI.+type ActivitySweepJob struct {+ activityService *services.ActivityService+}++func NewActivitySweepJob(activityService *services.ActivityService) *ActivitySweepJob {+ return &ActivitySweepJob{activityService: activityService}+}++func (j *ActivitySweepJob) Name() string {+ return ActivitySweepJobName+}++func (j *ActivitySweepJob) Schedule(_ context.Context) string {+ return "0 */5 * * * *"+}++func (j *ActivitySweepJob) Run(ctx context.Context) {+ swept, err := j.activityService.FailAbandonedActivities(ctx)+ if err != nil {+ slog.WarnContext(ctx, "activity sweep failed", "jobName", ActivitySweepJobName, "swept", swept, "error", err)+ return+ }+ if swept > 0 {+ slog.InfoContext(ctx, "marked abandoned activities as failed", "jobName", ActivitySweepJobName, "count", swept)+ }+}diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.goindex cb7f782f2..c1d5c498c 100644--- a/backend/pkg/scheduler/auto_update_job.go+++ b/backend/pkg/scheduler/auto_update_job.go@@ -3,14 +3,25 @@ package schedulerimport ("context""log/slog"+ "sync/atomic""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/types/v2/updater")+// pendingUpdateApplierInternal is the slice of UpdaterService the job needs,+// kept as an interface so the overlap guard is testable with a fake.+type pendingUpdateApplierInternal interface {+ ApplyPending(ctx context.Context, options updater.Options) (*updater.Result, error)+}+type AutoUpdateJob struct {- updaterService *services.UpdaterService+ updaterService pendingUpdateApplierInternalsettingsService *services.SettingsService+ // running guards against overlapping cron ticks: an update-all run can+ // outlast its schedule interval, and stacked runs exhaust the shared+ // activity slots.+ running atomic.Bool}func NewAutoUpdateJob(updaterService *services.UpdaterService, settingsService *services.SettingsService) *AutoUpdateJob {@@ -47,6 +58,12 @@ func (j *AutoUpdateJob) Run(ctx context.Context) {return}+ if !j.running.CompareAndSwap(false, true) {+ slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")+ return+ }+ defer j.running.Store(false)+slog.InfoContext(ctx, "auto-update run started")result, err := j.updaterService.ApplyPending(ctx, updater.Options{})diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.goindex 06a2c78c5..f370aa85e 100644--- a/backend/pkg/scheduler/auto_update_job_test.go+++ b/backend/pkg/scheduler/auto_update_job_test.go@@ -2,8 +2,11 @@ package schedulerimport ("context"+ "sync/atomic""testing"+ "time"+ "github.com/getarcaneapp/arcane/types/v2/updater""github.com/stretchr/testify/require")@@ -23,3 +26,53 @@ func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T)require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", false))require.False(t, job.ShouldSchedule(ctx))}++type blockingApplierFakeInternal struct {+ calls atomic.Int32+ started chan struct{}+ release chan struct{}+}++func (f *blockingApplierFakeInternal) ApplyPending(context.Context, updater.Options) (*updater.Result, error) {+ f.calls.Add(1)+ select {+ case f.started <- struct{}{}:+ default:+ }+ <-f.release+ return &updater.Result{}, nil+}++func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {+ ctx := context.Background()+ _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)+ require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", true))++ applier := &blockingApplierFakeInternal{+ started: make(chan struct{}, 1),+ release: make(chan struct{}),+ }+ job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc}++ firstDone := make(chan struct{})+ go func() {+ job.Run(ctx)+ close(firstDone)+ }()+ <-applier.started++ // Overlapping tick returns immediately without a second ApplyPending.+ job.Run(ctx)+ require.Equal(t, int32(1), applier.calls.Load())++ close(applier.release)+ select {+ case <-firstDone:+ case <-time.After(time.Second):+ t.Fatal("first auto-update run did not finish")+ }++ // The guard resets once the run finishes.+ job.Run(ctx)+ require.Equal(t, int32(2), applier.calls.Load())+}diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.goindex 17aa1dafc..4221e4080 100644--- a/backend/pkg/scheduler/image_update_watcher.go+++ b/backend/pkg/scheduler/image_update_watcher.go@@ -9,6 +9,7 @@ import ("sync/atomic""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/types/v2/containerregistry"@@ -167,7 +168,11 @@ func (w *ImageUpdateWatcher) RefreshSchedule() {}}-// RunNow performs the same full-host image scan used by automatic watcher triggers.+// RunNow performs the same full-host image scan used by automatic watcher+// triggers. It does not wait behind an active scan: a request during one+// returns ImageScanInProgressError immediately instead of parking+// (potentially forever, given the detached run-now context) on the+// single-flight gate. The triggered-scan loop waits and retries itself.func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil || w.metadataReady == nil {return errors.New("image update watcher is not initialized")@@ -179,20 +184,11 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {}run := &imageUpdateScanRunInternal{done: make(chan struct{})}- for {- activeRun := w.activeRun.Load()- if activeRun == nil && w.activeRun.CompareAndSwap(nil, run) {- break- }- if activeRun == nil {- continue- }-- select {- case <-ctx.Done():- return ctx.Err()- case <-activeRun.done:+ for !w.activeRun.CompareAndSwap(nil, run) {+ if w.activeRun.Load() != nil {+ return &common.ImageScanInProgressError{}}+ // Lost the CAS to a run that has since been released; retry.}defer func() {w.activeRun.Store(nil)@@ -285,7 +281,18 @@ func (w *ImageUpdateWatcher) runTriggeredScansInternal(ctx context.Context) {if !w.waitForDebounceInternal(ctx) {return}- if err := w.RunNow(ctx); err != nil && !errors.Is(err, context.Canceled) {+ err := w.RunNow(ctx)+ var inProgress *common.ImageScanInProgressError+ if errors.As(err, &inProgress) {+ // A manual scan holds the gate; wait it out and re-queue the+ // trigger so this scan attempt is not lost.+ if !w.waitForActiveScanInternal(ctx) {+ return+ }+ w.Trigger()+ continue+ }+ if err != nil && !errors.Is(err, context.Canceled) {slog.ErrorContext(ctx, "image update watcher scan failed", "error", err)}}@@ -332,6 +339,21 @@ func (w *ImageUpdateWatcher) runScheduledPollsInternal(ctx context.Context) {}}+// waitForActiveScanInternal blocks until any in-flight scan releases the+// single-flight gate. It returns false when ctx is cancelled first.+func (w *ImageUpdateWatcher) waitForActiveScanInternal(ctx context.Context) bool {+ activeRun := w.activeRun.Load()+ if activeRun == nil {+ return true+ }+ select {+ case <-ctx.Done():+ return false+ case <-activeRun.done:+ return true+ }+}+func (w *ImageUpdateWatcher) waitForDebounceInternal(ctx context.Context) bool {timer := time.NewTimer(w.debounce)defer timer.Stop()diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex 6df45a569..06b9430f5 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -10,6 +10,7 @@ import ("testing""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/imageupdate""github.com/moby/moby/api/types/events"@@ -162,7 +163,7 @@ func (b *lockedBufferInternal) Write(p []byte) (int, error) {func (b *lockedBufferInternal) stringInternal() string {b.mu.Lock()defer b.mu.Unlock()- return b.Buffer.String()+ return b.String()}func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {@@ -288,51 +289,64 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {require.Eventually(t, func() bool { return scanner.countInternal() == 2 }, time.Second, 5*time.Millisecond)}-func TestImageUpdateWatcher_RunNowSerializesConcurrentCalls(t *testing.T) {+func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *testing.T) {releaseCh := make(chan struct{})scanner := &imageUpdateScannerFakeInternal{- startedCh: make(chan int, 2),+ startedCh: make(chan int, 1),releaseCh: releaseCh,}settings := &pollingSettingReaderFakeInternal{enabled: true}watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)markImageUpdateWatcherMetadataReadyForTestInternal(watcher)- errCh := make(chan error, 2)- go func() { errCh <- watcher.RunNow(context.Background()) }()+ firstErrCh := make(chan error, 1)+ go func() { firstErrCh <- watcher.RunNow(context.Background()) }()require.Equal(t, 1, <-scanner.startedCh)- go func() { errCh <- watcher.RunNow(context.Background()) }()- time.Sleep(20 * time.Millisecond)+ var inProgress *common.ImageScanInProgressError+ require.ErrorAs(t, watcher.RunNow(context.Background()), &inProgress)require.Equal(t, 1, scanner.countInternal())+close(releaseCh)- require.Equal(t, 2, <-scanner.startedCh)- require.NoError(t, <-errCh)- require.NoError(t, <-errCh)+ require.NoError(t, <-firstErrCh)require.Equal(t, 1, scanner.maxActiveInternal())++ // The gate is free again after the first scan finishes.+ require.NoError(t, watcher.RunNow(context.Background()))+ require.Equal(t, 2, scanner.countInternal())}-func TestImageUpdateWatcher_RunNowWaiterHonorsCancellation(t *testing.T) {+func TestImageUpdateWatcher_TriggeredScanRetriesAfterManualScanFinishes(t *testing.T) {releaseCh := make(chan struct{})scanner := &imageUpdateScannerFakeInternal{- startedCh: make(chan int, 1),+ startedCh: make(chan int, 4),releaseCh: releaseCh,}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)- markImageUpdateWatcherMetadataReadyForTestInternal(watcher)+ eventBus := bus.NewDockerEventBus()+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)+ startImageUpdateWatcherForTestInternal(t, watcher)- firstErrCh := make(chan error, 1)- go func() { firstErrCh <- watcher.RunNow(context.Background()) }()+ // Let the startup scan finish so the gate is free.require.Equal(t, 1, <-scanner.startedCh)+ releaseCh <- struct{}{}+ require.Eventually(t, func() bool { return watcher.activeRun.Load() == nil }, time.Second, time.Millisecond)- waitingCtx, cancel := context.WithCancel(context.Background())- cancel()- require.ErrorIs(t, watcher.RunNow(waitingCtx), context.Canceled)- require.Equal(t, 1, scanner.countInternal())+ // A manual scan takes the gate, then a Docker event fires: the triggered+ // loop must wait out the manual scan and still run its own scan after.+ manualErrCh := make(chan error, 1)+ go func() { manualErrCh <- watcher.RunNow(context.Background()) }()+ require.Equal(t, 2, <-scanner.startedCh)+ eventBus.Publish(events.Message{Type: events.ImageEventType, Action: events.ActionPull})- close(releaseCh)- require.NoError(t, <-firstErrCh)+ time.Sleep(30 * time.Millisecond)+ require.Equal(t, 2, scanner.countInternal())+ releaseCh <- struct{}{}+ require.NoError(t, <-manualErrCh)++ require.Equal(t, 3, <-scanner.startedCh)+ releaseCh <- struct{}{}+ require.Equal(t, 1, scanner.maxActiveInternal())}func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *testing.T) {