Skip to content

Modern Docker Management, Designed for Everyone

fix: prune long running stuck jobs (#2828)

Kyle Mendell9dbf241466e4
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 7188b6b28..29808f5a8 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -17,6 +17,15 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
// the agent-mode gating, the startup heartbeat, and the settings callbacks.
jobs := di.InitializeJobs(appCtx, appConfig, appServices)
+ if appServices.Activity != nil {
+ failed, err := appServices.Activity.FailStaleImageUpdateChecks(appCtx)
+ if err != nil {
+ slog.WarnContext(appCtx, "Failed to mark stale image update checks as failed", "count", failed, "error", err)
+ } else if failed > 0 {
+ slog.InfoContext(appCtx, "Marked stale image update checks as failed", "count", failed)
+ }
+ }
+
newScheduler.RegisterJob(jobs.AutoUpdate)
newScheduler.RegisterJob(jobs.ImagePolling)
newScheduler.RegisterJob(jobs.DockerClientRefresh)
diff --git a/backend/internal/services/activity_service.go b/backend/internal/services/activity_service.go
index 2a826bd57..409efddca 100644
--- a/backend/internal/services/activity_service.go
+++ b/backend/internal/services/activity_service.go
@@ -24,6 +24,7 @@ const (
defaultActivityRetentionDays = 30
defaultActivityHistoryLimit = 1000
defaultActivityMessages = 500
+ staleImageUpdateCheckAge = 6 * time.Hour
)
type ActivityService struct {
@@ -459,6 +460,37 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act
const cancelledMessageInternal = "Cancelled by user"
+// FailStaleImageUpdateChecks marks image update checks that were left running
+// across a prior process lifetime as failed. It intentionally scopes cleanup to
+// old image-update-check activities so startup repair cannot affect other work.
+func (s *ActivityService) FailStaleImageUpdateChecks(ctx context.Context) (int64, error) {
+ if s == nil || s.db == nil {
+ return 0, nil
+ }
+
+ cutoff := time.Now().Add(-staleImageUpdateCheckAge)
+ var staleChecks []models.Activity
+ if err := s.db.WithContext(ctx).
+ Where("type = ? AND status = ? AND started_at < ?", models.ActivityTypeImageUpdateCheck, models.ActivityStatusRunning, cutoff).
+ Find(&staleChecks).Error; err != nil {
+ return 0, fmt.Errorf("find stale image update checks: %w", err)
+ }
+
+ const message = "Image update check failed because it was stale after Arcane restarted"
+ errMessage := message
+ var failed int64
+ var failErrs []error
+ for i := range staleChecks {
+ if _, err := s.CompleteActivity(ctx, staleChecks[i].ID, models.ActivityStatusFailed, message, &errMessage, "Image update check failed"); err != nil {
+ failErrs = append(failErrs, fmt.Errorf("fail stale image update check %s: %w", staleChecks[i].ID, err))
+ continue
+ }
+ failed++
+ }
+
+ return failed, errors.Join(failErrs...)
+}
+
func completeActivityUpdatesInternal(startedAt time.Time, status models.ActivityStatus, finalMessage string, errMessage *string, finalStep []string, now time.Time) map[string]any {
updates := map[string]any{
"status": status,
diff --git a/backend/internal/services/activity_service_test.go b/backend/internal/services/activity_service_test.go
index 6419fbb5b..e8cbdfa70 100644
--- a/backend/internal/services/activity_service_test.go
+++ b/backend/internal/services/activity_service_test.go
@@ -349,6 +349,74 @@ func TestActivityServiceCancelActivityInternal(t *testing.T) {
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
}
+func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {
+ ctx := context.Background()
+ db := setupActivityServiceTestDBInternal(t)
+ service := NewActivityService(db)
+
+ staleCheck, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImageUpdateCheck,
+ LatestMessage: "checking",
+ })
+ require.NoError(t, err)
+ freshCheck, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImageUpdateCheck,
+ LatestMessage: "checking",
+ })
+ require.NoError(t, err)
+ staleOtherType, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImagePull,
+ LatestMessage: "pulling",
+ })
+ require.NoError(t, err)
+ completedCheck, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImageUpdateCheck,
+ LatestMessage: "checking",
+ })
+ require.NoError(t, err)
+ _, err = service.CompleteActivity(ctx, completedCheck.ID, models.ActivityStatusSuccess, "complete", nil)
+ require.NoError(t, err)
+
+ oldStartedAt := time.Now().Add(-7 * time.Hour)
+ for _, id := range []string{staleCheck.ID, staleOtherType.ID, completedCheck.ID} {
+ require.NoError(t, db.Model(&models.Activity{}).Where("id = ?", id).Updates(map[string]any{
+ "started_at": oldStartedAt,
+ "updated_at": oldStartedAt,
+ }).Error)
+ }
+
+ failed, err := service.FailStaleImageUpdateChecks(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, 1, failed)
+
+ var stale models.Activity
+ require.NoError(t, db.First(&stale, "id = ?", staleCheck.ID).Error)
+ require.Equal(t, models.ActivityStatusFailed, stale.Status)
+ require.NotNil(t, stale.EndedAt)
+ require.NotNil(t, stale.DurationMs)
+ require.Contains(t, stale.LatestMessage, "stale")
+ require.NotNil(t, stale.Error)
+ require.Contains(t, *stale.Error, "stale")
+
+ var fresh models.Activity
+ require.NoError(t, db.First(&fresh, "id = ?", freshCheck.ID).Error)
+ require.Equal(t, models.ActivityStatusRunning, fresh.Status)
+ require.Nil(t, fresh.EndedAt)
+
+ var other models.Activity
+ require.NoError(t, db.First(&other, "id = ?", staleOtherType.ID).Error)
+ require.Equal(t, models.ActivityStatusRunning, other.Status)
+ require.Nil(t, other.EndedAt)
+
+ var completed models.Activity
+ require.NoError(t, db.First(&completed, "id = ?", completedCheck.ID).Error)
+ require.Equal(t, models.ActivityStatusSuccess, completed.Status)
+}
+
func receiveActivityEventInternal(t *testing.T, events <-chan activitytypes.StreamEvent) activitytypes.StreamEvent {
t.Helper()
diff --git a/backend/internal/services/image_update_service.go b/backend/internal/services/image_update_service.go
index 5845d5040..a7ddc2a9c 100644
--- a/backend/internal/services/image_update_service.go
+++ b/backend/internal/services/image_update_service.go
@@ -17,6 +17,7 @@ import (
"github.com/getarcaneapp/arcane/backend/pkg/libarcane/crypto"
"github.com/getarcaneapp/arcane/backend/pkg/libarcane/ratelimit"
registry "github.com/getarcaneapp/arcane/backend/pkg/libarcane/registryauth"
+ "github.com/getarcaneapp/arcane/backend/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/pkg/utils"
"github.com/getarcaneapp/arcane/types/containerregistry"
"github.com/getarcaneapp/arcane/types/imageupdate"
@@ -65,6 +66,33 @@ func NewImageUpdateService(db *database.DB, settingsService *SettingsService, re
}
}
+func (s *ImageUpdateService) dockerAPIContextInternal(ctx context.Context) (context.Context, context.CancelFunc) {
+ timeoutSeconds := 0
+ if s != nil && s.settingsService != nil {
+ timeoutSeconds = s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt()
+ }
+ return timeouts.WithTimeout(ctx, timeoutSeconds, timeouts.DefaultDockerAPI)
+}
+
+func (s *ImageUpdateService) registryContextInternal(ctx context.Context) (context.Context, context.CancelFunc) {
+ timeoutSeconds := 0
+ if s != nil && s.settingsService != nil {
+ timeoutSeconds = s.settingsService.GetSettingsConfig().RegistryTimeout.AsInt()
+ }
+ return timeouts.WithTimeout(ctx, timeoutSeconds, timeouts.DefaultRegistry)
+}
+
+func (s *ImageUpdateService) dockerClientInternal(ctx context.Context) (*client.Client, error) {
+ if s == nil || s.dockerService == nil {
+ return nil, errors.New("docker service unavailable")
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ }
+ return dockerClient, nil
+}
+
func (s *ImageUpdateService) startImageUpdateActivityInternal(ctx context.Context, resourceName string, count int) string {
if s.activityService == nil {
return ""
@@ -214,7 +242,9 @@ func (s *ImageUpdateService) checkDigestUpdateWithSnapshotInternal(ctx context.C
imageRef := fmt.Sprintf("%s/%s:%s", parts.Registry, parts.Repository, parts.Tag)
start := time.Now()
- digestResult, err := s.registryService.inspectImageDigestInternal(ctx, imageRef, nil)
+ registryCtx, registryCancel := s.registryContextInternal(ctx)
+ digestResult, err := s.registryService.inspectImageDigestInternal(registryCtx, imageRef, nil)
+ registryCancel()
elapsed := time.Since(start)
if err != nil {
partial := digestResult // may contain auth metadata even on error
@@ -359,9 +389,9 @@ func (s *ImageUpdateService) parseImageReferenceFallback(imageRef string) *Image
}
func (s *ImageUpdateService) getImageRefByIDInternal(ctx context.Context, imageID string) (string, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ dockerClient, err := s.dockerClientInternal(ctx)
if err != nil {
- return "", fmt.Errorf("failed to connect to Docker: %w", err)
+ return "", err
}
imageID = strings.TrimPrefix(imageID, "sha256:")
@@ -380,7 +410,10 @@ func (s *ImageUpdateService) getImageRefByIDInternal(ctx context.Context, imageI
}
func (s *ImageUpdateService) resolveImageRefFromInspect(ctx context.Context, dockerClient client.APIClient, imageID string) (string, error) {
- inspectResponse, err := dockerClient.ImageInspect(ctx, imageID)
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ inspectResponse, err := dockerClient.ImageInspect(apiCtx, imageID)
if err != nil {
return "", err
}
@@ -401,7 +434,10 @@ func (s *ImageUpdateService) resolveImageRefFromInspect(ctx context.Context, doc
func (s *ImageUpdateService) resolveImageRefFromContainers(ctx context.Context, dockerClient client.APIClient, imageID string) (string, error) {
fullID := "sha256:" + imageID
- containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ containers, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})
if err != nil {
return "", err
}
@@ -417,12 +453,15 @@ func (s *ImageUpdateService) resolveImageRefFromContainers(ctx context.Context,
}
func (s *ImageUpdateService) getAllImageRefsInternal(ctx context.Context, limit int) ([]string, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ dockerClient, err := s.dockerClientInternal(ctx)
if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ return nil, err
}
- imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{})
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ imageList, err := dockerClient.ImageList(apiCtx, client.ImageListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to list Docker images: %w", err)
}
@@ -451,12 +490,15 @@ func dedupeImageRefsFromSummariesInternal(images []image.Summary, limit int) []s
}
func (s *ImageUpdateService) inspectLocalImageSnapshotInternal(ctx context.Context, imageRef string) (*localImageSnapshot, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ dockerClient, err := s.dockerClientInternal(ctx)
if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ return nil, err
}
- inspectResponse, err := dockerClient.ImageInspect(ctx, imageRef)
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ inspectResponse, err := dockerClient.ImageInspect(apiCtx, imageRef)
if err != nil {
return nil, fmt.Errorf("failed to inspect image: %w", err)
}
@@ -725,12 +767,15 @@ func savePreparedUpdateResultWithTxInternal(tx *gorm.DB, imageID, repo, tag stri
}
func (s *ImageUpdateService) saveUpdateResultByIDInternal(ctx context.Context, imageID string, result *imageupdate.Response) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ dockerClient, err := s.dockerClientInternal(ctx)
if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
+ return err
}
- dockerImage, err := dockerClient.ImageInspect(ctx, imageID)
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ dockerImage, err := dockerClient.ImageInspect(apiCtx, imageID)
if err != nil {
return fmt.Errorf("failed to inspect image: %w", err)
}
@@ -746,12 +791,15 @@ func (s *ImageUpdateService) savePreparedUpdateResultInternal(ctx context.Contex
}
func (s *ImageUpdateService) getImageIDByRef(ctx context.Context, imageRef string) (string, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ dockerClient, err := s.dockerClientInternal(ctx)
if err != nil {
- return "", fmt.Errorf("failed to connect to Docker: %w", err)
+ return "", err
}
- inspectResponse, err := dockerClient.ImageInspect(ctx, imageRef)
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ inspectResponse, err := dockerClient.ImageInspect(apiCtx, imageRef)
if err != nil {
return "", fmt.Errorf("image not found: %w", err)
}
@@ -855,6 +903,25 @@ type batchImage struct {
parts *ImageParts
}
+type batchImageProgressRecorder struct {
+ mu sync.Mutex
+ results map[string]*imageupdate.Response
+ completed int
+ total int
+}
+
+func (r *batchImageProgressRecorder) recordInternal(refs []string, res *imageupdate.Response) int {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+
+ r.completed++
+ progress := 10 + int(float64(r.completed)/float64(r.total)*80)
+ for _, ref := range refs {
+ r.results[ref] = res
+ }
+ return progress
+}
+
func (s *ImageUpdateService) parseAndGroupImagesInternal(imageRefs []string) (map[string]map[string]struct{}, map[string]*imageupdate.Response, []batchImage) {
regRepos := make(map[string]map[string]struct{})
results := make(map[string]*imageupdate.Response)
@@ -902,7 +969,9 @@ func (s *ImageUpdateService) checkSingleImageInBatchInternal(ctx context.Context
start := time.Now()
imageRef := fmt.Sprintf("%s/%s:%s", parts.Registry, parts.Repository, parts.Tag)
- digestResult, digestErr := s.registryService.inspectImageDigestInternal(ctx, imageRef, externalCreds)
+ registryCtx, registryCancel := s.registryContextInternal(ctx)
+ digestResult, digestErr := s.registryService.inspectImageDigestInternal(registryCtx, imageRef, externalCreds)
+ registryCancel()
if digestErr != nil {
resp := &imageupdate.Response{
Error: digestErr.Error(),
@@ -1004,6 +1073,52 @@ func (s *ImageUpdateService) resolveBatchCredentialsInternal(ctx context.Context
return credentials
}
+func (s *ImageUpdateService) checkBatchImageInternal(ctx context.Context, activityID string, resolvedCreds []containerregistry.Credential, img batchImage, recorder *batchImageProgressRecorder) error {
+ registry := img.parts.Registry
+
+ if err := s.registryLimiter.Acquire(ctx, registry); err != nil {
+ slog.DebugContext(ctx, "skipping image check: registry limiter acquire failed",
+ "imageRef", img.canonicalRef,
+ "registry", registry,
+ "error", err.Error())
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+
+ res := &imageupdate.Response{
+ Error: err.Error(),
+ CheckTime: time.Now(),
+ ResponseTimeMs: 0,
+ ActivityID: utils.StringPtrFromTrimmed(activityID),
+ }
+ s.recordBatchImageCheckInternal(ctx, activityID, img, res, nil, recorder)
+ return nil
+ }
+ defer s.registryLimiter.Release(registry)
+
+ res, snapshot := s.checkSingleImageInBatchInternal(ctx, resolvedCreds, img.parts)
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if res != nil {
+ res.ActivityID = utils.StringPtrFromTrimmed(activityID)
+ }
+
+ s.recordBatchImageCheckInternal(ctx, activityID, img, res, snapshot, recorder)
+ return nil
+}
+
+func (s *ImageUpdateService) recordBatchImageCheckInternal(ctx context.Context, activityID string, img batchImage, res *imageupdate.Response, snapshot *localImageSnapshot, recorder *batchImageProgressRecorder) {
+ progress := recorder.recordInternal(img.refs, res)
+
+ level, message := imageCheckResultMessageInternal(img.canonicalRef, res)
+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, level, message, progress, "Checking image")
+
+ if err := s.saveUpdateResultWithSnapshotInternal(ctx, img.canonicalRef, res, snapshot); err != nil {
+ slog.WarnContext(ctx, "Failed to save update result", "imageRef", img.canonicalRef, "error", err.Error())
+ }
+}
+
func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs []string, externalCreds []containerregistry.Credential) (map[string]*imageupdate.Response, error) {
startBatch := time.Now()
results := make(map[string]*imageupdate.Response, len(imageRefs))
@@ -1028,44 +1143,16 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs
slog.DebugContext(ctx, "Resolved batch registry credentials", "credentialCount", len(resolvedCreds), "registryCount", len(regRepos))
- var mu sync.Mutex
- completed := 0
+ recorder := &batchImageProgressRecorder{
+ results: results,
+ total: len(images),
+ }
g, groupCtx := errgroup.WithContext(ctx)
g.SetLimit(10) // Limit concurrency
for _, img := range images {
g.Go(func() error {
- registry := img.parts.Registry
-
- if err := s.registryLimiter.Acquire(groupCtx, registry); err != nil {
- slog.DebugContext(groupCtx, "skipping image check: registry limiter acquire failed",
- "imageRef", img.canonicalRef,
- "registry", registry,
- "error", err.Error())
- return err
- }
- defer s.registryLimiter.Release(registry)
-
- res, snapshot := s.checkSingleImageInBatchInternal(groupCtx, resolvedCreds, img.parts)
- if res != nil {
- res.ActivityID = utils.StringPtrFromTrimmed(activityID)
- }
-
- mu.Lock()
- completed++
- progress := 10 + int(float64(completed)/float64(len(images))*80)
- for _, ref := range img.refs {
- results[ref] = res
- }
- mu.Unlock()
-
- level, message := imageCheckResultMessageInternal(img.canonicalRef, res)
- s.appendImageUpdateActivityMessageInternal(groupCtx, activityID, level, message, progress, "Checking image")
-
- if err := s.saveUpdateResultWithSnapshotInternal(groupCtx, img.canonicalRef, res, snapshot); err != nil {
- slog.WarnContext(groupCtx, "Failed to save update result", "imageRef", img.canonicalRef, "error", err.Error())
- }
- return nil
+ return s.checkBatchImageInternal(groupCtx, activityID, resolvedCreds, img, recorder)
})
}
@@ -1167,13 +1254,16 @@ func (s *ImageUpdateService) CleanupOrphanedRecords(ctx context.Context) error {
return nil
}
- dockerClient, err := s.dockerService.GetClient(ctx)
+ dockerClient, err := s.dockerClientInternal(ctx)
if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
+ return err
}
// Get all image IDs from Docker
- dockerImagesResult, err := dockerClient.ImageList(ctx, client.ImageListOptions{})
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ dockerImagesResult, err := dockerClient.ImageList(apiCtx, client.ImageListOptions{})
if err != nil {
return fmt.Errorf("failed to list Docker images: %w", err)
}
@@ -1203,12 +1293,15 @@ func (s *ImageUpdateService) CleanupOrphanedRecords(ctx context.Context) error {
}
func (s *ImageUpdateService) GetUpdateSummary(ctx context.Context) (*imageupdate.Summary, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ dockerClient, err := s.dockerClientInternal(ctx)
if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ return nil, err
}
- dockerImagesResult, err := dockerClient.ImageList(ctx, client.ImageListOptions{})
+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)
+ defer cancel()
+
+ dockerImagesResult, err := dockerClient.ImageList(apiCtx, client.ImageListOptions{})
if err != nil {
return nil, fmt.Errorf("failed to list Docker images: %w", err)
}
diff --git a/backend/internal/services/image_update_service_test.go b/backend/internal/services/image_update_service_test.go
index 2b243a617..25fabb86d 100644
--- a/backend/internal/services/image_update_service_test.go
+++ b/backend/internal/services/image_update_service_test.go
@@ -629,6 +629,83 @@ func TestImageUpdateService_CheckMultipleImagesCompletesActivityWhenRequestConte
assert.Contains(t, activity.LatestMessage, "Image update check failed")
}
+func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInternal(t *testing.T) {
+ db := setupImageUpdateTestDB(t)
+ require.NoError(t, db.AutoMigrate(&models.Activity{}, &models.ActivityMessage{}))
+
+ settingsService := newImageUpdateTestSettingsServiceInternal("1", "30")
+ activityService := NewActivityService(db)
+ dockerServer := newImageUpdateRegistryOnlyServer(t, "team/app:1.2.3", digest.FromString("unused").String())
+ defer dockerServer.Close()
+ registryService := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ <-ctx.Done()
+ return client.DistributionInspectResult{}, ctx.Err()
+ },
+ }, nil
+ }, nil)
+
+ parentCtx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond)
+ defer cancel()
+
+ svc := NewImageUpdateService(db, settingsService, registryService, &DockerClientService{client: newTestDockerClient(t, dockerServer)}, nil, nil, activityService)
+
+ start := time.Now()
+ results, err := svc.CheckMultipleImages(parentCtx, []string{"registry.example.com/team/app:1.2.3"}, nil)
+ elapsed := time.Since(start)
+
+ require.NoError(t, err)
+ require.Less(t, elapsed, 2*time.Second)
+ require.Contains(t, results, "registry.example.com/team/app:1.2.3")
+ require.NotNil(t, results["registry.example.com/team/app:1.2.3"])
+ require.Contains(t, results["registry.example.com/team/app:1.2.3"].Error, context.DeadlineExceeded.Error())
+
+ var activity models.Activity
+ require.NoError(t, db.Where("type = ?", models.ActivityTypeImageUpdateCheck).First(&activity).Error)
+ require.Equal(t, models.ActivityStatusFailed, activity.Status)
+ require.NotNil(t, activity.EndedAt)
+ require.NotNil(t, activity.DurationMs)
+ require.Equal(t, "Image update check complete", activity.Step)
+ require.Contains(t, activity.LatestMessage, "0 checked, 1 errors")
+}
+
+func TestImageUpdateService_GetAllImageRefsUsesDockerAPITimeoutInternal(t *testing.T) {
+ db := setupImageUpdateTestDB(t)
+ settingsService := newImageUpdateTestSettingsServiceInternal("30", "1")
+ server := newBlockedDockerAPIServerInternal(t, "/images/json")
+ svc := NewImageUpdateService(db, settingsService, nil, &DockerClientService{client: newTestDockerClient(t, server)}, nil, nil, nil)
+
+ parentCtx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond)
+ defer cancel()
+
+ start := time.Now()
+ _, err := svc.getAllImageRefsInternal(parentCtx, 0)
+ elapsed := time.Since(start)
+
+ require.Error(t, err)
+ require.Less(t, elapsed, 2*time.Second)
+ require.Contains(t, err.Error(), context.DeadlineExceeded.Error())
+}
+
+func TestImageUpdateService_InspectLocalImageSnapshotUsesDockerAPITimeoutInternal(t *testing.T) {
+ db := setupImageUpdateTestDB(t)
+ settingsService := newImageUpdateTestSettingsServiceInternal("30", "1")
+ server := newBlockedDockerAPIServerInternal(t, "/images/")
+ svc := NewImageUpdateService(db, settingsService, nil, &DockerClientService{client: newTestDockerClient(t, server)}, nil, nil, nil)
+
+ parentCtx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond)
+ defer cancel()
+
+ start := time.Now()
+ _, err := svc.inspectLocalImageSnapshotInternal(parentCtx, "registry.example.com/team/app:1.2.3")
+ elapsed := time.Since(start)
+
+ require.Error(t, err)
+ require.Less(t, elapsed, 2*time.Second)
+ require.Contains(t, err.Error(), context.DeadlineExceeded.Error())
+}
+
func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstAttempt(t *testing.T) {
_, db := setupImageServiceAuthTest(t)
require.NoError(t, db.AutoMigrate(&models.ImageUpdateRecord{}, &models.Event{}))
@@ -1259,6 +1336,29 @@ func TestDedupeImageRefsFromSummaries_WithLimit(t *testing.T) {
assert.Equal(t, []string{"nginx:latest", "redis:7"}, refsLimited)
}
+func newImageUpdateTestSettingsServiceInternal(registryTimeout, dockerAPITimeout string) *SettingsService {
+ settingsService := &SettingsService{}
+ cfg := DefaultSettingsConfig()
+ cfg.RegistryTimeout.Value = registryTimeout
+ cfg.DockerAPITimeout.Value = dockerAPITimeout
+ settingsService.config.Store(cfg)
+ return settingsService
+}
+
+func newBlockedDockerAPIServerInternal(t *testing.T, pathContains string) *httptest.Server {
+ t.Helper()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, pathContains) {
+ <-r.Context().Done()
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ t.Cleanup(server.Close)
+ return server
+}
+
func containsAll(set map[string]struct{}, refs ...string) bool {
for _, ref := range refs {
if _, ok := set[ref]; !ok {