Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: move project image backfill into image watcher startup (#3307)
Kyle Mendelld6764706db2c
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex 38ebcc351..ce07794f0 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -184,7 +184,6 @@ func initializeStartupState(appCtx context.Context, cfg *config.Config, appServiif err := appServices.Project.RecoverProjectRenameJournals(appCtx); err != nil {slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)}- appServices.Project.BackfillProjectImageRefs(appCtx)}if !cfg.AgentMode {diff --git a/backend/internal/di/wire_gen.go b/backend/internal/di/wire_gen.goindex 55ef195c7..cca73a9cc 100644--- a/backend/internal/di/wire_gen.go+++ b/backend/internal/di/wire_gen.go@@ -124,7 +124,8 @@ func InitializeJobs(ctx context.Context, cfg *config.Config, svcs *Services) *JoimageUpdateService := svcs.ImageUpdateenvironmentService := svcs.EnvironmentdockerClientService := svcs.Docker- imageUpdateWatcher := scheduler.NewImageUpdateWatcher(imageUpdateService, settingsService, environmentService, dockerClientService)+ projectService := svcs.Project+ imageUpdateWatcher := scheduler.NewImageUpdateWatcher(imageUpdateService, settingsService, environmentService, dockerClientService, projectService)dockerClientRefreshJob := scheduler.NewDockerClientRefreshJob(dockerClientService, settingsService)kvService := svcs.KVanalyticsJob := provideAnalyticsJobInternal(settingsService, kvService, cfg)@@ -138,7 +139,6 @@ func InitializeJobs(ctx context.Context, cfg *config.Config, svcs *Services) *JosystemService := svcs.SystemnotificationService := svcs.NotificationscheduledPruneJob := scheduler.NewScheduledPruneJob(systemService, settingsService, notificationService)- projectService := svcs.ProjecttemplateService := svcs.TemplatefilesystemWatcherJob := provideFilesystemWatcherJobInternal(ctx, projectService, templateService, settingsService, cfg)vulnerabilityService := svcs.Vulnerabilitydiff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.goindex 4ab0bf0c1..1e4cf4d50 100644--- a/backend/internal/services/project_service.go+++ b/backend/internal/services/project_service.go@@ -630,21 +630,24 @@ func (s *ProjectService) HandleProjectFilesChanged(ctx context.Context, paths []}}-func (s *ProjectService) BackfillProjectImageRefs(ctx context.Context) {+func (s *ProjectService) BackfillProjectImageRefs(ctx context.Context) (int, error) {if s.db == nil {- return+ return 0, nil}var projectsList []models.Projectif err := s.db.WithContext(ctx).Where("build_image_refs_json IS NULL").Find(&projectsList).Error; err != nil {- slog.WarnContext(ctx, "failed to list projects for image ref backfill", "error", err)- return+ return 0, fmt.Errorf("list projects for image ref backfill: %w", err)}for i := range projectsList {+ if err := ctx.Err(); err != nil {+ return i, err+ }s.refreshProjectImageRefsInternal(ctx, &projectsList[i])}+ return len(projectsList), nil}func (s *ProjectService) resolveProjectsByChangedPathsInternal(ctx context.Context, paths []string) ([]models.Project, error) {diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.goindex 6b531c1e1..55ead3037 100644--- a/backend/internal/services/project_service_test.go+++ b/backend/internal/services/project_service_test.go@@ -142,7 +142,9 @@ func TestProjectService_BackfillProjectImageRefs_RetriesOnlyMissingMetadata(t *trequire.NoError(t, db.Create(regular).Error)svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())- svc.BackfillProjectImageRefs(ctx)+ count, err := svc.BackfillProjectImageRefs(ctx)+ require.NoError(t, err)+ require.Equal(t, 3, count)var savedValid models.Projectrequire.NoError(t, db.First(&savedValid, "id = ?", valid.ID).Error)@@ -160,7 +162,9 @@ func TestProjectService_BackfillProjectImageRefs_RetriesOnlyMissingMetadata(t *trequire.NoError(t, os.Remove(filepath.Join(validPath, "compose.yaml")))require.NoError(t, os.Remove(filepath.Join(regularPath, "compose.yaml")))- svc.BackfillProjectImageRefs(ctx)+ count, err = svc.BackfillProjectImageRefs(ctx)+ require.NoError(t, err)+ require.Equal(t, 1, count)require.NoError(t, db.First(&savedValid, "id = ?", valid.ID).Error)require.NotNil(t, savedValid.BuildImageRefsJSON)assert.Equal(t, []string{"local-app:latest"}, projects.ParseImageRefsJSON(*savedValid.BuildImageRefsJSON))diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.goindex 2d217bc5c..797d9937e 100644--- a/backend/pkg/scheduler/image_update_watcher.go+++ b/backend/pkg/scheduler/image_update_watcher.go@@ -16,7 +16,10 @@ import ("go.getarcane.app/streams/bus")-const imageUpdateWatcherDebounce = 2 * time.Second+const (+ imageUpdateWatcherDebounce = 2 * time.Second+ imageUpdateWatcherBackfillRetry = 5 * time.Second+)type imageUpdateScannerInternal interface {CheckAllImages(ctx context.Context, limit int, externalCreds []containerregistry.Credential) (map[string]*imageupdate.Response, error)@@ -34,6 +37,10 @@ type dockerEventBusProviderInternal interface {EventBus() *bus.DockerEventBus}+type projectImageRefsBackfillerInternal interface {+ BackfillProjectImageRefs(ctx context.Context) (int, error)+}+type imageUpdateScanRunInternal struct {done chan struct{}}@@ -44,20 +51,27 @@ type ImageUpdateWatcher struct {settingsService pollingSettingReaderInternalenvironmentService registryCredentialLoaderInternaldockerService dockerEventBusProviderInternal+ projectService projectImageRefsBackfillerInternaltriggerCh chan struct{}debounce time.Duration+ backfillRetry time.Duration+ metadataReady chan struct{}+ metadataReadyOnce sync.OnceactiveRun atomic.Pointer[imageUpdateScanRunInternal]}// NewImageUpdateWatcher constructs the image update watcher from the existing services.-func NewImageUpdateWatcher(imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService) *ImageUpdateWatcher {+func NewImageUpdateWatcher(imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService, projectService *services.ProjectService) *ImageUpdateWatcher {return &ImageUpdateWatcher{imageUpdateService: imageUpdateService,settingsService: settingsService,environmentService: environmentService,dockerService: dockerService,+ projectService: projectService,triggerCh: make(chan struct{}, 1),debounce: imageUpdateWatcherDebounce,+ backfillRetry: imageUpdateWatcherBackfillRetry,+ metadataReady: make(chan struct{}),}}@@ -66,17 +80,22 @@ func (w *ImageUpdateWatcher) Name() string {return "image-polling"}-// Start subscribes to Docker image events and runs scans until ctx is canceled.+// Start subscribes to Docker image events, prepares project metadata, and runs scans until ctx is canceled.func (w *ImageUpdateWatcher) Start(ctx context.Context) error {if w == nil || w.dockerService == nil || w.dockerService.EventBus() == nil {return errors.New("docker event bus unavailable")}+ if w.projectService == nil || w.metadataReady == nil {+ return errors.New("project image metadata backfiller unavailable")+ }eventCh, unsubscribe := w.dockerService.EventBus().Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))defer unsubscribe()var listener sync.WaitGroup+ listenerStarted := make(chan struct{})listener.Go(func() {+ close(listenerStarted)for {select {case <-ctx.Done():@@ -90,11 +109,25 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {}})+ select {+ case <-ctx.Done():+ listener.Wait()+ return nil+ case <-listenerStarted:+ }+slog.InfoContext(ctx, "image update watcher started")+ defer func() {+ listener.Wait()+ slog.InfoContext(ctx, "image update watcher stopped")+ }()++ if !w.backfillProjectImageRefsInternal(ctx) {+ return nil+ }+w.Trigger()w.runTriggeredScansInternal(ctx)- listener.Wait()- slog.InfoContext(ctx, "image update watcher stopped")return nil}@@ -111,9 +144,14 @@ func (w *ImageUpdateWatcher) Trigger() {// RunNow performs the same full-host image scan used by automatic watcher triggers.func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {- if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil {+ if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil || w.metadataReady == nil {return errors.New("image update watcher is not initialized")}+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-w.metadataReady:+ }run := &imageUpdateScanRunInternal{done: make(chan struct{})}for {@@ -177,6 +215,40 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {return nil}+func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context) bool {+ retryDelay := w.backfillRetry+ if retryDelay <= 0 {+ retryDelay = imageUpdateWatcherBackfillRetry+ }++ for attempt := 1; ; attempt++ {+ if ctx.Err() != nil {+ return false+ }++ startedAt := time.Now()+ count, err := w.projectService.BackfillProjectImageRefs(ctx)+ duration := time.Since(startedAt)+ if err == nil {+ slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", duration, "attempt", attempt)+ w.metadataReadyOnce.Do(func() { close(w.metadataReady) })+ return true+ }+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {+ return false+ }++ slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", duration, "attempt", attempt, "retryIn", retryDelay, "error", err)+ timer := time.NewTimer(retryDelay)+ select {+ case <-ctx.Done():+ timer.Stop()+ return false+ case <-timer.C:+ }+ }+}+func (w *ImageUpdateWatcher) runTriggeredScansInternal(ctx context.Context) {for {select {diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex ca5297d4b..1837c3a2f 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -1,8 +1,11 @@package schedulerimport (+ "bytes""context""errors"+ "log/slog"+ "strings""sync""testing""time"@@ -102,17 +105,77 @@ func (p dockerEventBusProviderFakeInternal) EventBus() *bus.DockerEventBus {return p.eventBus}-func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus) *ImageUpdateWatcher {+type projectImageRefsBackfillerFakeInternal struct {+ mu sync.Mutex+ calls int+ run func(ctx context.Context, call int) (int, error)+ started chan int+}++func (b *projectImageRefsBackfillerFakeInternal) BackfillProjectImageRefs(ctx context.Context) (int, error) {+ b.mu.Lock()+ b.calls+++ call := b.calls+ run := b.run+ started := b.started+ b.mu.Unlock()++ if started != nil {+ select {+ case started <- call:+ default:+ }+ }+ if run == nil {+ return 0, nil+ }+ return run(ctx, call)+}++func (b *projectImageRefsBackfillerFakeInternal) countInternal() int {+ b.mu.Lock()+ defer b.mu.Unlock()+ return b.calls+}++type lockedBufferInternal struct {+ mu sync.Mutex+ bytes.Buffer+}++func (b *lockedBufferInternal) Write(p []byte) (int, error) {+ b.mu.Lock()+ defer b.mu.Unlock()+ return b.Buffer.Write(p)+}++func (b *lockedBufferInternal) stringInternal() string {+ b.mu.Lock()+ defer b.mu.Unlock()+ return b.Buffer.String()+}++func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {+ if backfiller == nil {+ backfiller = &projectImageRefsBackfillerFakeInternal{}+ }return &ImageUpdateWatcher{imageUpdateService: scanner,settingsService: settings,environmentService: registryCredentialLoaderFakeInternal{},dockerService: dockerEventBusProviderFakeInternal{eventBus: eventBus},+ projectService: backfiller,triggerCh: make(chan struct{}, 1),debounce: 10 * time.Millisecond,+ backfillRetry: 10 * time.Millisecond,+ metadataReady: make(chan struct{}),}}+func markImageUpdateWatcherMetadataReadyForTestInternal(watcher *ImageUpdateWatcher) {+ watcher.metadataReadyOnce.Do(func() { close(watcher.metadataReady) })+}+func startImageUpdateWatcherForTestInternal(t *testing.T, watcher *ImageUpdateWatcher) (context.CancelFunc, <-chan error) {t.Helper()ctx, cancel := context.WithCancel(context.Background())@@ -131,7 +194,7 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tesscanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -167,7 +230,7 @@ func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testin}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Equal(t, 1, <-scanner.startedCh)@@ -186,7 +249,7 @@ func TestImageUpdateWatcher_DisabledTriggersAreSkippedUntilEnabled(t *testing.T)scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: false}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)time.Sleep(30 * time.Millisecond)@@ -205,7 +268,7 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{errors: []error{errors.New("registry unavailable")}}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -220,7 +283,8 @@ func TestImageUpdateWatcher_RunNowSerializesConcurrentCalls(t *testing.T) {releaseCh: releaseCh,}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus())+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)+ markImageUpdateWatcherMetadataReadyForTestInternal(watcher)errCh := make(chan error, 2)go func() { errCh <- watcher.RunNow(context.Background()) }()@@ -243,7 +307,8 @@ func TestImageUpdateWatcher_RunNowWaiterHonorsCancellation(t *testing.T) {releaseCh: releaseCh,}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus())+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)+ markImageUpdateWatcherMetadataReadyForTestInternal(watcher)firstErrCh := make(chan error, 1)go func() { firstErrCh <- watcher.RunNow(context.Background()) }()@@ -257,3 +322,124 @@ func TestImageUpdateWatcher_RunNowWaiterHonorsCancellation(t *testing.T) {close(releaseCh)require.NoError(t, <-firstErrCh)}++func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *testing.T) {+ const (+ projectCount = 2500+ eventCount = 10000+ )++ var logBuffer lockedBufferInternal+ previousLogger := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{Level: slog.LevelInfo})))+ t.Cleanup(func() { slog.SetDefault(previousLogger) })++ backfillStarted := make(chan struct{})+ releaseBackfill := make(chan struct{})+ backfiller := &projectImageRefsBackfillerFakeInternal{+ run: func(ctx context.Context, _ int) (int, error) {+ close(backfillStarted)+ select {+ case <-ctx.Done():+ return 0, ctx.Err()+ case <-releaseBackfill:+ return projectCount, nil+ }+ },+ }+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ eventBus := bus.NewDockerEventBus()+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, backfiller)+ startImageUpdateWatcherForTestInternal(t, watcher)++ <-backfillStarted+ burstStartedAt := time.Now()+ for range eventCount {+ eventBus.Publish(events.Message{Type: events.ImageEventType, Action: events.ActionPull})+ }+ require.Never(t, func() bool { return scanner.countInternal() > 0 }, 30*time.Millisecond, 5*time.Millisecond)++ close(releaseBackfill)+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)+ time.Sleep(30 * time.Millisecond)+ require.Equal(t, 1, scanner.countInternal())++ logs := logBuffer.stringInternal()+ require.True(t, strings.Contains(logs, "project image metadata backfill completed"), logs)+ require.True(t, strings.Contains(logs, "projects=2500"), logs)+ require.True(t, strings.Contains(logs, "duration="), logs)+ t.Logf("coalesced %d image events into one scan after backfilling %d projects in %s", eventCount, projectCount, time.Since(burstStartedAt))+}++func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {+ secondAttemptStarted := make(chan struct{})+ releaseSecondAttempt := make(chan struct{})+ backfiller := &projectImageRefsBackfillerFakeInternal{+ run: func(ctx context.Context, call int) (int, error) {+ if call == 1 {+ return 0, errors.New("database unavailable")+ }+ close(secondAttemptStarted)+ select {+ case <-ctx.Done():+ return 0, ctx.Err()+ case <-releaseSecondAttempt:+ return 42, nil+ }+ },+ }+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)+ startImageUpdateWatcherForTestInternal(t, watcher)++ select {+ case <-secondAttemptStarted:+ case <-time.After(time.Second):+ t.Fatal("backfill was not retried")+ }+ require.Zero(t, scanner.countInternal())+ close(releaseSecondAttempt)++ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)+ require.Equal(t, 2, backfiller.countInternal())+}++func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.T) {+ backfillStarted := make(chan struct{})+ backfiller := &projectImageRefsBackfillerFakeInternal{+ run: func(ctx context.Context, _ int) (int, error) {+ close(backfillStarted)+ <-ctx.Done()+ return 0, ctx.Err()+ },+ }+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)+ ctx, cancel := context.WithCancel(context.Background())+ errCh := make(chan error, 1)+ go func() { errCh <- watcher.Start(ctx) }()++ <-backfillStarted+ cancel()+ select {+ case err := <-errCh:+ require.NoError(t, err)+ case <-time.After(time.Second):+ t.Fatal("watcher did not stop after cancellation")+ }+ require.Zero(t, scanner.countInternal())+}++func TestImageUpdateWatcher_RunNowWaitsForMetadataReadiness(t *testing.T) {+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)++ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)+ defer cancel()+ require.ErrorIs(t, watcher.RunNow(ctx), context.DeadlineExceeded)+ require.Zero(t, scanner.countInternal())+}