Skip to content

Modern Docker Management, Designed for Everyone

fix: move project image backfill into image watcher startup (#3307)

Kyle Mendelld6764706db2c
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 38ebcc351..ce07794f0 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -184,7 +184,6 @@ func initializeStartupState(appCtx context.Context, cfg *config.Config, appServi
if err := appServices.Project.RecoverProjectRenameJournals(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)
}
- appServices.Project.BackfillProjectImageRefs(appCtx)
}
if !cfg.AgentMode {
diff --git a/backend/internal/di/wire_gen.go b/backend/internal/di/wire_gen.go
index 55ef195c7..cca73a9cc 100644
--- a/backend/internal/di/wire_gen.go
+++ b/backend/internal/di/wire_gen.go
@@ -124,7 +124,8 @@ func InitializeJobs(ctx context.Context, cfg *config.Config, svcs *Services) *Jo
imageUpdateService := svcs.ImageUpdate
environmentService := svcs.Environment
dockerClientService := svcs.Docker
- imageUpdateWatcher := scheduler.NewImageUpdateWatcher(imageUpdateService, settingsService, environmentService, dockerClientService)
+ projectService := svcs.Project
+ imageUpdateWatcher := scheduler.NewImageUpdateWatcher(imageUpdateService, settingsService, environmentService, dockerClientService, projectService)
dockerClientRefreshJob := scheduler.NewDockerClientRefreshJob(dockerClientService, settingsService)
kvService := svcs.KV
analyticsJob := provideAnalyticsJobInternal(settingsService, kvService, cfg)
@@ -138,7 +139,6 @@ func InitializeJobs(ctx context.Context, cfg *config.Config, svcs *Services) *Jo
systemService := svcs.System
notificationService := svcs.Notification
scheduledPruneJob := scheduler.NewScheduledPruneJob(systemService, settingsService, notificationService)
- projectService := svcs.Project
templateService := svcs.Template
filesystemWatcherJob := provideFilesystemWatcherJobInternal(ctx, projectService, templateService, settingsService, cfg)
vulnerabilityService := svcs.Vulnerability
diff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.go
index 4ab0bf0c1..1e4cf4d50 100644
--- a/backend/internal/services/project_service.go
+++ b/backend/internal/services/project_service.go
@@ -630,21 +630,24 @@ func (s *ProjectService) HandleProjectFilesChanged(ctx context.Context, paths []
}
}
-func (s *ProjectService) BackfillProjectImageRefs(ctx context.Context) {
+func (s *ProjectService) BackfillProjectImageRefs(ctx context.Context) (int, error) {
if s.db == nil {
- return
+ return 0, nil
}
var projectsList []models.Project
if err := s.db.WithContext(ctx).
Where("build_image_refs_json IS NULL").
Find(&projectsList).Error; err != nil {
- slog.WarnContext(ctx, "failed to list projects for image ref backfill", "error", err)
- return
+ return 0, fmt.Errorf("list projects for image ref backfill: %w", err)
}
for i := range projectsList {
+ if err := ctx.Err(); err != nil {
+ return i, err
+ }
s.refreshProjectImageRefsInternal(ctx, &projectsList[i])
}
+ return len(projectsList), nil
}
func (s *ProjectService) resolveProjectsByChangedPathsInternal(ctx context.Context, paths []string) ([]models.Project, error) {
diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.go
index 6b531c1e1..55ead3037 100644
--- a/backend/internal/services/project_service_test.go
+++ b/backend/internal/services/project_service_test.go
@@ -142,7 +142,9 @@ func TestProjectService_BackfillProjectImageRefs_RetriesOnlyMissingMetadata(t *t
require.NoError(t, db.Create(regular).Error)
svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
- svc.BackfillProjectImageRefs(ctx)
+ count, err := svc.BackfillProjectImageRefs(ctx)
+ require.NoError(t, err)
+ require.Equal(t, 3, count)
var savedValid models.Project
require.NoError(t, db.First(&savedValid, "id = ?", valid.ID).Error)
@@ -160,7 +162,9 @@ func TestProjectService_BackfillProjectImageRefs_RetriesOnlyMissingMetadata(t *t
require.NoError(t, os.Remove(filepath.Join(validPath, "compose.yaml")))
require.NoError(t, os.Remove(filepath.Join(regularPath, "compose.yaml")))
- svc.BackfillProjectImageRefs(ctx)
+ count, err = svc.BackfillProjectImageRefs(ctx)
+ require.NoError(t, err)
+ require.Equal(t, 1, count)
require.NoError(t, db.First(&savedValid, "id = ?", valid.ID).Error)
require.NotNil(t, savedValid.BuildImageRefsJSON)
assert.Equal(t, []string{"local-app:latest"}, projects.ParseImageRefsJSON(*savedValid.BuildImageRefsJSON))
diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.go
index 2d217bc5c..797d9937e 100644
--- a/backend/pkg/scheduler/image_update_watcher.go
+++ b/backend/pkg/scheduler/image_update_watcher.go
@@ -16,7 +16,10 @@ import (
"go.getarcane.app/streams/bus"
)
-const imageUpdateWatcherDebounce = 2 * time.Second
+const (
+ imageUpdateWatcherDebounce = 2 * time.Second
+ imageUpdateWatcherBackfillRetry = 5 * time.Second
+)
type imageUpdateScannerInternal interface {
CheckAllImages(ctx context.Context, limit int, externalCreds []containerregistry.Credential) (map[string]*imageupdate.Response, error)
@@ -34,6 +37,10 @@ type dockerEventBusProviderInternal interface {
EventBus() *bus.DockerEventBus
}
+type projectImageRefsBackfillerInternal interface {
+ BackfillProjectImageRefs(ctx context.Context) (int, error)
+}
+
type imageUpdateScanRunInternal struct {
done chan struct{}
}
@@ -44,20 +51,27 @@ type ImageUpdateWatcher struct {
settingsService pollingSettingReaderInternal
environmentService registryCredentialLoaderInternal
dockerService dockerEventBusProviderInternal
+ projectService projectImageRefsBackfillerInternal
triggerCh chan struct{}
debounce time.Duration
+ backfillRetry time.Duration
+ metadataReady chan struct{}
+ metadataReadyOnce sync.Once
activeRun atomic.Pointer[imageUpdateScanRunInternal]
}
// NewImageUpdateWatcher constructs the image update watcher from the existing services.
-func NewImageUpdateWatcher(imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService) *ImageUpdateWatcher {
+func NewImageUpdateWatcher(imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService, projectService *services.ProjectService) *ImageUpdateWatcher {
return &ImageUpdateWatcher{
imageUpdateService: imageUpdateService,
settingsService: settingsService,
environmentService: environmentService,
dockerService: dockerService,
+ projectService: projectService,
triggerCh: make(chan struct{}, 1),
debounce: imageUpdateWatcherDebounce,
+ backfillRetry: imageUpdateWatcherBackfillRetry,
+ metadataReady: make(chan struct{}),
}
}
@@ -66,17 +80,22 @@ func (w *ImageUpdateWatcher) Name() string {
return "image-polling"
}
-// Start subscribes to Docker image events and runs scans until ctx is canceled.
+// Start subscribes to Docker image events, prepares project metadata, and runs scans until ctx is canceled.
func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
if w == nil || w.dockerService == nil || w.dockerService.EventBus() == nil {
return errors.New("docker event bus unavailable")
}
+ if w.projectService == nil || w.metadataReady == nil {
+ return errors.New("project image metadata backfiller unavailable")
+ }
eventCh, unsubscribe := w.dockerService.EventBus().Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
defer unsubscribe()
var listener sync.WaitGroup
+ listenerStarted := make(chan struct{})
listener.Go(func() {
+ close(listenerStarted)
for {
select {
case <-ctx.Done():
@@ -90,11 +109,25 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
}
})
+ select {
+ case <-ctx.Done():
+ listener.Wait()
+ return nil
+ case <-listenerStarted:
+ }
+
slog.InfoContext(ctx, "image update watcher started")
+ defer func() {
+ listener.Wait()
+ slog.InfoContext(ctx, "image update watcher stopped")
+ }()
+
+ if !w.backfillProjectImageRefsInternal(ctx) {
+ return nil
+ }
+
w.Trigger()
w.runTriggeredScansInternal(ctx)
- listener.Wait()
- slog.InfoContext(ctx, "image update watcher stopped")
return nil
}
@@ -111,9 +144,14 @@ func (w *ImageUpdateWatcher) Trigger() {
// RunNow performs the same full-host image scan used by automatic watcher triggers.
func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
- if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil {
+ if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil || w.metadataReady == nil {
return errors.New("image update watcher is not initialized")
}
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-w.metadataReady:
+ }
run := &imageUpdateScanRunInternal{done: make(chan struct{})}
for {
@@ -177,6 +215,40 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
return nil
}
+func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context) bool {
+ retryDelay := w.backfillRetry
+ if retryDelay <= 0 {
+ retryDelay = imageUpdateWatcherBackfillRetry
+ }
+
+ for attempt := 1; ; attempt++ {
+ if ctx.Err() != nil {
+ return false
+ }
+
+ startedAt := time.Now()
+ count, err := w.projectService.BackfillProjectImageRefs(ctx)
+ duration := time.Since(startedAt)
+ if err == nil {
+ slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", duration, "attempt", attempt)
+ w.metadataReadyOnce.Do(func() { close(w.metadataReady) })
+ return true
+ }
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {
+ return false
+ }
+
+ slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", duration, "attempt", attempt, "retryIn", retryDelay, "error", err)
+ timer := time.NewTimer(retryDelay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return false
+ case <-timer.C:
+ }
+ }
+}
+
func (w *ImageUpdateWatcher) runTriggeredScansInternal(ctx context.Context) {
for {
select {
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index ca5297d4b..1837c3a2f 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -1,8 +1,11 @@
package scheduler
import (
+ "bytes"
"context"
"errors"
+ "log/slog"
+ "strings"
"sync"
"testing"
"time"
@@ -102,17 +105,77 @@ func (p dockerEventBusProviderFakeInternal) EventBus() *bus.DockerEventBus {
return p.eventBus
}
-func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus) *ImageUpdateWatcher {
+type projectImageRefsBackfillerFakeInternal struct {
+ mu sync.Mutex
+ calls int
+ run func(ctx context.Context, call int) (int, error)
+ started chan int
+}
+
+func (b *projectImageRefsBackfillerFakeInternal) BackfillProjectImageRefs(ctx context.Context) (int, error) {
+ b.mu.Lock()
+ b.calls++
+ call := b.calls
+ run := b.run
+ started := b.started
+ b.mu.Unlock()
+
+ if started != nil {
+ select {
+ case started <- call:
+ default:
+ }
+ }
+ if run == nil {
+ return 0, nil
+ }
+ return run(ctx, call)
+}
+
+func (b *projectImageRefsBackfillerFakeInternal) countInternal() int {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.calls
+}
+
+type lockedBufferInternal struct {
+ mu sync.Mutex
+ bytes.Buffer
+}
+
+func (b *lockedBufferInternal) Write(p []byte) (int, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.Buffer.Write(p)
+}
+
+func (b *lockedBufferInternal) stringInternal() string {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.Buffer.String()
+}
+
+func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {
+ if backfiller == nil {
+ backfiller = &projectImageRefsBackfillerFakeInternal{}
+ }
return &ImageUpdateWatcher{
imageUpdateService: scanner,
settingsService: settings,
environmentService: registryCredentialLoaderFakeInternal{},
dockerService: dockerEventBusProviderFakeInternal{eventBus: eventBus},
+ projectService: backfiller,
triggerCh: make(chan struct{}, 1),
debounce: 10 * time.Millisecond,
+ backfillRetry: 10 * time.Millisecond,
+ metadataReady: make(chan struct{}),
}
}
+func markImageUpdateWatcherMetadataReadyForTestInternal(watcher *ImageUpdateWatcher) {
+ watcher.metadataReadyOnce.Do(func() { close(watcher.metadataReady) })
+}
+
func startImageUpdateWatcherForTestInternal(t *testing.T, watcher *ImageUpdateWatcher) (context.CancelFunc, <-chan error) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
@@ -131,7 +194,7 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tes
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -167,7 +230,7 @@ func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testin
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Equal(t, 1, <-scanner.startedCh)
@@ -186,7 +249,7 @@ func TestImageUpdateWatcher_DisabledTriggersAreSkippedUntilEnabled(t *testing.T)
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: false}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
time.Sleep(30 * time.Millisecond)
@@ -205,7 +268,7 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{errors: []error{errors.New("registry unavailable")}}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus)
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -220,7 +283,8 @@ func TestImageUpdateWatcher_RunNowSerializesConcurrentCalls(t *testing.T) {
releaseCh: releaseCh,
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus())
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
+ markImageUpdateWatcherMetadataReadyForTestInternal(watcher)
errCh := make(chan error, 2)
go func() { errCh <- watcher.RunNow(context.Background()) }()
@@ -243,7 +307,8 @@ func TestImageUpdateWatcher_RunNowWaiterHonorsCancellation(t *testing.T) {
releaseCh: releaseCh,
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus())
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
+ markImageUpdateWatcherMetadataReadyForTestInternal(watcher)
firstErrCh := make(chan error, 1)
go func() { firstErrCh <- watcher.RunNow(context.Background()) }()
@@ -257,3 +322,124 @@ func TestImageUpdateWatcher_RunNowWaiterHonorsCancellation(t *testing.T) {
close(releaseCh)
require.NoError(t, <-firstErrCh)
}
+
+func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *testing.T) {
+ const (
+ projectCount = 2500
+ eventCount = 10000
+ )
+
+ var logBuffer lockedBufferInternal
+ previousLogger := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{Level: slog.LevelInfo})))
+ t.Cleanup(func() { slog.SetDefault(previousLogger) })
+
+ backfillStarted := make(chan struct{})
+ releaseBackfill := make(chan struct{})
+ backfiller := &projectImageRefsBackfillerFakeInternal{
+ run: func(ctx context.Context, _ int) (int, error) {
+ close(backfillStarted)
+ select {
+ case <-ctx.Done():
+ return 0, ctx.Err()
+ case <-releaseBackfill:
+ return projectCount, nil
+ }
+ },
+ }
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ eventBus := bus.NewDockerEventBus()
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, backfiller)
+ startImageUpdateWatcherForTestInternal(t, watcher)
+
+ <-backfillStarted
+ burstStartedAt := time.Now()
+ for range eventCount {
+ eventBus.Publish(events.Message{Type: events.ImageEventType, Action: events.ActionPull})
+ }
+ require.Never(t, func() bool { return scanner.countInternal() > 0 }, 30*time.Millisecond, 5*time.Millisecond)
+
+ close(releaseBackfill)
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
+ time.Sleep(30 * time.Millisecond)
+ require.Equal(t, 1, scanner.countInternal())
+
+ logs := logBuffer.stringInternal()
+ require.True(t, strings.Contains(logs, "project image metadata backfill completed"), logs)
+ require.True(t, strings.Contains(logs, "projects=2500"), logs)
+ require.True(t, strings.Contains(logs, "duration="), logs)
+ t.Logf("coalesced %d image events into one scan after backfilling %d projects in %s", eventCount, projectCount, time.Since(burstStartedAt))
+}
+
+func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {
+ secondAttemptStarted := make(chan struct{})
+ releaseSecondAttempt := make(chan struct{})
+ backfiller := &projectImageRefsBackfillerFakeInternal{
+ run: func(ctx context.Context, call int) (int, error) {
+ if call == 1 {
+ return 0, errors.New("database unavailable")
+ }
+ close(secondAttemptStarted)
+ select {
+ case <-ctx.Done():
+ return 0, ctx.Err()
+ case <-releaseSecondAttempt:
+ return 42, nil
+ }
+ },
+ }
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)
+ startImageUpdateWatcherForTestInternal(t, watcher)
+
+ select {
+ case <-secondAttemptStarted:
+ case <-time.After(time.Second):
+ t.Fatal("backfill was not retried")
+ }
+ require.Zero(t, scanner.countInternal())
+ close(releaseSecondAttempt)
+
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
+ require.Equal(t, 2, backfiller.countInternal())
+}
+
+func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.T) {
+ backfillStarted := make(chan struct{})
+ backfiller := &projectImageRefsBackfillerFakeInternal{
+ run: func(ctx context.Context, _ int) (int, error) {
+ close(backfillStarted)
+ <-ctx.Done()
+ return 0, ctx.Err()
+ },
+ }
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)
+ ctx, cancel := context.WithCancel(context.Background())
+ errCh := make(chan error, 1)
+ go func() { errCh <- watcher.Start(ctx) }()
+
+ <-backfillStarted
+ cancel()
+ select {
+ case err := <-errCh:
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("watcher did not stop after cancellation")
+ }
+ require.Zero(t, scanner.countInternal())
+}
+
+func TestImageUpdateWatcher_RunNowWaitsForMetadataReadiness(t *testing.T) {
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ require.ErrorIs(t, watcher.RunNow(ctx), context.DeadlineExceeded)
+ require.Zero(t, scanner.countInternal())
+}