Skip to content

Modern Docker Management, Designed for Everyone

12.4 KB
Raw Blame
package scheduler

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"sync"
	"sync/atomic"
	"time"

	imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"
	schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
	"github.com/moby/moby/api/types/events"
	"go.getarcane.app/streams/bus"

	"github.com/getarcaneapp/arcane/backend/v2/internal/common"
	"github.com/getarcaneapp/arcane/backend/v2/internal/config"
	"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
	"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
	"github.com/getarcaneapp/arcane/backend/v2/internal/project"
	"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)

const (
	imageUpdateWatcherDebounce        = 2 * time.Second
	imageUpdateWatcherBackfillRetry   = 5 * time.Second
	imageUpdateWatcherDefaultSchedule = "0 0 * * * *"
)

type imageUpdateScannerInternal interface {
	RunImageCheck(ctx context.Context, request imageupdatetypes.CheckRequest) (schedulertypes.Outcome, error)
}

type pollingSettingReaderInternal interface {
	GetBoolSetting(ctx context.Context, key string, fallback bool) bool
	GetStringSetting(ctx context.Context, key, defaultValue string) string
}

type dockerEventBusProviderInternal interface {
	EventBus() *bus.DockerEventBus
}

type projectImageRefsBackfillerInternal interface {
	BackfillProjectImageRefs(ctx context.Context) (int, error)
}

// ImageUpdateWatcher debounces Docker events and admits durable image scans.
type ImageUpdateWatcher struct {
	imageUpdateService     imageUpdateScannerInternal
	settingsService        pollingSettingReaderInternal
	dockerService          dockerEventBusProviderInternal
	projectService         projectImageRefsBackfillerInternal
	dispatchMu             sync.RWMutex
	coordinator            *runs.Coordinator
	eventDegraded          atomic.Bool
	running                atomic.Bool
	scanRunning            atomic.Bool
	stopping               atomic.Bool
	triggerGeneration      atomic.Uint64
	acknowledgedGeneration atomic.Uint64
	triggeredAt            atomic.Int64
	trigger                chan struct{}
	scheduleRefresh        chan struct{}
	runMu                  sync.Mutex
	cancel                 context.CancelFunc
	done                   chan struct{}
	location               *time.Location
	debounce               time.Duration
	backfillRetry          time.Duration
	metadataReady          chan struct{}
	metadataReadyOnce      sync.Once
	stopped                chan struct{}
	stoppedOnce            sync.Once
}

func NewImageUpdateWatcher(cfg *config.Config,
	imageUpdateService *imageupdate.ImageUpdateService,
	settingsService *settings.SettingsService,
	dockerService *docker.DockerClientService,
	projectService *project.ProjectService) (*ImageUpdateWatcher,
	error,
) {
	if imageUpdateService == nil || settingsService == nil || dockerService == nil || projectService == nil {
		return nil, errors.New("image update watcher dependencies unavailable")
	}
	location := time.UTC
	if cfg != nil {
		location = cfg.GetLocation()
	}
	return &ImageUpdateWatcher{
			imageUpdateService: imageUpdateService,
			settingsService:    settingsService,
			dockerService:      dockerService,
			projectService:     projectService,
			location:           location,
			debounce:           imageUpdateWatcherDebounce,
			backfillRetry:      imageUpdateWatcherBackfillRetry,
			metadataReady:      make(chan struct{}),
			stopped:            make(chan struct{}),
			trigger: make(chan struct{},
				1),
			scheduleRefresh: make(chan struct{},
				1),
		},
		nil
}

func (w *ImageUpdateWatcher) Name() string { return "image-polling" }

// Start owns the debounce loop and joins its workers before returning.
func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
	if w.stopping.Load() {
		return errors.New("image update watcher cannot be restarted after stop")
	}
	if !w.running.CompareAndSwap(false, true) {
		return errors.New("image update watcher already started")
	}
	defer w.running.Store(false)
	runCtx, cancel := context.WithCancel(ctx)
	done := make(chan struct{})
	w.runMu.Lock()
	if w.stopping.Load() {
		w.runMu.Unlock()
		cancel()
		return errors.New("image update watcher stopped")
	}
	w.cancel = cancel
	w.done = done
	w.runMu.Unlock()
	defer cancel()
	defer close(done)
	return w.runEventLoop(runCtx)
}

func (w *ImageUpdateWatcher) runEventLoop(runCtx context.Context) error {
	eventCh, unsubscribe := w.dockerService.EventBus().Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
	defer unsubscribe()
	timer := time.NewTimer(time.Hour)
	timer.Stop()
	defer timer.Stop()
	var workers sync.WaitGroup
	defer workers.Wait()
	metadata := make(chan struct{}, 1)
	submitted := make(chan error, 1)
	submitting := false
	workers.Go(func() { w.runMetadataBackfillInternal(runCtx, metadata) })
	w.RefreshSchedule()
	ready := false
	for {
		select {
		case <-runCtx.Done():
			return nil
		case <-metadata:
			ready = true
			w.metadataReadyOnce.Do(func() { close(w.metadataReady) })
			w.Trigger()
		case <-w.trigger:
			if ready && !submitting && w.settingsService.GetBoolSetting(runCtx, "pollingEnabled", true) {
				timer.Reset(w.debounce)
			}
		case <-timer.C:
			if !w.scanAdmissionReadyInternal(timer, ready, submitting) {
				continue
			}
			submitting = true
			generation := w.triggerGeneration.Load()
			workers.Go(func() { w.submitTriggeredScanInternal(runCtx, generation, submitted) })
		case err := <-submitted:
			submitting = false
			if err != nil && runCtx.Err() == nil {
				slog.ErrorContext(runCtx, "image scan admission failed", "error", err)
			}
			if w.triggerGeneration.Load() != w.acknowledgedGeneration.Load() {
				timer.Reset(w.debounce)
			}
		case <-w.scheduleRefresh:
			if err := w.refreshSchedule(runCtx); err != nil {
				slog.ErrorContext(runCtx, "Failed to checkpoint image polling schedule", "error", err)
			}
		case _, ok := <-eventCh:
			if !ok {
				eventCh = nil
				w.eventDegraded.Store(true)
				continue
			}
			if w.settingsService.GetBoolSetting(runCtx, "imageEventWatcherEnabled", false) {
				w.Trigger()
			}
		}
	}
}

func (w *ImageUpdateWatcher) runMetadataBackfillInternal(ctx context.Context, metadata chan<- struct{}) {
	for attempt := 1; ctx.Err() == nil; attempt++ {
		var err error
		func() {
			defer utils.RecoverToError(&err, "image metadata backfill")
			err = w.backfillProjectImageRefsInternal(ctx, attempt)
		}()
		if err == nil {
			select {
			case metadata <- struct{}{}:
			case <-ctx.Done():
			}
			return
		}
		delay := w.backfillRetry
		if delay <= 0 {
			delay = imageUpdateWatcherBackfillRetry
		}
		retry := time.NewTimer(delay)
		select {
		case <-retry.C:
		case <-ctx.Done():
			retry.Stop()
			return
		}
	}
}

func (w *ImageUpdateWatcher) scanAdmissionReadyInternal(timer *time.Timer, ready, submitting bool) bool {
	if !ready || submitting || w.triggerGeneration.Load() == w.acknowledgedGeneration.Load() {
		return false
	}
	if remaining := time.Until(time.Unix(0, w.triggeredAt.Load()).Add(w.debounce)); remaining > 0 {
		timer.Reset(remaining)
		return false
	}
	return true
}

func (w *ImageUpdateWatcher) submitTriggeredScanInternal(ctx context.Context, generation uint64, submitted chan<- error) {
	var err error
	func() {
		defer utils.RecoverToError(&err, "image scan admission")
		w.dispatchMu.RLock()
		coordinator := w.coordinator
		w.dispatchMu.RUnlock()
		if coordinator == nil {
			err = errors.New("durable job coordinator unavailable")
			return
		}
		_, err = coordinator.Submit(ctx, schedulertypes.Request{JobID: w.Name(), EnvironmentID: "0", Trigger: "watcher"})
	}()
	if err == nil {
		w.acknowledgedGeneration.Store(generation)
	}
	select {
	case submitted <- err:
	case <-ctx.Done():
	}
}

func (w *ImageUpdateWatcher) Stop(ctx context.Context) error {
	w.stopping.Store(true)
	w.stoppedOnce.Do(func() { close(w.stopped) })
	w.runMu.Lock()
	cancel, done := w.cancel, w.done
	w.runMu.Unlock()
	if cancel == nil {
		return nil
	}
	cancel()
	select {
	case <-done:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (w *ImageUpdateWatcher) Trigger() {
	w.triggeredAt.Store(time.Now().UnixNano())
	w.triggerGeneration.Add(1)
	select {
	case w.trigger <- struct{}{}:
	default:
	}
}

func (w *ImageUpdateWatcher) RefreshSchedule() {
	select {
	case w.scheduleRefresh <- struct{}{}:
	default:
	}
}

func (w *ImageUpdateWatcher) RunNow(ctx context.Context) (err error) {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-w.stopped:
		return errors.New("image update watcher is not running")
	case <-w.metadataReady:
	}
	if !w.running.Load() {
		return errors.New("image update watcher is not running")
	}
	if _, executing := jobcontext.Run(ctx); executing {
		if !w.scanRunning.CompareAndSwap(false, true) {
			return common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress"))
		}
		defer w.scanRunning.Store(false)
		defer utils.RecoverToError(&err, "image update scan worker")
		return w.executeScanInternal(ctx)
	}
	if w.scanRunning.Load() {
		return common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress"))
	}
	w.dispatchMu.RLock()
	coordinator := w.coordinator
	w.dispatchMu.RUnlock()
	if coordinator == nil {
		return errors.New("durable job coordinator unavailable")
	}
	run, err := coordinator.Submit(ctx, schedulertypes.Request{JobID: w.Name(), EnvironmentID: "0", Trigger: "manual"})
	if err != nil {
		return err
	}
	// Back off from 50 ms to 1 s so quick runs return fast without polling long scans hard.
	for delay := 50 * time.Millisecond; ; delay = min(delay*2, time.Second) {
		current, getErr := coordinator.Get(ctx, run.EnvironmentID, run.JobID, run.ID)
		if getErr != nil {
			return getErr
		}
		if current.Status.Terminal() || current.Status == schedulertypes.NeedsAttention {
			if current.Status == schedulertypes.Succeeded {
				return nil
			}
			return &schedulertypes.OutcomeError{Outcome: current.Outcome}
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(delay):
		case <-w.stopped:
			return errors.New("image update watcher stopped")
		}
	}
}

func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
	if err := ctx.Err(); err != nil {
		return err
	}
	if !w.settingsService.GetBoolSetting(ctx, "pollingEnabled", true) {
		slog.DebugContext(ctx, "image update watcher disabled; skipping image scan")
		return nil
	}

	slog.InfoContext(ctx, "image scan run started")
	outcome, err := w.imageUpdateService.RunImageCheck(ctx, imageupdatetypes.CheckRequest{All: true})
	if err != nil {
		return fmt.Errorf("image scan failed: %w", err)
	}
	slog.InfoContext(ctx, "image scan run completed", "status", outcome.Status, "message", outcome.Message)
	if outcome.Status != schedulertypes.Succeeded {
		return &schedulertypes.OutcomeError{Outcome: outcome}
	}
	return nil
}

func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context, attempt int) error {
	started := time.Now()
	count, err := w.projectService.BackfillProjectImageRefs(ctx)
	if err == nil {
		slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", time.Since(started), "attempt", attempt)
	} else if ctx.Err() == nil {
		slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", time.Since(started), "attempt", attempt, "retryIn", w.backfillRetry, "error", err)
	}
	return err
}

func (w *ImageUpdateWatcher) refreshSchedule(ctx context.Context) error {
	w.dispatchMu.RLock()
	coordinator := w.coordinator
	w.dispatchMu.RUnlock()
	if coordinator == nil {
		return errors.New("durable job coordinator unavailable")
	}
	if !w.settingsService.GetBoolSetting(ctx, "pollingEnabled", true) {
		return coordinator.Checkpoint(ctx, w.Name(), "", time.Time{})
	}
	spec := w.settingsService.GetStringSetting(ctx, "pollingInterval", imageUpdateWatcherDefaultSchedule)
	schedule, err := cronScheduleParser.Parse(spec)
	if err != nil {
		slog.WarnContext(ctx, "invalid pollingInterval cron expression; using default schedule", "pollingInterval", spec, "error", err)
		spec = imageUpdateWatcherDefaultSchedule
		schedule, err = cronScheduleParser.Parse(spec)
		if err != nil {
			return err
		}
	}
	return coordinator.Checkpoint(ctx, w.Name(), spec, schedule.Next(time.Now().In(w.location)))
}

func (w *ImageUpdateWatcher) SetCoordinator(coordinator *runs.Coordinator) {
	w.dispatchMu.Lock()
	w.coordinator = coordinator
	w.dispatchMu.Unlock()
}