Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: bound shutdown waits and harden scheduler lifecycle (#4493)
Kyle Mendell10ee66bbc4d2
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex 84e06b20d..2a9b8b33a 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -97,7 +97,7 @@ func Bootstrap(ctx context.Context) error {return fmt.Errorf("setup telemetry: %w", err)}defer func() {- shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)+ shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 4*time.Second)defer cancel()if shutdownErr := providers.Shutdown(shutdownCtx); shutdownErr != nil {slog.ErrorContext(shutdownCtx, "Error shutting down telemetry", "error", shutdownErr)@@ -163,6 +163,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *databaseinitializeStartupState,registerJobs,startEdgeTunnelClient,+ startGlobalVariableSync,),fx.WithLogger(func() fxevent.Logger {logger := &fxevent.SlogLogger{Logger: slog.Default()}@@ -170,7 +171,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *databasereturn logger}),fx.StartTimeout(5*time.Minute),- fx.StopTimeout(8*time.Second),+ fx.StopTimeout(7*time.Second),)}@@ -210,11 +211,15 @@ type initializeStartupStateParams struct {Vulnerability *vulnerability.VulnerabilityService}+const startupTaskTimeout = 30 * time.Second+func initializeStartupState(p initializeStartupStateParams) {appCtx := p.AppCtxcfg := p.Config- startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)+ helperCtx, cancelHelpers := context.WithTimeout(appCtx, startupTaskTimeout)+ startup.CleanupOrphanedVolumeHelpers(helperCtx, p.Volume.CleanupOrphanedVolumeHelpers)+ cancelHelpers()runtimeCfg := &startup.RuntimeConfig{AgentMode: cfg.AgentMode,@@ -264,9 +269,11 @@ func initializeStartupState(p initializeStartupStateParams) {{name: "recover stuck project statuses", run: p.Project.RecoverStuckProjectStatuses},{name: "fail interrupted image patches", run: p.Image.RecoverInterruptedPatches},} {- if err := task.run(appCtx); err != nil {+ taskCtx, cancelTask := context.WithTimeout(appCtx, startupTaskTimeout)+ if err := task.run(taskCtx); err != nil {slog.WarnContext(appCtx, "Startup task failed", "task", task.name, "error", err)}+ cancelTask()}p.Vulnerability.ImportLegacyReportFiles(appCtx)p.Vulnerability.BackfillScanItems(appCtx)@@ -282,10 +289,11 @@ func initializeStartupState(p initializeStartupStateParams) {if err := p.Variable.ImportLegacyLocalEnvFile(appCtx); err != nil {slog.WarnContext(appCtx, "Failed to import legacy global variables", "error", err)}- go p.Variable.SyncAll(appCtx)}- startup.TestDockerConnection(appCtx, func(ctx context.Context) error {+ dockerCtx, cancelDocker := context.WithTimeout(appCtx, startupTaskTimeout)+ defer cancelDocker()+ startup.TestDockerConnection(dockerCtx, func(ctx context.Context) error {_, err := p.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (struct{}, error) {version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})if err != nil {@@ -298,7 +306,7 @@ func initializeStartupState(p initializeStartupStateParams) {})return err})- if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {+ if err := p.Swarm.SyncSwarmEnabledState(dockerCtx); err != nil {slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)}@@ -376,6 +384,16 @@ func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.})}+func startGlobalVariableSync(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, variables *variable.VariableService, _ *http.Server) {+ if cfg.AgentMode {+ return+ }+ lc.Append(fx.Hook{OnStart: func(context.Context) error {+ go variables.SyncAll(appCtx)+ return nil+ }})+}+func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelCauseFunc) {lc.Append(fx.Hook{OnStop: func(context.Context) error {diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex ad820fd05..9b82b07ce 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -341,11 +341,11 @@ func TestHTTPServerStopCancelsStreamingRequestContexts(t *testing.T) {defer func() { _ = resp.Body.Close() }()<-handlerEntered- shutdownCtx, cancelShutdown := context.WithTimeout(t.Context(), 2*time.Second)+ shutdownCtx, cancelShutdown := context.WithTimeout(t.Context(), 6*time.Second)defer cancelShutdown()start := time.Now()require.NoError(t, lifecycle.Stop(shutdownCtx))- require.Less(t, time.Since(start), time.Second)+ require.Less(t, time.Since(start), 4*time.Second)}func TestNewHTTPServerRejectsInvalidTLSCertificate(t *testing.T) {diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 21ea08179..ae5719f5f 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -7,6 +7,7 @@ import ("errors""fmt""log/slog"+ "maps""net/http""slices""sync"@@ -113,14 +114,18 @@ func newJobScheduler(},OnStop: func(ctx context.Context) error {cancelScheduler()- if resumeDone != nil {+ resumeErr := func() error {+ if resumeDone == nil {+ return nil+ }select {case <-resumeDone:+ return nilcase <-ctx.Done():return ctx.Err()}- }- if stopErr := errors.Join(jobService.Coordinator().Stop(ctx), jobScheduler.Stop(ctx)); stopErr != nil {+ }()+ if stopErr := errors.Join(resumeErr, jobService.Coordinator().Stop(ctx), jobScheduler.Stop(ctx)); stopErr != nil {slog.ErrorContext(ctx, "Job scheduler exited with error", "error", stopErr)return stopErr}@@ -464,12 +469,14 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)+ params.FilesystemWatcher.RequestRestart()}})subscribe([]string{"templatesDirectory"}, func(_ []libarcane.SettingUpdate) {if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)+ params.FilesystemWatcher.RequestRestart()}})@@ -498,7 +505,7 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {type timeoutSyncWorker struct {mu sync.Mutex- updates [][]libarcane.SettingUpdate+ updates map[string]stringwake chan struct{}done chan struct{}ctx context.Context@@ -536,15 +543,11 @@ func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscrreturncase <-worker.wake:}- for {- worker.mu.Lock()- if len(worker.updates) == 0 {- worker.mu.Unlock()- break- }- updates := worker.updates[0]- worker.updates = worker.updates[1:]- worker.mu.Unlock()+ worker.mu.Lock()+ updates := worker.updates+ worker.updates = nil+ worker.mu.Unlock()+ if len(updates) > 0 {syncTimeoutSettingsToAgents(ctx, params.Environment, updates)}}@@ -555,7 +558,12 @@ func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscrif worker.closed {return}- worker.updates = append(worker.updates, slices.Clone(updates))+ if worker.updates == nil {+ worker.updates = make(map[string]string, len(updates))+ }+ for _, update := range updates {+ worker.updates[update.Key] = update.Value+ }select {case worker.wake <- struct{}{}:default:@@ -565,7 +573,7 @@ func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscr}// syncTimeoutSettingsToAgents syncs timeout settings to all connected remote environments-func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSettingsEnvironment, timeoutSettings []libarcane.SettingUpdate) {+func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSettingsEnvironment, settingsMap map[string]string) {envs, err := localEnvironment.ListRemoteEnvironments(ctx)if err != nil {slog.WarnContext(ctx, "Failed to list remote environments for timeout sync", "error", err)@@ -576,20 +584,13 @@ func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSereturn}- // Build the settings update payload- settingsMap := make(map[string]string, len(timeoutSettings))- keys := make([]string, 0, len(timeoutSettings))- for _, update := range timeoutSettings {- settingsMap[update.Key] = update.Value- keys = append(keys, update.Key)- }body, err := json.Marshal(settingsMap)if err != nil {slog.WarnContext(ctx, "Failed to marshal timeout settings for sync", "error", err)return}- slog.InfoContext(ctx, "Syncing environment settings to remote environments", "count", len(envs), "keys", keys)+ slog.InfoContext(ctx, "Syncing environment settings to remote environments", "count", len(envs), "keys", slices.Sorted(maps.Keys(settingsMap)))for _, env := range envs {responseBody, statusCode, proxyRequestErr := localEnvironment.ProxyRequest(ctx, env.ID, http.MethodPut, "/api/environments/0/settings", body)diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.goindex ecb12c5f8..88f5f0686 100644--- a/backend/internal/bootstrap/server_bootstrap.go+++ b/backend/internal/bootstrap/server_bootstrap.go@@ -124,16 +124,36 @@ func NewHTTPServer(lc fx.Lifecycle, p HTTPServerParams) (*http.Server, error) {return nil},OnStop: func(ctx context.Context) error {- // Shutdown does not cancel active request contexts, so unblock streaming- // handlers before waiting for the server to drain.- cancelBase()- shutdownErr := srv.Shutdown(ctx)+ shutdownDone := make(chan error, 1)+ go func() { shutdownDone <- srv.Shutdown(ctx) }()+ drainTimer := time.NewTimer(3 * time.Second)+ var shutdownErr error+ select {+ case shutdownErr = <-shutdownDone:+ drainTimer.Stop()+ cancelBase()+ case <-drainTimer.C:+ cancelBase()+ shutdownErr = <-shutdownDone+ }if shutdownErr != nil {slog.ErrorContext(ctx, "Server forced to shutdown", "error", shutdownErr)}if grpcServer != nil {- grpcServer.GracefulStop()+ grpcStopped := make(chan struct{})+ go func() {+ grpcServer.GracefulStop()+ close(grpcStopped)+ }()+ grpcTimer := time.NewTimer(2 * time.Second)+ select {+ case <-grpcStopped:+ grpcTimer.Stop()+ case <-grpcTimer.C:+ grpcServer.Stop()+ <-grpcStopped+ }}if shutdownErr == nil {slog.InfoContext(ctx, "Server stopped gracefully")diff --git a/backend/internal/database/database.go b/backend/internal/database/database.goindex 3564192b8..1514c67ca 100644--- a/backend/internal/database/database.go+++ b/backend/internal/database/database.go@@ -1,6 +1,7 @@package databaseimport (+ "cmp""context""database/sql""errors"@@ -455,6 +456,17 @@ func ParseSQLiteConnectionString(connString string) (string, error) {qs := sqliteutil.NormalizeQuery(connStringUrl.Query())// Foreign-key enforcement is required on every pooled connection.qs.Add("_pragma", "foreign_keys(1)")+ if !sqliteutil.HasPragma(qs, "busy_timeout") {+ qs.Add("_pragma", "busy_timeout(10000)")+ }+ if !slices.Contains([]string{"ro", "memory"}, qs.Get("mode")) && !strings.HasPrefix(cmp.Or(connStringUrl.Opaque, connStringUrl.Path), ":memory:") {+ if !sqliteutil.HasPragma(qs, "journal_mode") {+ qs.Add("_pragma", "journal_mode(WAL)")+ }+ if qs.Get("_txlock") == "" {+ qs.Set("_txlock", "immediate")+ }+ }return (&url.URL{Scheme: connStringUrl.Scheme,Opaque: connStringUrl.Opaque,diff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.goindex bef1fde13..b8ad586c7 100644--- a/backend/internal/database/database_test.go+++ b/backend/internal/database/database_test.go@@ -1464,10 +1464,63 @@ func TestSQLiteDSN(t *testing.T) {require.NoError(t, err)require.Equal(t, "immediate", parsed.Query().Get("_txlock"))require.Equal(t, "shared", parsed.Query().Get("cache"))- require.ElementsMatch(t, []string{"foreign_keys(1)", "synchronous(NORMAL)", "busy_timeout(2500)"}, parsed.Query()["_pragma"])+ require.ElementsMatch(t, []string{"foreign_keys(1)", "synchronous(NORMAL)", "busy_timeout(2500)", "journal_mode(WAL)"}, parsed.Query()["_pragma"])require.Empty(t, parsed.Query().Get("_fk"))require.Empty(t, parsed.Query().Get("_foreign_keys"))require.Empty(t, parsed.Query().Get("_busy_timeout"))})}}++func TestSQLiteDSN_ConcurrencyDefaults(t *testing.T) {+ tests := []struct {+ name string+ input string+ pragmas []string+ txlock string+ }{+ {+ name: "defaults added",+ input: "file:arcane.db",+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)", "journal_mode(WAL)"},+ txlock: "immediate",+ },+ {+ name: "explicit values kept",+ input: "file:arcane.db?_journal_mode=DELETE&_timeout=500&_txlock=deferred",+ pragmas: []string{"foreign_keys(1)", "busy_timeout(500)", "journal_mode(DELETE)"},+ txlock: "deferred",+ },+ {+ name: "explicit pragmas kept",+ input: "file:arcane.db?_pragma=journal_mode(TRUNCATE)&_pragma=busy_timeout(2000)",+ pragmas: []string{"foreign_keys(1)", "busy_timeout(2000)", "journal_mode(TRUNCATE)"},+ txlock: "immediate",+ },+ {+ name: "read-only skips WAL and txlock",+ input: "file:arcane.db?mode=ro",+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)"},+ },+ {+ name: "memory path skips WAL and txlock",+ input: "file::memory:?cache=shared",+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)"},+ },+ {+ name: "memory mode skips WAL and txlock",+ input: "file:arcane?mode=memory&cache=shared",+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)"},+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ dsn, err := ParseSQLiteConnectionString(tt.input)+ require.NoError(t, err)+ parsed, err := url.Parse(dsn)+ require.NoError(t, err)+ require.ElementsMatch(t, tt.pragmas, parsed.Query()["_pragma"])+ require.Equal(t, tt.txlock, parsed.Query().Get("_txlock"))+ })+ }+}diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex 7c0e390d6..c16059591 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -141,7 +141,9 @@ func provideTunnelRegistry(lc fx.Lifecycle) *edge.TunnelRegistry {edge.SetDefaultRegistry(localRegistry)lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {defer edge.ClearDefaultRegistry(localRegistry)- return localRegistry.Stop(ctx)+ stopCtx, cancel := context.WithTimeout(ctx, 2*time.Second)+ defer cancel()+ return localRegistry.Stop(stopCtx)}})return localRegistry}@@ -340,27 +342,7 @@ func provideFilesystemWatcherJob(var stop func(context.Context) errorlc.Append(fx.Hook{OnStart: func(context.Context) error {- stop, err = concurrency.StartSupervised(ctx, "filesystem watcher", func(runCtx context.Context) error {- delay := 5 * time.Second- for runCtx.Err() == nil {- startErr := job.Start(runCtx)- if startErr == nil {- break- }- slog.WarnContext(runCtx, "Filesystem watcher startup failed; retrying", "error", startErr)-- timer := time.NewTimer(delay)- select {- case <-runCtx.Done():- timer.Stop()- return nil- case <-timer.C:- }- delay = min(delay*2, 5*time.Minute)- }- <-runCtx.Done()- return nil- })+ stop, err = concurrency.StartSupervised(ctx, "filesystem watcher", job.Supervise)if err != nil {return err}@@ -414,7 +396,12 @@ func provideVolumeService(if err := service.RegisterWorkflows(engine); err != nil {return nil, err}- lc.Append(fx.Hook{OnStop: func(ctx context.Context) error { service.CleanupHelperContainers(ctx); return nil }})+ lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {+ cleanupCtx, cancel := context.WithTimeout(ctx, 2*time.Second)+ defer cancel()+ service.CleanupHelperContainers(cleanupCtx)+ return nil+ }})return service, nil}diff --git a/backend/internal/webhook/service.go b/backend/internal/webhook/service.goindex d5ece7b55..bb6f4e23c 100644--- a/backend/internal/webhook/service.go+++ b/backend/internal/webhook/service.go@@ -626,6 +626,8 @@ func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) er// DrainActions blocks until all accepted background webhook actions finish or// ctx expires, so shutdown does not silently drop acknowledged work.func (s *WebhookService) DrainActions(ctx context.Context) error {+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()return utils.WaitGroup(ctx, &s.actions)}diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.goindex 286d7f31b..f5005a69a 100644--- a/backend/pkg/fswatch/watcher.go+++ b/backend/pkg/fswatch/watcher.go@@ -8,9 +8,11 @@ import ("context""errors""log/slog"+ "maps""os""path/filepath""runtime"+ "slices""strings""sync""time"@@ -34,6 +36,9 @@ type Watcher struct {cancel context.CancelFuncwatchAliases map[string]stringpendingPaths map[string]struct{}+ readyPaths map[string]struct{}+ changed bool+ wake chan struct{}mu sync.Mutexstarted boolstopped bool@@ -73,6 +78,8 @@ func NewWatcher(watchPath string, opts WatcherOptions) (*Watcher, error) {stoppedCh: make(chan struct{}),watchAliases: make(map[string]string),pendingPaths: make(map[string]struct{}),+ readyPaths: make(map[string]struct{}),+ wake: make(chan struct{}, 1),}, nil}@@ -110,6 +117,7 @@ func (fw *Watcher) Start(ctx context.Context) error {watchCtx, cancel := context.WithCancel(ctx)fw.cancel = cancelgo fw.watchLoopInternal(watchCtx)+ fw.callbacks.Go(func() { fw.notifyLoop(watchCtx) })fw.started = trueslog.InfoContext(ctx, "Filesystem watcher started", "path", fw.watchedPath)@@ -118,13 +126,6 @@ func (fw *Watcher) Start(ctx context.Context) error {// Stop closes the watcher and waits for accepted callbacks.func (fw *Watcher) Stop() error {- err := fw.StopWatching()- fw.callbacks.Wait()- return err-}--// StopWatching closes the watch loop before its owner joins callbacks.-func (fw *Watcher) StopWatching() error {fw.stopOnce.Do(func() {fw.mu.Lock()fw.stopped = true@@ -145,6 +146,8 @@ func (fw *Watcher) StopWatching() error {fw.mu.Unlock()})+ fw.callbacks.Wait()+fw.mu.Lock()defer fw.mu.Unlock()@@ -170,21 +173,21 @@ func (fw *Watcher) watchLoopInternal(ctx context.Context) {returncase event, ok := <-fw.watcher.Events:if fw.processEventInternal(ctx, event, ok, debounceTimer, &debouncePending) {- if !fw.reconnectInternal(ctx) {+ if !fw.reconnect(ctx) {return}}case <-debounceTimer.C:- if fw.fireDebounceInternal(ctx, &debouncePending, &lastGoroutineLog) {+ if fw.fireDebounce(ctx, &debouncePending, &lastGoroutineLog) {continue}case <-healthTicker.C:- if len(fw.watcher.WatchList()) == 0 && !fw.reconnectInternal(ctx) {+ if len(fw.watcher.WatchList()) == 0 && !fw.reconnect(ctx) {return}case err, ok := <-fw.watcher.Errors:if !ok {- if !fw.reconnectInternal(ctx) {+ if !fw.reconnect(ctx) {return}continue@@ -200,13 +203,13 @@ func (fw *Watcher) processEventInternal(ctx context.Context, event fsnotify.Evenreturn true}- // handleEventInternal runs for every event — its job is to keep the+ // handleEvent runs for every event — its job is to keep the// underlying fsnotify subscriptions in sync (attaching newly-created// directories so we can discover compose files that land inside them- // afterwards). Sync-firing is gated separately by shouldHandleEventInternal.- fw.handleEventInternal(ctx, event)+ // afterwards). Sync-firing is gated separately by shouldHandleEvent.+ fw.handleEvent(ctx, event)- if !fw.shouldHandleEventInternal(event) {+ if !fw.shouldHandleEvent(event) {return false}fw.recordPendingPathInternal(event.Name)@@ -215,8 +218,8 @@ func (fw *Watcher) processEventInternal(ctx context.Context, event fsnotify.Evenreturn false}-// fireDebounceInternal runs the debounced onChange callback; returns true if nothing was pending (caller should continue).-func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bool, lastGoroutineLog *time.Time) bool {+// fireDebounce runs the debounced onChange callback; returns true if nothing was pending (caller should continue).+func (fw *Watcher) fireDebounce(ctx context.Context, debouncePending *bool, lastGoroutineLog *time.Time) bool {if !*debouncePending {return true}@@ -227,28 +230,48 @@ func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bo"goroutines", runtime.NumGoroutine())*lastGoroutineLog = time.Now()}- var paths []string- if fw.onChangePaths != nil {- paths = fw.drainPendingPathsInternal()- }- fw.notifyChangesInternal(ctx, paths)+ fw.queueChanges(fw.drainPendingPaths())return false}-func (fw *Watcher) notifyChangesInternal(ctx context.Context, paths []string) {- if fw.onChange != nil {- fw.callbacks.Go(func() {- if ctx.Err() == nil {- fw.onChange(ctx)- }- })+func (fw *Watcher) queueChanges(paths []string) {+ fw.mu.Lock()+ fw.changed = true+ for _, path := range paths {+ fw.readyPaths[path] = struct{}{}}- if fw.onChangePaths != nil && len(paths) > 0 {- fw.callbacks.Go(func() {- if ctx.Err() == nil {- fw.onChangePaths(ctx, paths)- }- })+ fw.mu.Unlock()+ select {+ case fw.wake <- struct{}{}:+ default:+ }+}++func (fw *Watcher) notifyLoop(ctx context.Context) {+ for {+ select {+ case <-ctx.Done():+ return+ case <-fw.wake:+ }+ if ctx.Err() != nil {+ return+ }+ fw.mu.Lock()+ changed := fw.changed+ paths := slices.Collect(maps.Keys(fw.readyPaths))+ fw.changed = false+ clear(fw.readyPaths)+ fw.mu.Unlock()+ if !changed {+ continue+ }+ if fw.onChange != nil {+ fw.onChange(ctx)+ }+ if fw.onChangePaths != nil && len(paths) > 0 {+ fw.onChangePaths(ctx, paths)+ }}}@@ -261,24 +284,29 @@ func (fw *Watcher) recordPendingPathInternal(path string) {fw.mu.Unlock()}-func (fw *Watcher) drainPendingPathsInternal() []string {+func (fw *Watcher) drainPendingPaths() []string {fw.mu.Lock()defer fw.mu.Unlock()- paths := make([]string, 0, len(fw.pendingPaths))- for path := range fw.pendingPaths {- paths = append(paths, path)- delete(fw.pendingPaths, path)- }+ paths := slices.Collect(maps.Keys(fw.pendingPaths))+ clear(fw.pendingPaths)return paths}-// handleEventInternal keeps the underlying fsnotify subscriptions in sync with+// handleEvent keeps the underlying fsnotify subscriptions in sync with// the filesystem: when a new directory is created inside a watched path, it is// attached recursively so that compose files dropped inside it are discovered.// It does NOT decide whether the event should trigger a sync — that is-// shouldHandleEventInternal's job.-func (fw *Watcher) handleEventInternal(ctx context.Context, event fsnotify.Event) {+// shouldHandleEvent's job.+func (fw *Watcher) handleEvent(ctx context.Context, event fsnotify.Event) {+ if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {+ removed := filepath.Clean(event.Name)+ if _, watched := fw.watchAliases[removed]; watched {+ maps.DeleteFunc(fw.watchAliases, func(watchPath, _ string) bool {+ return watchPath == removed || strings.HasPrefix(watchPath, removed+string(os.PathSeparator))+ })+ }+ }if event.Has(fsnotify.Create) && !projects.IsIgnoredProjectDirName(filepath.Base(event.Name)) {logicalName := fw.logicalPathForWatchEventInternal(event.Name)if fw.isWatchableDirectory(event.Name) {@@ -297,25 +325,17 @@ func (fw *Watcher) handleEventInternal(ctx context.Context, event fsnotify.Event"operation", event.Op.String())}-// shouldHandleEventInternal reports whether an fsnotify event should trigger a-// project sync. Only events on known compose / env files qualify; bare-// directory events do not, because build tools (ESPHome, PlatformIO, npm,-// etc.) churn through subdirectories and would otherwise spin the sync loop-// forever. New project directories are still discovered: once the directory-// create event has been processed by handleEventInternal (which attaches a-// watch), the compose file's own Create event reaches this filter and passes-// via IsProjectFile.-func (fw *Watcher) shouldHandleEventInternal(event fsnotify.Event) bool {+func (fw *Watcher) shouldHandleEvent(event fsnotify.Event) bool {if !event.Has(fsnotify.Write) && !event.Has(fsnotify.Create) && !event.Has(fsnotify.Rename) && !event.Has(fsnotify.Remove) && !event.Has(fsnotify.Chmod) {return false}+ if !projects.IsProjectFile(filepath.Base(event.Name)) {+ return false+ }// Arcane's own scratch directories (update backups, GitOps staging) churn// compose files that must not re-sync the live projects.rel, err := filepath.Rel(fw.watchedPath, fw.logicalPathForWatchEventInternal(event.Name))- if err == nil && projects.PathContainsIgnoredDirectory(rel) {- return false- }- return projects.IsProjectFile(filepath.Base(event.Name))+ return err != nil || !projects.PathContainsIgnoredDirectory(rel)}func (fw *Watcher) addExistingDirectories(ctx context.Context, root string) error {@@ -478,8 +498,8 @@ func (fw *Watcher) isWatchableDirectory(path string) bool {return resolvedInfo.IsDir()}-// reconnectInternal replaces a lost filesystem subscription without abandoning its owner.-func (fw *Watcher) reconnectInternal(ctx context.Context) bool {+// reconnect replaces a lost filesystem subscription without abandoning its owner.+func (fw *Watcher) reconnect(ctx context.Context) bool {if err := fw.watcher.Close(); err != nil {slog.DebugContext(ctx, "Closing lost filesystem subscription", "error", err)}@@ -505,7 +525,7 @@ func (fw *Watcher) reconnectInternal(ctx context.Context) bool {slog.WarnContext(ctx, "Some filesystem subscriptions could not be restored", "error", addExistingDirectoriesErr)}// Changes while disconnected need a full reconciliation.- fw.notifyChangesInternal(ctx, []string{fw.watchedPath})+ fw.queueChanges([]string{fw.watchedPath})return true}if closeErr := watcher.Close(); closeErr != nil {diff --git a/backend/pkg/scheduler/event_cleanup_job.go b/backend/pkg/scheduler/event_cleanup_job.goindex c38988a8c..287ee4cf9 100644--- a/backend/pkg/scheduler/event_cleanup_job.go+++ b/backend/pkg/scheduler/event_cleanup_job.go@@ -2,6 +2,8 @@ package schedulerimport ("context"+ "errors"+ "fmt""log/slog""time"@@ -52,26 +54,21 @@ func (j *EventCleanupJob) Schedule(ctx context.Context) string {func (j *EventCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {slog.InfoContext(ctx, "Running event cleanup job", "jobName", EventCleanupJobName)- // Delete events older than 36 hours+ var cleanupErr errorolderThan := 36 * time.Hourif err := j.eventService.DeleteOldEvents(ctx, olderThan); err != nil {- slog.ErrorContext(ctx, "Failed to delete old events", "jobName", EventCleanupJobName, "olderThan", olderThan.String(), "error", err)- return schedulertypes.Outcome{}, err+ cleanupErr = fmt.Errorf("delete old events: %w", err)+ } else {+ slog.InfoContext(ctx, "Event cleanup completed successfully",+ "jobName", EventCleanupJobName,+ "olderThan", olderThan.String())}- slog.InfoContext(ctx, "Event cleanup job completed successfully",- "jobName", EventCleanupJobName,- "olderThan", olderThan.String())-- if j.activityService != nil {- retentionDays := j.settingsService.GetIntSetting(ctx, "activityHistoryRetentionDays", 30)- maxEntries := j.settingsService.GetIntSetting(ctx, "activityHistoryMaxEntries", 1000)- deleted, err := j.activityService.PruneHistory(ctx, retentionDays, maxEntries)- if err != nil {- slog.ErrorContext(ctx, "Failed to prune activity history", "jobName", EventCleanupJobName, "error", err)- return schedulertypes.Outcome{}, err- }-+ retentionDays := j.settingsService.GetIntSetting(ctx, "activityHistoryRetentionDays", 30)+ maxEntries := j.settingsService.GetIntSetting(ctx, "activityHistoryMaxEntries", 1000)+ if deleted, err := j.activityService.PruneHistory(ctx, retentionDays, maxEntries); err != nil {+ cleanupErr = errors.Join(cleanupErr, fmt.Errorf("prune activity history: %w", err))+ } else {slog.InfoContext(ctx, "Activity history cleanup completed successfully","jobName", EventCleanupJobName,"retentionDays", retentionDays,@@ -80,29 +77,29 @@ func (j *EventCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, erro}buildRetentionDays := j.settingsService.GetIntSetting(ctx, "buildHistoryRetentionDays", 30)- deletedBuilds, err := j.buildService.PruneHistory(ctx, buildRetentionDays)- if err != nil {- slog.ErrorContext(ctx, "Failed to prune image build history", "jobName", EventCleanupJobName, "error", err)- return schedulertypes.Outcome{}, err+ if deletedBuilds, err := j.buildService.PruneHistory(ctx, buildRetentionDays); err != nil {+ cleanupErr = errors.Join(cleanupErr, fmt.Errorf("prune image build history: %w", err))+ } else {+ slog.InfoContext(ctx, "Image build history cleanup completed successfully",+ "jobName", EventCleanupJobName,+ "retentionDays", buildRetentionDays,+ "deleted", deletedBuilds)}- slog.InfoContext(ctx, "Image build history cleanup completed successfully",- "jobName", EventCleanupJobName,- "retentionDays", buildRetentionDays,- "deleted", deletedBuilds)-updateRetentionDays := j.settingsService.GetIntSetting(ctx, "activityHistoryRetentionDays", 30)- deletedUpdates, err := j.updaterService.PruneHistory(ctx, updateRetentionDays)- if err != nil {- slog.ErrorContext(ctx, "Failed to prune auto-update history", "jobName", EventCleanupJobName, "error", err)- return schedulertypes.Outcome{}, err+ if deletedUpdates, err := j.updaterService.PruneHistory(ctx, updateRetentionDays); err != nil {+ cleanupErr = errors.Join(cleanupErr, fmt.Errorf("prune auto-update history: %w", err))+ } else {+ slog.InfoContext(ctx, "Auto-update history cleanup completed successfully",+ "jobName", EventCleanupJobName,+ "retentionDays", updateRetentionDays,+ "deleted", deletedUpdates)}- slog.InfoContext(ctx, "Auto-update history cleanup completed successfully",- "jobName", EventCleanupJobName,- "retentionDays", updateRetentionDays,- "deleted", deletedUpdates)-+ if cleanupErr != nil {+ slog.ErrorContext(ctx, "Event cleanup job failed", "jobName", EventCleanupJobName, "error", cleanupErr)+ return schedulertypes.Outcome{}, cleanupErr+ }return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.goindex e7df5399e..73579b3a2 100644--- a/backend/pkg/scheduler/filesystem_watcher_job.go+++ b/backend/pkg/scheduler/filesystem_watcher_job.go@@ -23,9 +23,10 @@ type FilesystemWatcherJob struct {lifecycleCtx context.ContextprojectsWatcher *fswatch.WatchertemplatesWatcher *fswatch.Watcher+ restart chan struct{}+ restartMu sync.Mutexmu sync.Mutexstopped bool- retired []*fswatch.Watcher}func NewFilesystemWatcherJob(@@ -41,6 +42,7 @@ func NewFilesystemWatcherJob(settingsService: settingsService,projectScanDepth: projectScanDepth,lifecycleCtx: ctx,+ restart: make(chan struct{}, 1),}, nil}@@ -48,27 +50,49 @@ func (j *FilesystemWatcherJob) Start(ctx context.Context) error {if err := j.RestartProjectsWatcher(ctx); err != nil {return err}- if err := j.RestartTemplatesWatcher(ctx); err != nil {- j.mu.Lock()- watcher := j.projectsWatcher- j.projectsWatcher = nil- j.mu.Unlock()- if watcher != nil {- return errors.Join(err, watcher.Stop())+ return j.RestartTemplatesWatcher(ctx)+}++func (j *FilesystemWatcherJob) Supervise(ctx context.Context) error {+ delay := 5 * time.Second+ for ctx.Err() == nil {+ startErr := j.Start(ctx)+ if startErr == nil {+ delay = 5 * time.Second+ select {+ case <-ctx.Done():+ return nil+ case <-j.restart:+ continue+ }}- return err+ slog.WarnContext(ctx, "Filesystem watcher startup failed; retrying", "error", startErr)++ timer := time.NewTimer(delay)+ select {+ case <-ctx.Done():+ timer.Stop()+ return nil+ case <-timer.C:+ }+ delay = min(delay*2, 5*time.Minute)}return nil}+func (j *FilesystemWatcherJob) RequestRestart() {+ select {+ case j.restart <- struct{}{}:+ default:+ }+}+func (j *FilesystemWatcherJob) Stop(ctx context.Context) error {j.mu.Lock()j.stopped = trueprojectsWatcher, templatesWatcher := j.projectsWatcher, j.templatesWatcherj.projectsWatcher = nilj.templatesWatcher = nil- retired := j.retired- j.retired = nilj.mu.Unlock()var err errorif projectsWatcher != nil {@@ -77,9 +101,6 @@ func (j *FilesystemWatcherJob) Stop(ctx context.Context) error {if templatesWatcher != nil {err = errors.Join(err, templatesWatcher.Stop())}- for _, watcher := range retired {- err = errors.Join(err, watcher.Stop())- }return err}@@ -120,35 +141,52 @@ func (j *FilesystemWatcherJob) handleTemplatesChangeInternal(ctx context.Contextfunc (j *FilesystemWatcherJob) RestartProjectsWatcher(ctx context.Context) error {slog.InfoContext(ctx, "Restarting projects filesystem watcher")- j.mu.Lock()- defer j.mu.Unlock()- if j.stopped {- return errors.New("filesystem watcher stopped")- }if err := ctx.Err(); err != nil {return err}- if j.projectsWatcher != nil {- previous := j.projectsWatcher- j.projectsWatcher = nil- j.retired = append(j.retired, previous)- // Callbacks need the domain mutex, so stop the watch loop separately from joining them.- if err := previous.StopWatching(); err != nil {- return err- }- }- watcher, err := j.startProjectsWatcherInternal(ctx)+ j.restartMu.Lock()+ defer j.restartMu.Unlock()+ watcher, err := j.startProjectsWatcher(ctx)if err != nil {- if watcher != nil {- _ = watcher.StopWatching()- }return err}- j.projectsWatcher = watcher- return nil+ return j.replaceWatcher(ctx, &j.projectsWatcher, watcher, func(syncCtx context.Context) {+ if j.projectService == nil {+ return+ }+ if syncErr := j.projectService.SyncProjectsFromFileSystem(syncCtx); syncErr != nil {+ slog.ErrorContext(syncCtx, "Initial project sync after watcher start failed", "error", syncErr)+ }+ })+}++func (j *FilesystemWatcherJob) replaceWatcher(ctx context.Context, slot **fswatch.Watcher, watcher *fswatch.Watcher, initialSync func(context.Context)) error {+ j.mu.Lock()+ if j.stopped {+ j.mu.Unlock()+ if watcher == nil {+ return errors.New("filesystem watcher stopped")+ }+ return errors.Join(errors.New("filesystem watcher stopped"), watcher.Stop())+ }+ previous := *slot+ *slot = watcher+ j.mu.Unlock()++ var err error+ if previous != nil {+ err = previous.Stop()+ }++ j.mu.Lock()+ defer j.mu.Unlock()+ if !j.stopped && ctx.Err() == nil {+ initialSync(ctx)+ }+ return err}-func (j *FilesystemWatcherJob) startProjectsWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {+func (j *FilesystemWatcherJob) startProjectsWatcher(ctx context.Context) (*fswatch.Watcher, error) {localSettings, err := j.settingsService.GetSettings(ctx)if err != nil {return nil, err@@ -164,15 +202,10 @@ func (j *FilesystemWatcherJob) startProjectsWatcherInternal(ctx context.Context)return nil, err}if startErr := watcher.Start(j.lifecycleCtx); startErr != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.- return watcher, startErr+ return nil, startErr}slog.InfoContext(ctx, "Projects filesystem watcher started", "path", projectsDirectory)- if j.projectService != nil {- if syncProjectsFromFileSystemErr := j.projectService.SyncProjectsFromFileSystem(ctx); syncProjectsFromFileSystemErr != nil {- slog.ErrorContext(ctx, "Initial project sync after watcher start failed", "error", syncProjectsFromFileSystemErr)- }- }return watcher, nil}@@ -198,35 +231,26 @@ func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlinfunc (j *FilesystemWatcherJob) RestartTemplatesWatcher(ctx context.Context) error {slog.InfoContext(ctx, "Restarting templates filesystem watcher")- j.mu.Lock()- defer j.mu.Unlock()- if j.stopped {- return errors.New("filesystem watcher stopped")- }if err := ctx.Err(); err != nil {return err}- if j.templatesWatcher != nil {- previous := j.templatesWatcher- j.templatesWatcher = nil- j.retired = append(j.retired, previous)- // Callbacks need the domain mutex, so stop the watch loop separately from joining them.- if err := previous.StopWatching(); err != nil {- return err- }- }- watcher, err := j.startTemplatesWatcherInternal(ctx)+ j.restartMu.Lock()+ defer j.restartMu.Unlock()+ watcher, err := j.startTemplatesWatcher(ctx)if err != nil {- if watcher != nil {- _ = watcher.StopWatching()- }return err}- j.templatesWatcher = watcher- return nil+ return j.replaceWatcher(ctx, &j.templatesWatcher, watcher, func(syncCtx context.Context) {+ if j.templateService == nil {+ return+ }+ if syncErr := j.templateService.SyncLocalTemplatesFromFilesystem(syncCtx); syncErr != nil {+ slog.ErrorContext(syncCtx, "Initial template sync failed", "error", syncErr)+ }+ })}-func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {+func (j *FilesystemWatcherJob) startTemplatesWatcher(ctx context.Context) (*fswatch.Watcher, error) {if j.templateService == nil {return nil, nil}@@ -244,10 +268,6 @@ func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Contextreturn nil, err}- if syncLocalTemplatesFromFilesystemErr := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); syncLocalTemplatesFromFilesystemErr != nil {- slog.ErrorContext(ctx, "Initial template sync failed", "error", syncLocalTemplatesFromFilesystemErr)- }-if directoriesOverlapInternal(projectsDirectory, templatesDir) {slog.ErrorContext(ctx,"Templates and projects directories overlap; templates watcher disabled",@@ -265,7 +285,7 @@ func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Contextreturn nil, err}if startErr := watcher.Start(j.lifecycleCtx); startErr != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.- return watcher, startErr+ return nil, startErr}slog.InfoContext(ctx, "Templates filesystem watcher started", "path", templatesDir)diff --git a/backend/pkg/scheduler/filesystem_watcher_job_test.go b/backend/pkg/scheduler/filesystem_watcher_job_test.goindex cadad8e85..280a00519 100644--- a/backend/pkg/scheduler/filesystem_watcher_job_test.go+++ b/backend/pkg/scheduler/filesystem_watcher_job_test.go@@ -8,6 +8,8 @@ import ("github.com/stretchr/testify/assert""github.com/stretchr/testify/require"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/fswatch")func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *testing.T) {@@ -57,3 +59,33 @@ func TestFilesystemWatcherJob_ConcurrentProjectRestartsAreSerializedInternal(t *defer cancel()require.NoError(t, job.Stop(stopCtx))}++func TestFilesystemWatcherJob_RequestRestartRearmsSupervisor(t *testing.T) {+ _, settingsService, _ := setupAnalyticsStateServicesInternal(t)+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "projectsDirectory", t.TempDir()))+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "templatesDirectory", t.TempDir()))++ job, err := NewFilesystemWatcherJob(t.Context(), nil, nil, settingsService, 2)+ require.NoError(t, err)++ superviseCtx, cancelSupervise := context.WithCancel(t.Context())+ done := make(chan error, 1)+ go func() { done <- job.Supervise(superviseCtx) }()++ currentWatcher := func() *fswatch.Watcher {+ job.mu.Lock()+ defer job.mu.Unlock()+ return job.projectsWatcher+ }+ require.Eventually(t, func() bool { return currentWatcher() != nil }, 5*time.Second, 10*time.Millisecond)+ firstWatcher := currentWatcher()++ job.RequestRestart()+ require.Eventually(t, func() bool { return currentWatcher() != firstWatcher }, 5*time.Second, 10*time.Millisecond)++ cancelSupervise()+ require.NoError(t, <-done)+ stopCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ require.NoError(t, job.Stop(stopCtx))+}diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.goindex 6a34989e9..7fa9fe37a 100644--- a/backend/pkg/scheduler/image_update_watcher.go+++ b/backend/pkg/scheduler/image_update_watcher.go@@ -73,8 +73,6 @@ type ImageUpdateWatcher struct {backfillRetry time.DurationmetadataReady chan struct{}metadataReadyOnce sync.Once- started chan struct{}- startedOnce sync.Oncestopped chan struct{}stoppedOnce sync.Once}@@ -102,7 +100,6 @@ func NewImageUpdateWatcher(cfg *config.Config,debounce: imageUpdateWatcherDebounce,backfillRetry: imageUpdateWatcherBackfillRetry,metadataReady: make(chan struct{}),- started: make(chan struct{}),stopped: make(chan struct{}),trigger: make(chan struct{},1),@@ -136,18 +133,12 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {w.runMu.Unlock()defer cancel()defer close(done)- return w.runEventLoopInternal(runCtx)+ return w.runEventLoop(runCtx)}-func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error {- eventBus := w.dockerService.EventBus()- if eventBus == nil {- return errors.New("docker event bus unavailable")- }- eventCh, unsubscribe := eventBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))+func (w *ImageUpdateWatcher) runEventLoop(runCtx context.Context) error {+ eventCh, unsubscribe := w.dockerService.EventBus().Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))defer unsubscribe()- reconnect := time.NewTicker(5 * time.Second)- defer reconnect.Stop()timer := time.NewTimer(time.Hour)timer.Stop()defer timer.Stop()@@ -157,7 +148,6 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) errorsubmitted := make(chan error, 1)submitting := falseworkers.Go(func() { w.runMetadataBackfillInternal(runCtx, metadata) })- w.startedOnce.Do(func() { close(w.started) })w.RefreshSchedule()ready := falsefor {@@ -169,7 +159,7 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) errorw.metadataReadyOnce.Do(func() { close(w.metadataReady) })w.Trigger()case <-w.trigger:- if ready && !submitting {+ if ready && !submitting && w.settingsService.GetBoolSetting(runCtx, "pollingEnabled", true) {timer.Reset(w.debounce)}case <-timer.C:@@ -188,11 +178,9 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) errortimer.Reset(w.debounce)}case <-w.scheduleRefresh:- if err := w.refreshScheduleInternal(runCtx); err != nil {+ if err := w.refreshSchedule(runCtx); err != nil {slog.ErrorContext(runCtx, "Failed to checkpoint image polling schedule", "error", err)}- case <-reconnect.C:- eventCh, unsubscribe = w.reconnectEventsInternal(eventCh, unsubscribe)case _, ok := <-eventCh:if !ok {eventCh = nil@@ -206,20 +194,6 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error}}-func (w *ImageUpdateWatcher) reconnectEventsInternal(eventCh <-chan events.Message, unsubscribe func()) (<-chan events.Message, func()) {- if eventCh != nil {- return eventCh, unsubscribe- }- currentBus := w.dockerService.EventBus()- if currentBus == nil {- return eventCh, unsubscribe- }- unsubscribe()- eventCh, unsubscribe = currentBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))- w.eventDegraded.Store(false)- return eventCh, unsubscribe-}-func (w *ImageUpdateWatcher) runMetadataBackfillInternal(ctx context.Context, metadata chan<- struct{}) {for attempt := 1; ctx.Err() == nil; attempt++ {var err error@@ -316,13 +290,6 @@ func (w *ImageUpdateWatcher) RefreshSchedule() {}func (w *ImageUpdateWatcher) RunNow(ctx context.Context) (err error) {- select {- case <-ctx.Done():- return ctx.Err()- case <-w.stopped:- return errors.New("image update watcher is not running")- case <-w.started:- }select {case <-ctx.Done():return ctx.Err()@@ -408,7 +375,16 @@ func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Contexreturn err}-func (w *ImageUpdateWatcher) refreshScheduleInternal(ctx context.Context) error {+func (w *ImageUpdateWatcher) refreshSchedule(ctx context.Context) error {+ w.dispatchMu.RLock()+ coordinator := w.coordinator+ w.dispatchMu.RUnlock()+ if coordinator == nil {+ return errors.New("durable job coordinator unavailable")+ }+ if !w.settingsService.GetBoolSetting(ctx, "pollingEnabled", true) {+ return coordinator.Checkpoint(ctx, w.Name(), "", time.Time{})+ }spec := w.settingsService.GetStringSetting(ctx, "pollingInterval", imageUpdateWatcherDefaultSchedule)schedule, err := cronScheduleParser.Parse(spec)if err != nil {@@ -419,12 +395,6 @@ func (w *ImageUpdateWatcher) refreshScheduleInternal(ctx context.Context) errorreturn err}}- w.dispatchMu.RLock()- coordinator := w.coordinator- w.dispatchMu.RUnlock()- if coordinator == nil {- return errors.New("durable job coordinator unavailable")- }return coordinator.Checkpoint(ctx, w.Name(), spec, schedule.Next(time.Now().In(w.location)))}diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex 2b27b58f4..5fb7fd1c7 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -177,7 +177,7 @@ func (b *lockedBufferInternal) stringInternal() string {return b.String()}-func newImageUpdateWatcherForTestInternal(t *testing.T,+func newImageUpdateWatcherForTest(t *testing.T,scanner imageUpdateScannerInternal,settings pollingSettingReaderInternal,eventBus *bus.DockerEventBus,@@ -198,7 +198,6 @@ func newImageUpdateWatcherForTestInternal(t *testing.T,debounce: 10 * time.Millisecond,backfillRetry: 10 * time.Millisecond,metadataReady: make(chan struct{}),- started: make(chan struct{}),stopped: make(chan struct{}),}coordinator, _ := newTestCoordinatorInternal(t, t.Context(), time.UTC, func(ctx context.Context, run schedulertypes.Run) (schedulertypes.Outcome, error) {@@ -229,7 +228,7 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tesscanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -261,7 +260,7 @@ func TestImageUpdateWatcher_EventTriggersAreOptIn(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -294,7 +293,7 @@ func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testin}settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Equal(t, 1, <-scanner.startedCh)@@ -313,7 +312,7 @@ func TestImageUpdateWatcher_DisabledTriggersAreSkippedUntilEnabled(t *testing.T)scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: false}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)time.Sleep(30 * time.Millisecond)@@ -332,7 +331,7 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{errors: []error{errors.New("registry unavailable")}}settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -347,7 +346,7 @@ func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *testreleaseCh: releaseCh,}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Equal(t, 1, <-scanner.startedCh)@@ -365,7 +364,7 @@ func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *testfunc TestImageUpdateWatcher_RunNowReturnsCompletedResultInternal(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)@@ -380,7 +379,7 @@ func TestImageUpdateWatcher_RunNowReturnsCompletedResultInternal(t *testing.T) {func TestImageUpdateWatcher_RunNowReturnsContainedScanPanicInternal(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{panics: []bool{false, true}}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)startImageUpdateWatcherForTestInternal(t, watcher)// Wait for durable startup admission and the startup scan itself to finish,// not just for the scanner to start; RunNow rejects a scan still running.@@ -402,7 +401,7 @@ func TestImageUpdateWatcher_TriggeredScanRetriesAfterManualScanFinishes(t *testi}settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)// Let the startup scan finish so the gate is free.@@ -462,7 +461,7 @@ func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *testscanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, backfiller)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, backfiller)startImageUpdateWatcherForTestInternal(t, watcher)<-backfillStarted@@ -495,7 +494,7 @@ func TestImageUpdateWatcher_BackfillRetriesContainedPanicInternal(t *testing.T)}scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), backfiller)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return backfiller.countInternal() == 2 }, time.Second, time.Millisecond)@@ -521,7 +520,7 @@ func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {}scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), backfiller)startImageUpdateWatcherForTestInternal(t, watcher)select {@@ -547,7 +546,7 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.}scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), backfiller)ctx, cancel := context.WithCancel(t.Context())errCh := make(chan error, 1)go func() { errCh <- watcher.Start(ctx) }()@@ -566,7 +565,7 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true, schedule: "* * * * * *"}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)jobScheduler, err := NewJobScheduler(t.Context(), watcher.coordinator, time.UTC)require.NoError(t, err)t.Cleanup(func() { require.NoError(t, jobScheduler.Stop(context.WithoutCancel(t.Context()))) })@@ -601,7 +600,7 @@ func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsWatcherRunning(t *testinscanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -614,7 +613,7 @@ func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsWatcherRunning(t *testinfunc TestImageUpdateWatcher_SupervisorRestartsAfterCancellationInternal(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, &pollingSettingReaderFakeInternal{enabled: true}, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, &pollingSettingReaderFakeInternal{enabled: true}, bus.NewDockerEventBus(), nil)supervisor := newWatcherSupervisorInternal(watcher)ctx, cancel := context.WithCancel(t.Context())defer cancel()@@ -633,7 +632,7 @@ func TestImageUpdateWatcher_SupervisorRestartsAfterCancellationInternal(t *testifunc TestImageUpdateWatcher_RunNowWaitsForMetadataReadiness(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)defer cancel()diff --git a/backend/pkg/scheduler/runs/activity.go b/backend/pkg/scheduler/runs/activity.goindex 3bdd04d89..0dbeac6c1 100644--- a/backend/pkg/scheduler/runs/activity.go+++ b/backend/pkg/scheduler/runs/activity.go@@ -6,7 +6,6 @@ import ("time""github.com/getarcaneapp/arcane/types/v2/scheduler"- "github.com/italypaleale/francis/actor"kit "go.getarcane.app/kit/pkg")@@ -22,27 +21,25 @@ func (q *Coordinator) associateActivity(run *scheduler.Run) {}}-func (q *Coordinator) observeRun(ctx context.Context, run scheduler.Run) {+func (q *Coordinator) observeRun(ctx context.Context, run scheduler.Run) bool {if q.observer == nil {- return+ return false}- q.observerMu.Lock()- defer q.observerMu.Unlock()- q.syncActivity(ctx, run)- q.signal()+ defer q.observers.Lock(run.ID)()+ return q.syncActivity(ctx, run)}-func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) {+func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) bool {ctx, cancel := context.WithTimeout(ctx, 5*time.Second)defer cancel()var state scheduler.CoordinatorStateid := kit.SHA256Hex(run.EnvironmentID + "\x00" + run.JobID)if err := q.service.GetState(ctx, coordinatorType, id, &state); err != nil {- return+ return true}version, pending := state.PendingActivitySync[run.ID]if !pending {- return+ return false}current, err := q.Get(ctx, run.EnvironmentID, run.JobID, run.ID)if err == nil {@@ -53,60 +50,20 @@ func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) {}if err != nil {slog.ErrorContext(ctx, "Job activity synchronization failed; will retry", "runId", run.ID, "error", err)+ return true}+ return false}-func (q *Coordinator) retryActivitySync(ctx context.Context) {- if q.observer == nil {- return- }- cursor := ""- for {- page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})- if err != nil {- slog.ErrorContext(ctx, "List pending job activities", "error", err)- return- }- for _, entry := range page.States {- var state scheduler.CoordinatorState- if decodeErr := entry.Data.Decode(&state); decodeErr != nil {- slog.ErrorContext(ctx, "Decode pending job activities", "error", decodeErr)- continue- }- for runID := range state.PendingActivitySync {- if ctx.Err() != nil {- return- }- q.observerMu.Lock()- q.syncActivity(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})- q.observerMu.Unlock()- }- }- cursor = page.AfterID()- if cursor == "" {- return- }- }-}--func (q *Coordinator) hasPendingActivitySync(ctx context.Context) bool {- ctx, cancel := context.WithTimeout(ctx, 5*time.Second)- defer cancel()- cursor := ""- for {- page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})- if err != nil {- return true- }- for _, entry := range page.States {- var state scheduler.CoordinatorState- if entry.Data.Decode(&state) != nil || len(state.PendingActivitySync) > 0 {+func (q *Coordinator) retryActivitySync(ctx context.Context, states []scheduler.CoordinatorState) bool {+ pending := false+ for _, state := range states {+ for runID := range state.PendingActivitySync {+ if ctx.Err() != nil {return true}- }- cursor = page.AfterID()- if cursor == "" {- return false+ pending = q.observeRun(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID}) || pending}}+ return pending}diff --git a/backend/pkg/scheduler/runs/activity_test.go b/backend/pkg/scheduler/runs/activity_test.goindex 8113b4c1c..5b9d948cc 100644--- a/backend/pkg/scheduler/runs/activity_test.go+++ b/backend/pkg/scheduler/runs/activity_test.go@@ -3,6 +3,7 @@ package runsimport ("context""errors"+ "slices""sync/atomic""testing""time"@@ -32,6 +33,18 @@ func (o *activityObserverFake) SyncRunActivity(context.Context, st.Run) error {return nil}+func coordinatorStates(t *testing.T, q *Coordinator) []st.CoordinatorState {+ t.Helper()+ states, err := q.states(t.Context())+ require.NoError(t, err)+ return states+}++func pendingActivitySync(t *testing.T, q *Coordinator) bool {+ t.Helper()+ return slices.ContainsFunc(coordinatorStates(t, q), func(state st.CoordinatorState) bool { return len(state.PendingActivitySync) > 0 })+}+func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {q := newResolutionQueue(t)observer := &activityObserverFake{}@@ -56,10 +69,10 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {persisted, getErr := q.Get(t.Context(), "0", run.JobID, run.ID)return getErr == nil && persisted.Status == st.Succeeded}, 5*time.Second, 10*time.Millisecond)- require.True(t, q.hasPendingActivitySync(t.Context()))+ require.True(t, pendingActivitySync(t, q))observer.unavailable.Store(false)- q.retryActivitySync(t.Context())- require.False(t, q.hasPendingActivitySync(t.Context()))+ require.False(t, q.retryActivitySync(t.Context(), coordinatorStates(t, q)))+ require.False(t, pendingActivitySync(t, q))require.Positive(t, observer.observed.Load())duplicate, err := q.Submit(t.Context(), st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "manual"})require.NoError(t, err)diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.goindex 0f1560899..bb6d2333a 100644--- a/backend/pkg/scheduler/runs/actors.go+++ b/backend/pkg/scheduler/runs/actors.go@@ -102,6 +102,14 @@ func (a *coordinatorActor) Invoke(ctx context.Context, method string, data actorreturn nil, a.q.service.SetState(ctx, coordinatorType, a.id, state, nil)case "repair":return nil, a.flush(ctx, &state)+ case "remove":+ if state.Record.Schedule != "" || len(state.PendingActivitySync) > 0 || len(state.Record.Runs) > 0 || len(state.Record.Receipts) > 0 {+ return false, nil+ }+ if err := a.q.service.DeleteState(ctx, coordinatorType, a.id); err != nil && !errors.Is(err, actor.ErrStateNotFound) {+ return nil, err+ }+ return true, nilcase "import":var command st.CoordinatorImportif err := data.Decode(&command); err != nil {@@ -352,23 +360,19 @@ func (a *coordinatorActor) Alarm(ctx context.Context, name string, data actor.En// repair flushes coordinators so schedule alarms and dispatches are restored.// Unless all is set, it skips records without unfinished runs so idle coordinators can deactivate.-func (q *Coordinator) repair(ctx context.Context, all bool) (busy bool, err error) {- records, err := q.Records(ctx)- if err != nil {- return false, err- }- unfinished := func(record st.QueueRecord) bool {- return slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention })+func (q *Coordinator) repair(ctx context.Context, states []st.CoordinatorState, all bool) (busy bool, err error) {+ unfinished := func(state st.CoordinatorState) bool {+ return slices.ContainsFunc(state.Record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention })}- for _, record := range records {- if !all && !unfinished(record) {+ for _, state := range states {+ if !all && !unfinished(state) {continue}- if _, invokeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {+ if _, invokeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(state.Record.EnvironmentID+"\x00"+state.Record.JobID), "repair", nil); invokeErr != nil {return false, invokeErr}}- return slices.ContainsFunc(records, unfinished), nil+ return slices.ContainsFunc(states, unfinished), nil}func (q *Coordinator) importLegacy(ctx context.Context) error {diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.goindex 978b6d7f3..4009abb70 100644--- a/backend/pkg/scheduler/runs/actors_test.go+++ b/backend/pkg/scheduler/runs/actors_test.go@@ -91,7 +91,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {run.Outcome.Targets = []st.TargetOutcome{{ID: "artifact", Status: st.Succeeded, RecoveryData: jsontext.Value(`{"backupId":"artifact"}`)}}return nil}))- require.True(t, q.hasPendingActivitySync(t.Context()))+ require.True(t, pendingActivitySync(t, q))q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {t.Error("startup must not execute jobs before activation")return st.Outcome{Status: st.Failed}, nil@@ -128,8 +128,8 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {require.Equal(t, 1, history.Total)require.Equal(t, time.UTC, history.Runs[0].CreatedAt.Location())observer.unavailable.Store(false)- restored.retryActivitySync(t.Context())- require.False(t, restored.hasPendingActivitySync(t.Context()))+ require.False(t, restored.retryActivitySync(t.Context(), coordinatorStates(t, restored)))+ require.False(t, pendingActivitySync(t, restored))require.Positive(t, observer.observed.Load())_, exists, err := store.Get(t.Context(), key)require.NoError(t, err)@@ -268,10 +268,10 @@ func TestReconciliationRepairsMissingAlarm(t *testing.T) {record, err := q.ScheduleState(t.Context(), "repair")require.NoError(t, err)id := kit.SHA256Hex("0\x00repair")- _, err = q.repair(t.Context(), true)+ _, err = q.repair(t.Context(), coordinatorStates(t, q), true)require.NoError(t, err)require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))- _, err = q.repair(t.Context(), true)+ _, err = q.repair(t.Context(), coordinatorStates(t, q), true)require.NoError(t, err)require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))after, err := q.ScheduleState(t.Context(), "repair")@@ -358,7 +358,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {state.Dispatches[key] = intentstate.Revision++require.NoError(t, q.service.SetState(t.Context(), coordinatorType, id, state, nil))- _, err = q.repair(t.Context(), false)+ _, err = q.repair(t.Context(), coordinatorStates(t, q), false)require.NoError(t, err)require.NoError(t, q.service.GetState(t.Context(), coordinatorType, id, &state))require.Equal(t, original, state.Dispatches[key].JobID)diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.goindex d86dc9e7d..dd0b0710e 100644--- a/backend/pkg/scheduler/runs/coordinator.go+++ b/backend/pkg/scheduler/runs/coordinator.go@@ -44,7 +44,7 @@ type Coordinator struct {execute func(context.Context, st.Run) (st.Outcome, error)reconcile func(context.Context, st.Run) (st.Outcome, error)observer st.RunObserver- observerMu sync.Mutex+ observers utils.KeyedMutexlocation *time.Location}@@ -54,8 +54,9 @@ type Coordinator struct {// Cancellation of ctx affects admission only, not an accepted run's lifetime.func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, error) {q.mu.Lock()- defer q.mu.Unlock()- if q.stopping {+ stopping := q.stopping+ q.mu.Unlock()+ if stopping {return st.Run{}, errors.New("job queue is stopping")}request.EnvironmentID = cmp.Or(request.EnvironmentID, "0")@@ -110,7 +111,9 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, eRemoteOutcome: remoteOutcome,}q.associateActivity(&result)+ fresh := resulterr := q.mutate(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {+ result = freshif request.Trigger == "scheduled" || request.Trigger == "recovery" {record.LastEnqueuedAt = now}@@ -157,8 +160,8 @@ func separateTrigger(trigger string) bool {// Checkpoint records the next due occurrence atomically with overdue recovery.func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {q.mu.Lock()- defer q.mu.Unlock()first := !q.checkpointed[jobID]+ q.mu.Unlock()recovered := falsevar recoveredRun st.Runerr := q.mutate(ctx, "0", jobID, func(record *st.QueueRecord) error {@@ -190,7 +193,9 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, nereturn nil})if err == nil {+ q.mu.Lock()q.checkpointed[jobID] = true+ q.mu.Unlock()if recovered {q.observeRun(ctx, recoveredRun)q.signal()@@ -199,6 +204,17 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, nereturn err}+func (q *Coordinator) Remove(ctx context.Context, jobID string) error {+ if err := q.Checkpoint(ctx, jobID, "", time.Time{}); err != nil {+ return err+ }+ q.mu.Lock()+ delete(q.checkpointed, jobID)+ q.mu.Unlock()+ _, err := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex("0\x00"+jobID), "remove", nil)+ return err+}+// Start validates persisted state on ctx and starts dispatching on appCtx. Repeated calls// are harmless, but a stopped queue cannot be restarted; create a new instance.func (q *Coordinator) Start(ctx, appCtx context.Context) error {@@ -220,17 +236,15 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {return fmt.Errorf("import legacy job history: %w", err)}// Fail startup if persisted state cannot be read. Never replace it with empty state.- records, err := q.Records(ctx)+ states, err := q.states(ctx)if err != nil {return fmt.Errorf("read startup job history: %w", err)}- for _, record := range records {+ for index := range states {+ record := &states[index].Recordchanged := false- for _, run := range record.Runs {- if normalizeLegacyRun(&run) {- changed = true- break- }+ for i := range record.Runs {+ changed = normalizeLegacyRun(&record.Runs[i]) || changed}if !changed {continue@@ -244,7 +258,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {return fmt.Errorf("normalize startup job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)}}- if pruneErr := q.prune(ctx); pruneErr != nil {+ if pruneErr := q.prune(ctx, states); pruneErr != nil {return fmt.Errorf("prune startup job history: %w", pruneErr)}if errErr := ctx.Err(); errErr != nil {@@ -287,25 +301,16 @@ func (q *Coordinator) Stop(ctx context.Context) error {func (q *Coordinator) dispatch(ctx context.Context) {defer close(q.done)- initialRetention := true// Flush every coordinator once while enabled to restore schedule alarms; later passes only cover unfinished work.fullRepair := truenextRetention := time.Now()for ctx.Err() == nil {- q.retryActivitySync(ctx)- if initialRetention || !time.Now().Before(nextRetention) {- if err := q.prune(ctx); err != nil {- if ctx.Err() == nil {- slog.ErrorContext(ctx, "Job retention failed", "error", err)- }- } else {- initialRetention = q.hasPendingActivitySync(ctx)- }- nextRetention = time.Now().Add(time.Hour)- }enabled := q.enabled.Load()- busy, err := q.repair(ctx, fullRepair)- pendingActivity := q.hasPendingActivitySync(ctx)+ retain := !time.Now().Before(nextRetention)+ busy, pendingActivity, pruned, err := q.pass(ctx, fullRepair, retain)+ if retain {+ nextRetention = time.Now().Add(kit.Ternary(pruned, time.Hour, 5*time.Second))+ }nextWake := time.Now().Add(kit.Ternary(busy || pendingActivity, 15*time.Second, coordinatorIdleWake))if err != nil {if ctx.Err() == nil {@@ -336,6 +341,23 @@ func (q *Coordinator) dispatch(ctx context.Context) {}}+func (q *Coordinator) pass(ctx context.Context, all, retain bool) (busy, pendingActivity, pruned bool, err error) {+ states, err := q.states(ctx)+ if err != nil {+ return false, true, false, err+ }+ pendingActivity = q.retryActivitySync(ctx, states)+ if retain {+ pruneErr := q.prune(ctx, states)+ if pruneErr != nil && ctx.Err() == nil {+ slog.ErrorContext(ctx, "Job retention failed", "error", pruneErr)+ }+ pruned = pruneErr == nil+ }+ busy, err = q.repair(ctx, states, all)+ return busy, pendingActivity, pruned, err+}+func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) error {// Keep the execution slot while retrying, but release it after a bounded backoff.wait := time.Second@@ -348,18 +370,21 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome streturn ErrRunConflict}now := time.Now().UTC()- current.Status = outcome.Status- current.UpdatedAt = now+ merged := st.Outcome{+ Status: outcome.Status,+ Message: outcome.Message,+ ActivityID: cmp.Or(outcome.ActivityID, current.Outcome.ActivityID),+ Targets: slices.Clone(outcome.Targets),+ Steps: kit.Ternary(len(outcome.Steps) == 0, current.Outcome.Steps, outcome.Steps),+ }for _, target := range current.Outcome.Targets {if !slices.ContainsFunc(outcome.Targets, func(updated st.TargetOutcome) bool { return updated.ID == target.ID }) {- outcome.Targets = append(outcome.Targets, target)+ merged.Targets = append(merged.Targets, target)}}- outcome.ActivityID = cmp.Or(outcome.ActivityID, current.Outcome.ActivityID)- if len(outcome.Steps) == 0 {- outcome.Steps = current.Outcome.Steps- }- current.Outcome = outcome+ current.Status = outcome.Status+ current.UpdatedAt = now+ current.Outcome = mergedif outcome.Status == st.Retrying || outcome.Status == st.Waiting {delay := 5 * time.Second * time.Duration(1<<min(run.AttemptCount-1, 6))delay = min(delay, 5*time.Minute)@@ -381,7 +406,7 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome stif len(current.Attempts) > 0 {last := ¤t.Attempts[len(current.Attempts)-1]last.FinishedAt = &now- last.Outcome = outcome+ last.Outcome = mergedlast.Outcome.Steps = nil}return nildiff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.goindex 6cbe0b069..a6c759b9b 100644--- a/backend/pkg/scheduler/runs/coordinator_store.go+++ b/backend/pkg/scheduler/runs/coordinator_store.go@@ -6,6 +6,8 @@ import ("encoding/json/v2""errors""fmt"+ "maps"+ "math/rand/v2""slices""sort""time"@@ -16,6 +18,7 @@ import ("github.com/italypaleale/francis/actor""go.getarcane.app/kit/pkg"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/kv")@@ -24,7 +27,17 @@ const queuePrefix = "jobs."func (q *Coordinator) mutate(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {id := kit.SHA256Hex(environmentID + "\x00" + jobID)q.runtime.NameActor(id, jobID+"@"+environmentID)- for range 20 {+ for attempt := range 20 {+ if attempt > 0 {+ delay := 5 * time.Millisecond << min(attempt-1, 5)+ timer := time.NewTimer(delay/2 + rand.N(delay/2))+ select {+ case <-ctx.Done():+ timer.Stop()+ return ctx.Err()+ case <-timer.C:+ }+ }var state st.CoordinatorStateerr := q.service.GetState(ctx, coordinatorType, id, &state)if err != nil && !errors.Is(err, actor.ErrStateNotFound) {@@ -45,7 +58,7 @@ func (q *Coordinator) mutate(ctx context.Context, environmentID, jobID string, creturn nil}}- return errors.New("job coordinator state is busy; retry mutation")+ return common.Classify(common.ErrUnavailable, errors.New("job coordinator state is busy; retry mutation"))}func (q *Coordinator) prepareMutation(state *st.CoordinatorState, environmentID, jobID string, change func(*st.QueueRecord) error) error {@@ -147,7 +160,19 @@ func pruneDispatches(state *st.CoordinatorState) {}func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {- records := []st.QueueRecord{}+ states, err := q.states(ctx)+ if err != nil {+ return nil, err+ }+ records := make([]st.QueueRecord, 0, len(states))+ for _, state := range states {+ records = append(records, state.Record)+ }+ return records, nil+}++func (q *Coordinator) states(ctx context.Context) ([]st.CoordinatorState, error) {+ states := []st.CoordinatorState{}cursor := ""for {page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})@@ -161,11 +186,11 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {}normalizeRecordTimes(&state.Record)q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)- records = append(records, state.Record)+ states = append(states, state)}cursor = page.AfterID()if cursor == "" {- return records, nil+ return states, nil}}}@@ -256,21 +281,23 @@ func (q *Coordinator) UpdateRun(ctx context.Context, run st.Run, change func(*streturn err}-func (q *Coordinator) prune(ctx context.Context) error {- records, err := q.Records(ctx)- if err != nil {- return err- }+func (q *Coordinator) prune(ctx context.Context, states []st.CoordinatorState) error {now := time.Now().UTC()- for _, record := range records {- if !pruneRuns(&record, now) {+ for _, state := range states {+ record := st.QueueRecord{Runs: slices.Clone(state.Record.Runs), Receipts: maps.Clone(state.Record.Receipts)}+ if pruneRuns(&record, now) {+ if mutateErr := q.mutate(ctx, state.Record.EnvironmentID, state.Record.JobID, func(current *st.QueueRecord) error {+ pruneRuns(current, now)+ return nil+ }); mutateErr != nil {+ return fmt.Errorf("prune job %q in environment %q: %w", state.Record.JobID, state.Record.EnvironmentID, mutateErr)+ }+ }+ if state.Record.Schedule != "" || len(state.PendingActivitySync) > 0 || len(record.Runs) > 0 || len(record.Receipts) > 0 {continue}- if mutateErr := q.mutate(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {- pruneRuns(current, now)- return nil- }); mutateErr != nil {- return fmt.Errorf("prune job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)+ if _, removeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(state.Record.EnvironmentID+"\x00"+state.Record.JobID), "remove", nil); removeErr != nil {+ return fmt.Errorf("remove job %q in environment %q: %w", state.Record.JobID, state.Record.EnvironmentID, removeErr)}}return nildiff --git a/backend/pkg/scheduler/runs/coordinator_test.go b/backend/pkg/scheduler/runs/coordinator_test.gonew file mode 100644index 000000000..8dbf84094--- /dev/null+++ b/backend/pkg/scheduler/runs/coordinator_test.go@@ -0,0 +1,63 @@+package runs++import (+ "context"+ "sync"+ "testing"++ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/stretchr/testify/require"+)++type claimingObserverFake struct {+ once sync.Once+ target string+ claim func(st.Run)+ claimed bool+}++func (o *claimingObserverFake) ActivityID(run st.Run) string {+ if o.target != "" && run.ID == o.target && run.Status == st.Queued {+ o.once.Do(func() {+ o.claimed = true+ o.claim(run)+ })+ }+ return ""+}++func (o *claimingObserverFake) SyncRunActivity(context.Context, st.Run) error { return nil }++func TestSubmitRetryAfterClaimAppendsFreshRun(t *testing.T) {+ q := newResolutionQueue(t)+ observer := &claimingObserverFake{}+ q.SetObserver(observer)+ queued, err := q.Submit(t.Context(), st.Request{JobID: "coalesce", Trigger: "scheduled"})+ require.NoError(t, err)+ observer.target = queued.ID+ observer.claim = func(run st.Run) {+ claim, claimErr := q.claimExecution(t.Context(), st.ExecutionCommand{+ EnvironmentID: run.EnvironmentID,+ JobID: run.JobID,+ RunID: run.ID,+ Attempt: 1,+ })+ require.NoError(t, claimErr)+ require.True(t, claim.Claimed)+ }++ submitted, err := q.Submit(t.Context(), st.Request{JobID: "coalesce", Trigger: "scheduled"})+ require.NoError(t, err)+ require.True(t, observer.claimed)+ require.NotEqual(t, queued.ID, submitted.ID)+ require.Equal(t, st.Queued, submitted.Status)++ history, err := q.List(t.Context(), "0", "coalesce", 1, 10)+ require.NoError(t, err)+ require.Equal(t, 2, history.Total)+ statuses := map[string]st.RunStatus{}+ for _, run := range history.Runs {+ statuses[run.ID] = run.Status+ }+ require.Equal(t, map[string]st.RunStatus{queued.ID: st.Running, submitted.ID: st.Queued}, statuses)+}diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.goindex 87105f39c..a8b3e7113 100644--- a/backend/pkg/scheduler/scheduler.go+++ b/backend/pkg/scheduler/scheduler.go@@ -145,8 +145,8 @@ func (js *jobSchedulerInternal) StartScheduler(ctx context.Context) error {_, watching := js.watchers[record.JobID]js.mu.RUnlock()if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) && !watching {- if checkpointErr := js.coordinator.Checkpoint(ctx, record.JobID, "", time.Time{}); checkpointErr != nil {- return checkpointErr+ if removeErr := js.coordinator.Remove(ctx, record.JobID); removeErr != nil {+ return removeErr}}}@@ -202,7 +202,7 @@ func (js *jobSchedulerInternal) RemoveJob(ctx context.Context, id string) {started := js.startedjs.mu.Unlock()if started {- if err := js.coordinator.Checkpoint(ctx, id, "", time.Time{}); err != nil {+ if err := js.coordinator.Remove(ctx, id); err != nil {slog.ErrorContext(ctx, "Unschedule failed", "job", id, "error", err)}}diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.goindex 2dfd9e26e..52684364d 100644--- a/backend/pkg/scheduler/scheduler_test.go+++ b/backend/pkg/scheduler/scheduler_test.go@@ -8,6 +8,7 @@ import ("time"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/italypaleale/francis/actor""github.com/stretchr/testify/require")@@ -303,10 +304,8 @@ func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {require.NoError(t, err)require.Equal(t, "0 0 * * * *", polling.Schedule)require.True(t, nextPolling.Equal(polling.NextRun))- removed, err := js.coordinator.ScheduleState(t.Context(), "removed-job")- require.NoError(t, err)- require.Empty(t, removed.Schedule)- require.True(t, removed.NextRun.IsZero())+ _, err = js.coordinator.ScheduleState(t.Context(), "removed-job")+ require.ErrorIs(t, err, actor.ErrStateNotFound)select {case <-executed:case <-time.After(5 * time.Second):@@ -447,9 +446,8 @@ func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {js.RemoveJob(t.Context(), job.Name())require.False(t, js.HasJob(job.Name()))- record, err := js.coordinator.ScheduleState(t.Context(), job.Name())- require.NoError(t, err)- require.True(t, record.NextRun.IsZero())+ _, err := js.coordinator.ScheduleState(t.Context(), job.Name())+ require.ErrorIs(t, err, actor.ErrStateNotFound)}func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {diff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.goindex 8c6c49243..5a27b942e 100644--- a/backend/pkg/utils/francis/provider.go+++ b/backend/pkg/utils/francis/provider.go@@ -72,6 +72,9 @@ func StoreURL(databaseURL string) (string, error) {if !sqliteutil.HasPragma(query, "synchronous") {query.Add("_pragma", "synchronous(NORMAL)")}+ if !sqliteutil.HasPragma(query, "journal_mode") {+ query.Add("_pragma", "journal_mode(WAL)")+ }return "file:" + (&url.URL{Path: StorePath(databasePath)}).EscapedPath() + "?" + query.Encode(), nilcase strings.HasPrefix(databaseURL, "postgres"):return databaseURL, nildiff --git a/backend/pkg/utils/francis/provider_test.go b/backend/pkg/utils/francis/provider_test.goindex dd5ecef1a..e76893935 100644--- a/backend/pkg/utils/francis/provider_test.go+++ b/backend/pkg/utils/francis/provider_test.go@@ -18,15 +18,16 @@ func TestMigrateLegacyStore(t *testing.T) {databaseURL := "file:" + mainPath + "?_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate"storeURL, err := StoreURL(databaseURL)require.NoError(t, err)- const defaults = "_pragma=busy_timeout%2810000%29&_pragma=synchronous%28NORMAL%29"- require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+defaults+"&_txlock=immediate", storeURL)+ const tuned = "_pragma=busy_timeout%2810000%29&_pragma=synchronous%28NORMAL%29"+ const defaults = tuned + "&_pragma=journal_mode%28WAL%29"+ require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+tuned+"&_txlock=immediate", storeURL)for input, expected := range map[string]string{"file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db?" + defaults + "&_txlock=immediate","file:/data/arcane%2Edb": "file:/data/arcane.francis.db?" + defaults,"file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db?" + defaults,"file:data/arcane": "file:data/arcane.francis.db?" + defaults,- "file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29",- "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%2810000%29",+ "file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29&_pragma=journal_mode%28WAL%29",+ "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%2810000%29&_pragma=journal_mode%28WAL%29","file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%2810000%29","postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",} {