Skip to content

Modern Docker Management, Designed for Everyone

fix: bound shutdown waits and harden scheduler lifecycle (#4493)

Kyle Mendell10ee66bbc4d2
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 84e06b20d..2a9b8b33a 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -97,7 +97,7 @@ func Bootstrap(ctx context.Context) error {
return fmt.Errorf("setup telemetry: %w", err)
}
defer func() {
- shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+ shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 4*time.Second)
defer cancel()
if shutdownErr := providers.Shutdown(shutdownCtx); shutdownErr != nil {
slog.ErrorContext(shutdownCtx, "Error shutting down telemetry", "error", shutdownErr)
@@ -163,6 +163,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
initializeStartupState,
registerJobs,
startEdgeTunnelClient,
+ startGlobalVariableSync,
),
fx.WithLogger(func() fxevent.Logger {
logger := &fxevent.SlogLogger{Logger: slog.Default()}
@@ -170,7 +171,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
return logger
}),
fx.StartTimeout(5*time.Minute),
- fx.StopTimeout(8*time.Second),
+ fx.StopTimeout(7*time.Second),
)
}
@@ -210,11 +211,15 @@ type initializeStartupStateParams struct {
Vulnerability *vulnerability.VulnerabilityService
}
+const startupTaskTimeout = 30 * time.Second
+
func initializeStartupState(p initializeStartupStateParams) {
appCtx := p.AppCtx
cfg := p.Config
- startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)
+ helperCtx, cancelHelpers := context.WithTimeout(appCtx, startupTaskTimeout)
+ startup.CleanupOrphanedVolumeHelpers(helperCtx, p.Volume.CleanupOrphanedVolumeHelpers)
+ cancelHelpers()
runtimeCfg := &startup.RuntimeConfig{
AgentMode: cfg.AgentMode,
@@ -264,9 +269,11 @@ func initializeStartupState(p initializeStartupStateParams) {
{name: "recover stuck project statuses", run: p.Project.RecoverStuckProjectStatuses},
{name: "fail interrupted image patches", run: p.Image.RecoverInterruptedPatches},
} {
- if err := task.run(appCtx); err != nil {
+ taskCtx, cancelTask := context.WithTimeout(appCtx, startupTaskTimeout)
+ if err := task.run(taskCtx); err != nil {
slog.WarnContext(appCtx, "Startup task failed", "task", task.name, "error", err)
}
+ cancelTask()
}
p.Vulnerability.ImportLegacyReportFiles(appCtx)
p.Vulnerability.BackfillScanItems(appCtx)
@@ -282,10 +289,11 @@ func initializeStartupState(p initializeStartupStateParams) {
if err := p.Variable.ImportLegacyLocalEnvFile(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to import legacy global variables", "error", err)
}
- go p.Variable.SyncAll(appCtx)
}
- startup.TestDockerConnection(appCtx, func(ctx context.Context) error {
+ dockerCtx, cancelDocker := context.WithTimeout(appCtx, startupTaskTimeout)
+ defer cancelDocker()
+ startup.TestDockerConnection(dockerCtx, func(ctx context.Context) error {
_, err := p.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (struct{}, error) {
version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})
if err != nil {
@@ -298,7 +306,7 @@ func initializeStartupState(p initializeStartupStateParams) {
})
return err
})
- if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {
+ if err := p.Swarm.SyncSwarmEnabledState(dockerCtx); err != nil {
slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)
}
@@ -376,6 +384,16 @@ func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.
})
}
+func startGlobalVariableSync(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, variables *variable.VariableService, _ *http.Server) {
+ if cfg.AgentMode {
+ return
+ }
+ lc.Append(fx.Hook{OnStart: func(context.Context) error {
+ go variables.SyncAll(appCtx)
+ return nil
+ }})
+}
+
func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelCauseFunc) {
lc.Append(fx.Hook{
OnStop: func(context.Context) error {
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index ad820fd05..9b82b07ce 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -341,11 +341,11 @@ func TestHTTPServerStopCancelsStreamingRequestContexts(t *testing.T) {
defer func() { _ = resp.Body.Close() }()
<-handlerEntered
- shutdownCtx, cancelShutdown := context.WithTimeout(t.Context(), 2*time.Second)
+ shutdownCtx, cancelShutdown := context.WithTimeout(t.Context(), 6*time.Second)
defer cancelShutdown()
start := time.Now()
require.NoError(t, lifecycle.Stop(shutdownCtx))
- require.Less(t, time.Since(start), time.Second)
+ require.Less(t, time.Since(start), 4*time.Second)
}
func TestNewHTTPServerRejectsInvalidTLSCertificate(t *testing.T) {
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 21ea08179..ae5719f5f 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "maps"
"net/http"
"slices"
"sync"
@@ -113,14 +114,18 @@ func newJobScheduler(
},
OnStop: func(ctx context.Context) error {
cancelScheduler()
- if resumeDone != nil {
+ resumeErr := func() error {
+ if resumeDone == nil {
+ return nil
+ }
select {
case <-resumeDone:
+ return nil
case <-ctx.Done():
return ctx.Err()
}
- }
- if stopErr := errors.Join(jobService.Coordinator().Stop(ctx), jobScheduler.Stop(ctx)); stopErr != nil {
+ }()
+ if stopErr := errors.Join(resumeErr, jobService.Coordinator().Stop(ctx), jobScheduler.Stop(ctx)); stopErr != nil {
slog.ErrorContext(ctx, "Job scheduler exited with error", "error", stopErr)
return stopErr
}
@@ -464,12 +469,14 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {
subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {
if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)
+ params.FilesystemWatcher.RequestRestart()
}
})
subscribe([]string{"templatesDirectory"}, func(_ []libarcane.SettingUpdate) {
if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)
+ params.FilesystemWatcher.RequestRestart()
}
})
@@ -498,7 +505,7 @@ func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {
type timeoutSyncWorker struct {
mu sync.Mutex
- updates [][]libarcane.SettingUpdate
+ updates map[string]string
wake chan struct{}
done chan struct{}
ctx context.Context
@@ -536,15 +543,11 @@ func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscr
return
case <-worker.wake:
}
- for {
- worker.mu.Lock()
- if len(worker.updates) == 0 {
- worker.mu.Unlock()
- break
- }
- updates := worker.updates[0]
- worker.updates = worker.updates[1:]
- worker.mu.Unlock()
+ worker.mu.Lock()
+ updates := worker.updates
+ worker.updates = nil
+ worker.mu.Unlock()
+ if len(updates) > 0 {
syncTimeoutSettingsToAgents(ctx, params.Environment, updates)
}
}
@@ -555,7 +558,12 @@ func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscr
if worker.closed {
return
}
- worker.updates = append(worker.updates, slices.Clone(updates))
+ if worker.updates == nil {
+ worker.updates = make(map[string]string, len(updates))
+ }
+ for _, update := range updates {
+ worker.updates[update.Key] = update.Value
+ }
select {
case worker.wake <- struct{}{}:
default:
@@ -565,7 +573,7 @@ func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscr
}
// syncTimeoutSettingsToAgents syncs timeout settings to all connected remote environments
-func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSettingsEnvironment, timeoutSettings []libarcane.SettingUpdate) {
+func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSettingsEnvironment, settingsMap map[string]string) {
envs, err := localEnvironment.ListRemoteEnvironments(ctx)
if err != nil {
slog.WarnContext(ctx, "Failed to list remote environments for timeout sync", "error", err)
@@ -576,20 +584,13 @@ func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSe
return
}
- // Build the settings update payload
- settingsMap := make(map[string]string, len(timeoutSettings))
- keys := make([]string, 0, len(timeoutSettings))
- for _, update := range timeoutSettings {
- settingsMap[update.Key] = update.Value
- keys = append(keys, update.Key)
- }
body, err := json.Marshal(settingsMap)
if err != nil {
slog.WarnContext(ctx, "Failed to marshal timeout settings for sync", "error", err)
return
}
- slog.InfoContext(ctx, "Syncing environment settings to remote environments", "count", len(envs), "keys", keys)
+ slog.InfoContext(ctx, "Syncing environment settings to remote environments", "count", len(envs), "keys", slices.Sorted(maps.Keys(settingsMap)))
for _, env := range envs {
responseBody, statusCode, proxyRequestErr := localEnvironment.ProxyRequest(ctx, env.ID, http.MethodPut, "/api/environments/0/settings", body)
diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.go
index ecb12c5f8..88f5f0686 100644
--- a/backend/internal/bootstrap/server_bootstrap.go
+++ b/backend/internal/bootstrap/server_bootstrap.go
@@ -124,16 +124,36 @@ func NewHTTPServer(lc fx.Lifecycle, p HTTPServerParams) (*http.Server, error) {
return nil
},
OnStop: func(ctx context.Context) error {
- // Shutdown does not cancel active request contexts, so unblock streaming
- // handlers before waiting for the server to drain.
- cancelBase()
- shutdownErr := srv.Shutdown(ctx)
+ shutdownDone := make(chan error, 1)
+ go func() { shutdownDone <- srv.Shutdown(ctx) }()
+ drainTimer := time.NewTimer(3 * time.Second)
+ var shutdownErr error
+ select {
+ case shutdownErr = <-shutdownDone:
+ drainTimer.Stop()
+ cancelBase()
+ case <-drainTimer.C:
+ cancelBase()
+ shutdownErr = <-shutdownDone
+ }
if shutdownErr != nil {
slog.ErrorContext(ctx, "Server forced to shutdown", "error", shutdownErr)
}
if grpcServer != nil {
- grpcServer.GracefulStop()
+ grpcStopped := make(chan struct{})
+ go func() {
+ grpcServer.GracefulStop()
+ close(grpcStopped)
+ }()
+ grpcTimer := time.NewTimer(2 * time.Second)
+ select {
+ case <-grpcStopped:
+ grpcTimer.Stop()
+ case <-grpcTimer.C:
+ grpcServer.Stop()
+ <-grpcStopped
+ }
}
if shutdownErr == nil {
slog.InfoContext(ctx, "Server stopped gracefully")
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go
index 3564192b8..1514c67ca 100644
--- a/backend/internal/database/database.go
+++ b/backend/internal/database/database.go
@@ -1,6 +1,7 @@
package database
import (
+ "cmp"
"context"
"database/sql"
"errors"
@@ -455,6 +456,17 @@ func ParseSQLiteConnectionString(connString string) (string, error) {
qs := sqliteutil.NormalizeQuery(connStringUrl.Query())
// Foreign-key enforcement is required on every pooled connection.
qs.Add("_pragma", "foreign_keys(1)")
+ if !sqliteutil.HasPragma(qs, "busy_timeout") {
+ qs.Add("_pragma", "busy_timeout(10000)")
+ }
+ if !slices.Contains([]string{"ro", "memory"}, qs.Get("mode")) && !strings.HasPrefix(cmp.Or(connStringUrl.Opaque, connStringUrl.Path), ":memory:") {
+ if !sqliteutil.HasPragma(qs, "journal_mode") {
+ qs.Add("_pragma", "journal_mode(WAL)")
+ }
+ if qs.Get("_txlock") == "" {
+ qs.Set("_txlock", "immediate")
+ }
+ }
return (&url.URL{
Scheme: connStringUrl.Scheme,
Opaque: connStringUrl.Opaque,
diff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.go
index bef1fde13..b8ad586c7 100644
--- a/backend/internal/database/database_test.go
+++ b/backend/internal/database/database_test.go
@@ -1464,10 +1464,63 @@ func TestSQLiteDSN(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "immediate", parsed.Query().Get("_txlock"))
require.Equal(t, "shared", parsed.Query().Get("cache"))
- require.ElementsMatch(t, []string{"foreign_keys(1)", "synchronous(NORMAL)", "busy_timeout(2500)"}, parsed.Query()["_pragma"])
+ require.ElementsMatch(t, []string{"foreign_keys(1)", "synchronous(NORMAL)", "busy_timeout(2500)", "journal_mode(WAL)"}, parsed.Query()["_pragma"])
require.Empty(t, parsed.Query().Get("_fk"))
require.Empty(t, parsed.Query().Get("_foreign_keys"))
require.Empty(t, parsed.Query().Get("_busy_timeout"))
})
}
}
+
+func TestSQLiteDSN_ConcurrencyDefaults(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ pragmas []string
+ txlock string
+ }{
+ {
+ name: "defaults added",
+ input: "file:arcane.db",
+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)", "journal_mode(WAL)"},
+ txlock: "immediate",
+ },
+ {
+ name: "explicit values kept",
+ input: "file:arcane.db?_journal_mode=DELETE&_timeout=500&_txlock=deferred",
+ pragmas: []string{"foreign_keys(1)", "busy_timeout(500)", "journal_mode(DELETE)"},
+ txlock: "deferred",
+ },
+ {
+ name: "explicit pragmas kept",
+ input: "file:arcane.db?_pragma=journal_mode(TRUNCATE)&_pragma=busy_timeout(2000)",
+ pragmas: []string{"foreign_keys(1)", "busy_timeout(2000)", "journal_mode(TRUNCATE)"},
+ txlock: "immediate",
+ },
+ {
+ name: "read-only skips WAL and txlock",
+ input: "file:arcane.db?mode=ro",
+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)"},
+ },
+ {
+ name: "memory path skips WAL and txlock",
+ input: "file::memory:?cache=shared",
+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)"},
+ },
+ {
+ name: "memory mode skips WAL and txlock",
+ input: "file:arcane?mode=memory&cache=shared",
+ pragmas: []string{"foreign_keys(1)", "busy_timeout(10000)"},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ dsn, err := ParseSQLiteConnectionString(tt.input)
+ require.NoError(t, err)
+ parsed, err := url.Parse(dsn)
+ require.NoError(t, err)
+ require.ElementsMatch(t, tt.pragmas, parsed.Query()["_pragma"])
+ require.Equal(t, tt.txlock, parsed.Query().Get("_txlock"))
+ })
+ }
+}
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 7c0e390d6..c16059591 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -141,7 +141,9 @@ func provideTunnelRegistry(lc fx.Lifecycle) *edge.TunnelRegistry {
edge.SetDefaultRegistry(localRegistry)
lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {
defer edge.ClearDefaultRegistry(localRegistry)
- return localRegistry.Stop(ctx)
+ stopCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ return localRegistry.Stop(stopCtx)
}})
return localRegistry
}
@@ -340,27 +342,7 @@ func provideFilesystemWatcherJob(
var stop func(context.Context) error
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
- stop, err = concurrency.StartSupervised(ctx, "filesystem watcher", func(runCtx context.Context) error {
- delay := 5 * time.Second
- for runCtx.Err() == nil {
- startErr := job.Start(runCtx)
- if startErr == nil {
- break
- }
- slog.WarnContext(runCtx, "Filesystem watcher startup failed; retrying", "error", startErr)
-
- timer := time.NewTimer(delay)
- select {
- case <-runCtx.Done():
- timer.Stop()
- return nil
- case <-timer.C:
- }
- delay = min(delay*2, 5*time.Minute)
- }
- <-runCtx.Done()
- return nil
- })
+ stop, err = concurrency.StartSupervised(ctx, "filesystem watcher", job.Supervise)
if err != nil {
return err
}
@@ -414,7 +396,12 @@ func provideVolumeService(
if err := service.RegisterWorkflows(engine); err != nil {
return nil, err
}
- lc.Append(fx.Hook{OnStop: func(ctx context.Context) error { service.CleanupHelperContainers(ctx); return nil }})
+ lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {
+ cleanupCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ service.CleanupHelperContainers(cleanupCtx)
+ return nil
+ }})
return service, nil
}
diff --git a/backend/internal/webhook/service.go b/backend/internal/webhook/service.go
index d5ece7b55..bb6f4e23c 100644
--- a/backend/internal/webhook/service.go
+++ b/backend/internal/webhook/service.go
@@ -626,6 +626,8 @@ func (s *WebhookService) TriggerByToken(ctx context.Context, rawToken string) er
// DrainActions blocks until all accepted background webhook actions finish or
// ctx expires, so shutdown does not silently drop acknowledged work.
func (s *WebhookService) DrainActions(ctx context.Context) error {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
return utils.WaitGroup(ctx, &s.actions)
}
diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.go
index 286d7f31b..f5005a69a 100644
--- a/backend/pkg/fswatch/watcher.go
+++ b/backend/pkg/fswatch/watcher.go
@@ -8,9 +8,11 @@ import (
"context"
"errors"
"log/slog"
+ "maps"
"os"
"path/filepath"
"runtime"
+ "slices"
"strings"
"sync"
"time"
@@ -34,6 +36,9 @@ type Watcher struct {
cancel context.CancelFunc
watchAliases map[string]string
pendingPaths map[string]struct{}
+ readyPaths map[string]struct{}
+ changed bool
+ wake chan struct{}
mu sync.Mutex
started bool
stopped bool
@@ -73,6 +78,8 @@ func NewWatcher(watchPath string, opts WatcherOptions) (*Watcher, error) {
stoppedCh: make(chan struct{}),
watchAliases: make(map[string]string),
pendingPaths: make(map[string]struct{}),
+ readyPaths: make(map[string]struct{}),
+ wake: make(chan struct{}, 1),
}, nil
}
@@ -110,6 +117,7 @@ func (fw *Watcher) Start(ctx context.Context) error {
watchCtx, cancel := context.WithCancel(ctx)
fw.cancel = cancel
go fw.watchLoopInternal(watchCtx)
+ fw.callbacks.Go(func() { fw.notifyLoop(watchCtx) })
fw.started = true
slog.InfoContext(ctx, "Filesystem watcher started", "path", fw.watchedPath)
@@ -118,13 +126,6 @@ func (fw *Watcher) Start(ctx context.Context) error {
// Stop closes the watcher and waits for accepted callbacks.
func (fw *Watcher) Stop() error {
- err := fw.StopWatching()
- fw.callbacks.Wait()
- return err
-}
-
-// StopWatching closes the watch loop before its owner joins callbacks.
-func (fw *Watcher) StopWatching() error {
fw.stopOnce.Do(func() {
fw.mu.Lock()
fw.stopped = true
@@ -145,6 +146,8 @@ func (fw *Watcher) StopWatching() error {
fw.mu.Unlock()
})
+ fw.callbacks.Wait()
+
fw.mu.Lock()
defer fw.mu.Unlock()
@@ -170,21 +173,21 @@ func (fw *Watcher) watchLoopInternal(ctx context.Context) {
return
case event, ok := <-fw.watcher.Events:
if fw.processEventInternal(ctx, event, ok, debounceTimer, &debouncePending) {
- if !fw.reconnectInternal(ctx) {
+ if !fw.reconnect(ctx) {
return
}
}
case <-debounceTimer.C:
- if fw.fireDebounceInternal(ctx, &debouncePending, &lastGoroutineLog) {
+ if fw.fireDebounce(ctx, &debouncePending, &lastGoroutineLog) {
continue
}
case <-healthTicker.C:
- if len(fw.watcher.WatchList()) == 0 && !fw.reconnectInternal(ctx) {
+ if len(fw.watcher.WatchList()) == 0 && !fw.reconnect(ctx) {
return
}
case err, ok := <-fw.watcher.Errors:
if !ok {
- if !fw.reconnectInternal(ctx) {
+ if !fw.reconnect(ctx) {
return
}
continue
@@ -200,13 +203,13 @@ func (fw *Watcher) processEventInternal(ctx context.Context, event fsnotify.Even
return true
}
- // handleEventInternal runs for every event — its job is to keep the
+ // handleEvent runs for every event — its job is to keep the
// underlying fsnotify subscriptions in sync (attaching newly-created
// directories so we can discover compose files that land inside them
- // afterwards). Sync-firing is gated separately by shouldHandleEventInternal.
- fw.handleEventInternal(ctx, event)
+ // afterwards). Sync-firing is gated separately by shouldHandleEvent.
+ fw.handleEvent(ctx, event)
- if !fw.shouldHandleEventInternal(event) {
+ if !fw.shouldHandleEvent(event) {
return false
}
fw.recordPendingPathInternal(event.Name)
@@ -215,8 +218,8 @@ func (fw *Watcher) processEventInternal(ctx context.Context, event fsnotify.Even
return false
}
-// fireDebounceInternal runs the debounced onChange callback; returns true if nothing was pending (caller should continue).
-func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bool, lastGoroutineLog *time.Time) bool {
+// fireDebounce runs the debounced onChange callback; returns true if nothing was pending (caller should continue).
+func (fw *Watcher) fireDebounce(ctx context.Context, debouncePending *bool, lastGoroutineLog *time.Time) bool {
if !*debouncePending {
return true
}
@@ -227,28 +230,48 @@ func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bo
"goroutines", runtime.NumGoroutine())
*lastGoroutineLog = time.Now()
}
- var paths []string
- if fw.onChangePaths != nil {
- paths = fw.drainPendingPathsInternal()
- }
- fw.notifyChangesInternal(ctx, paths)
+ fw.queueChanges(fw.drainPendingPaths())
return false
}
-func (fw *Watcher) notifyChangesInternal(ctx context.Context, paths []string) {
- if fw.onChange != nil {
- fw.callbacks.Go(func() {
- if ctx.Err() == nil {
- fw.onChange(ctx)
- }
- })
+func (fw *Watcher) queueChanges(paths []string) {
+ fw.mu.Lock()
+ fw.changed = true
+ for _, path := range paths {
+ fw.readyPaths[path] = struct{}{}
}
- if fw.onChangePaths != nil && len(paths) > 0 {
- fw.callbacks.Go(func() {
- if ctx.Err() == nil {
- fw.onChangePaths(ctx, paths)
- }
- })
+ fw.mu.Unlock()
+ select {
+ case fw.wake <- struct{}{}:
+ default:
+ }
+}
+
+func (fw *Watcher) notifyLoop(ctx context.Context) {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-fw.wake:
+ }
+ if ctx.Err() != nil {
+ return
+ }
+ fw.mu.Lock()
+ changed := fw.changed
+ paths := slices.Collect(maps.Keys(fw.readyPaths))
+ fw.changed = false
+ clear(fw.readyPaths)
+ fw.mu.Unlock()
+ if !changed {
+ continue
+ }
+ if fw.onChange != nil {
+ fw.onChange(ctx)
+ }
+ if fw.onChangePaths != nil && len(paths) > 0 {
+ fw.onChangePaths(ctx, paths)
+ }
}
}
@@ -261,24 +284,29 @@ func (fw *Watcher) recordPendingPathInternal(path string) {
fw.mu.Unlock()
}
-func (fw *Watcher) drainPendingPathsInternal() []string {
+func (fw *Watcher) drainPendingPaths() []string {
fw.mu.Lock()
defer fw.mu.Unlock()
- paths := make([]string, 0, len(fw.pendingPaths))
- for path := range fw.pendingPaths {
- paths = append(paths, path)
- delete(fw.pendingPaths, path)
- }
+ paths := slices.Collect(maps.Keys(fw.pendingPaths))
+ clear(fw.pendingPaths)
return paths
}
-// handleEventInternal keeps the underlying fsnotify subscriptions in sync with
+// handleEvent keeps the underlying fsnotify subscriptions in sync with
// the filesystem: when a new directory is created inside a watched path, it is
// attached recursively so that compose files dropped inside it are discovered.
// It does NOT decide whether the event should trigger a sync — that is
-// shouldHandleEventInternal's job.
-func (fw *Watcher) handleEventInternal(ctx context.Context, event fsnotify.Event) {
+// shouldHandleEvent's job.
+func (fw *Watcher) handleEvent(ctx context.Context, event fsnotify.Event) {
+ if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
+ removed := filepath.Clean(event.Name)
+ if _, watched := fw.watchAliases[removed]; watched {
+ maps.DeleteFunc(fw.watchAliases, func(watchPath, _ string) bool {
+ return watchPath == removed || strings.HasPrefix(watchPath, removed+string(os.PathSeparator))
+ })
+ }
+ }
if event.Has(fsnotify.Create) && !projects.IsIgnoredProjectDirName(filepath.Base(event.Name)) {
logicalName := fw.logicalPathForWatchEventInternal(event.Name)
if fw.isWatchableDirectory(event.Name) {
@@ -297,25 +325,17 @@ func (fw *Watcher) handleEventInternal(ctx context.Context, event fsnotify.Event
"operation", event.Op.String())
}
-// shouldHandleEventInternal reports whether an fsnotify event should trigger a
-// project sync. Only events on known compose / env files qualify; bare
-// directory events do not, because build tools (ESPHome, PlatformIO, npm,
-// etc.) churn through subdirectories and would otherwise spin the sync loop
-// forever. New project directories are still discovered: once the directory
-// create event has been processed by handleEventInternal (which attaches a
-// watch), the compose file's own Create event reaches this filter and passes
-// via IsProjectFile.
-func (fw *Watcher) shouldHandleEventInternal(event fsnotify.Event) bool {
+func (fw *Watcher) shouldHandleEvent(event fsnotify.Event) bool {
if !event.Has(fsnotify.Write) && !event.Has(fsnotify.Create) && !event.Has(fsnotify.Rename) && !event.Has(fsnotify.Remove) && !event.Has(fsnotify.Chmod) {
return false
}
+ if !projects.IsProjectFile(filepath.Base(event.Name)) {
+ return false
+ }
// Arcane's own scratch directories (update backups, GitOps staging) churn
// compose files that must not re-sync the live projects.
rel, err := filepath.Rel(fw.watchedPath, fw.logicalPathForWatchEventInternal(event.Name))
- if err == nil && projects.PathContainsIgnoredDirectory(rel) {
- return false
- }
- return projects.IsProjectFile(filepath.Base(event.Name))
+ return err != nil || !projects.PathContainsIgnoredDirectory(rel)
}
func (fw *Watcher) addExistingDirectories(ctx context.Context, root string) error {
@@ -478,8 +498,8 @@ func (fw *Watcher) isWatchableDirectory(path string) bool {
return resolvedInfo.IsDir()
}
-// reconnectInternal replaces a lost filesystem subscription without abandoning its owner.
-func (fw *Watcher) reconnectInternal(ctx context.Context) bool {
+// reconnect replaces a lost filesystem subscription without abandoning its owner.
+func (fw *Watcher) reconnect(ctx context.Context) bool {
if err := fw.watcher.Close(); err != nil {
slog.DebugContext(ctx, "Closing lost filesystem subscription", "error", err)
}
@@ -505,7 +525,7 @@ func (fw *Watcher) reconnectInternal(ctx context.Context) bool {
slog.WarnContext(ctx, "Some filesystem subscriptions could not be restored", "error", addExistingDirectoriesErr)
}
// Changes while disconnected need a full reconciliation.
- fw.notifyChangesInternal(ctx, []string{fw.watchedPath})
+ fw.queueChanges([]string{fw.watchedPath})
return true
}
if closeErr := watcher.Close(); closeErr != nil {
diff --git a/backend/pkg/scheduler/event_cleanup_job.go b/backend/pkg/scheduler/event_cleanup_job.go
index c38988a8c..287ee4cf9 100644
--- a/backend/pkg/scheduler/event_cleanup_job.go
+++ b/backend/pkg/scheduler/event_cleanup_job.go
@@ -2,6 +2,8 @@ package scheduler
import (
"context"
+ "errors"
+ "fmt"
"log/slog"
"time"
@@ -52,26 +54,21 @@ func (j *EventCleanupJob) Schedule(ctx context.Context) string {
func (j *EventCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
slog.InfoContext(ctx, "Running event cleanup job", "jobName", EventCleanupJobName)
- // Delete events older than 36 hours
+ var cleanupErr error
olderThan := 36 * time.Hour
if err := j.eventService.DeleteOldEvents(ctx, olderThan); err != nil {
- slog.ErrorContext(ctx, "Failed to delete old events", "jobName", EventCleanupJobName, "olderThan", olderThan.String(), "error", err)
- return schedulertypes.Outcome{}, err
+ cleanupErr = fmt.Errorf("delete old events: %w", err)
+ } else {
+ slog.InfoContext(ctx, "Event cleanup completed successfully",
+ "jobName", EventCleanupJobName,
+ "olderThan", olderThan.String())
}
- slog.InfoContext(ctx, "Event cleanup job completed successfully",
- "jobName", EventCleanupJobName,
- "olderThan", olderThan.String())
-
- if j.activityService != nil {
- retentionDays := j.settingsService.GetIntSetting(ctx, "activityHistoryRetentionDays", 30)
- maxEntries := j.settingsService.GetIntSetting(ctx, "activityHistoryMaxEntries", 1000)
- deleted, err := j.activityService.PruneHistory(ctx, retentionDays, maxEntries)
- if err != nil {
- slog.ErrorContext(ctx, "Failed to prune activity history", "jobName", EventCleanupJobName, "error", err)
- return schedulertypes.Outcome{}, err
- }
-
+ retentionDays := j.settingsService.GetIntSetting(ctx, "activityHistoryRetentionDays", 30)
+ maxEntries := j.settingsService.GetIntSetting(ctx, "activityHistoryMaxEntries", 1000)
+ if deleted, err := j.activityService.PruneHistory(ctx, retentionDays, maxEntries); err != nil {
+ cleanupErr = errors.Join(cleanupErr, fmt.Errorf("prune activity history: %w", err))
+ } else {
slog.InfoContext(ctx, "Activity history cleanup completed successfully",
"jobName", EventCleanupJobName,
"retentionDays", retentionDays,
@@ -80,29 +77,29 @@ func (j *EventCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, erro
}
buildRetentionDays := j.settingsService.GetIntSetting(ctx, "buildHistoryRetentionDays", 30)
- deletedBuilds, err := j.buildService.PruneHistory(ctx, buildRetentionDays)
- if err != nil {
- slog.ErrorContext(ctx, "Failed to prune image build history", "jobName", EventCleanupJobName, "error", err)
- return schedulertypes.Outcome{}, err
+ if deletedBuilds, err := j.buildService.PruneHistory(ctx, buildRetentionDays); err != nil {
+ cleanupErr = errors.Join(cleanupErr, fmt.Errorf("prune image build history: %w", err))
+ } else {
+ slog.InfoContext(ctx, "Image build history cleanup completed successfully",
+ "jobName", EventCleanupJobName,
+ "retentionDays", buildRetentionDays,
+ "deleted", deletedBuilds)
}
- slog.InfoContext(ctx, "Image build history cleanup completed successfully",
- "jobName", EventCleanupJobName,
- "retentionDays", buildRetentionDays,
- "deleted", deletedBuilds)
-
updateRetentionDays := j.settingsService.GetIntSetting(ctx, "activityHistoryRetentionDays", 30)
- deletedUpdates, err := j.updaterService.PruneHistory(ctx, updateRetentionDays)
- if err != nil {
- slog.ErrorContext(ctx, "Failed to prune auto-update history", "jobName", EventCleanupJobName, "error", err)
- return schedulertypes.Outcome{}, err
+ if deletedUpdates, err := j.updaterService.PruneHistory(ctx, updateRetentionDays); err != nil {
+ cleanupErr = errors.Join(cleanupErr, fmt.Errorf("prune auto-update history: %w", err))
+ } else {
+ slog.InfoContext(ctx, "Auto-update history cleanup completed successfully",
+ "jobName", EventCleanupJobName,
+ "retentionDays", updateRetentionDays,
+ "deleted", deletedUpdates)
}
- slog.InfoContext(ctx, "Auto-update history cleanup completed successfully",
- "jobName", EventCleanupJobName,
- "retentionDays", updateRetentionDays,
- "deleted", deletedUpdates)
-
+ if cleanupErr != nil {
+ slog.ErrorContext(ctx, "Event cleanup job failed", "jobName", EventCleanupJobName, "error", cleanupErr)
+ return schedulertypes.Outcome{}, cleanupErr
+ }
return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.go
index e7df5399e..73579b3a2 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job.go
@@ -23,9 +23,10 @@ type FilesystemWatcherJob struct {
lifecycleCtx context.Context
projectsWatcher *fswatch.Watcher
templatesWatcher *fswatch.Watcher
+ restart chan struct{}
+ restartMu sync.Mutex
mu sync.Mutex
stopped bool
- retired []*fswatch.Watcher
}
func NewFilesystemWatcherJob(
@@ -41,6 +42,7 @@ func NewFilesystemWatcherJob(
settingsService: settingsService,
projectScanDepth: projectScanDepth,
lifecycleCtx: ctx,
+ restart: make(chan struct{}, 1),
}, nil
}
@@ -48,27 +50,49 @@ func (j *FilesystemWatcherJob) Start(ctx context.Context) error {
if err := j.RestartProjectsWatcher(ctx); err != nil {
return err
}
- if err := j.RestartTemplatesWatcher(ctx); err != nil {
- j.mu.Lock()
- watcher := j.projectsWatcher
- j.projectsWatcher = nil
- j.mu.Unlock()
- if watcher != nil {
- return errors.Join(err, watcher.Stop())
+ return j.RestartTemplatesWatcher(ctx)
+}
+
+func (j *FilesystemWatcherJob) Supervise(ctx context.Context) error {
+ delay := 5 * time.Second
+ for ctx.Err() == nil {
+ startErr := j.Start(ctx)
+ if startErr == nil {
+ delay = 5 * time.Second
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-j.restart:
+ continue
+ }
}
- return err
+ slog.WarnContext(ctx, "Filesystem watcher startup failed; retrying", "error", startErr)
+
+ timer := time.NewTimer(delay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return nil
+ case <-timer.C:
+ }
+ delay = min(delay*2, 5*time.Minute)
}
return nil
}
+func (j *FilesystemWatcherJob) RequestRestart() {
+ select {
+ case j.restart <- struct{}{}:
+ default:
+ }
+}
+
func (j *FilesystemWatcherJob) Stop(ctx context.Context) error {
j.mu.Lock()
j.stopped = true
projectsWatcher, templatesWatcher := j.projectsWatcher, j.templatesWatcher
j.projectsWatcher = nil
j.templatesWatcher = nil
- retired := j.retired
- j.retired = nil
j.mu.Unlock()
var err error
if projectsWatcher != nil {
@@ -77,9 +101,6 @@ func (j *FilesystemWatcherJob) Stop(ctx context.Context) error {
if templatesWatcher != nil {
err = errors.Join(err, templatesWatcher.Stop())
}
- for _, watcher := range retired {
- err = errors.Join(err, watcher.Stop())
- }
return err
}
@@ -120,35 +141,52 @@ func (j *FilesystemWatcherJob) handleTemplatesChangeInternal(ctx context.Context
func (j *FilesystemWatcherJob) RestartProjectsWatcher(ctx context.Context) error {
slog.InfoContext(ctx, "Restarting projects filesystem watcher")
- j.mu.Lock()
- defer j.mu.Unlock()
- if j.stopped {
- return errors.New("filesystem watcher stopped")
- }
if err := ctx.Err(); err != nil {
return err
}
- if j.projectsWatcher != nil {
- previous := j.projectsWatcher
- j.projectsWatcher = nil
- j.retired = append(j.retired, previous)
- // Callbacks need the domain mutex, so stop the watch loop separately from joining them.
- if err := previous.StopWatching(); err != nil {
- return err
- }
- }
- watcher, err := j.startProjectsWatcherInternal(ctx)
+ j.restartMu.Lock()
+ defer j.restartMu.Unlock()
+ watcher, err := j.startProjectsWatcher(ctx)
if err != nil {
- if watcher != nil {
- _ = watcher.StopWatching()
- }
return err
}
- j.projectsWatcher = watcher
- return nil
+ return j.replaceWatcher(ctx, &j.projectsWatcher, watcher, func(syncCtx context.Context) {
+ if j.projectService == nil {
+ return
+ }
+ if syncErr := j.projectService.SyncProjectsFromFileSystem(syncCtx); syncErr != nil {
+ slog.ErrorContext(syncCtx, "Initial project sync after watcher start failed", "error", syncErr)
+ }
+ })
+}
+
+func (j *FilesystemWatcherJob) replaceWatcher(ctx context.Context, slot **fswatch.Watcher, watcher *fswatch.Watcher, initialSync func(context.Context)) error {
+ j.mu.Lock()
+ if j.stopped {
+ j.mu.Unlock()
+ if watcher == nil {
+ return errors.New("filesystem watcher stopped")
+ }
+ return errors.Join(errors.New("filesystem watcher stopped"), watcher.Stop())
+ }
+ previous := *slot
+ *slot = watcher
+ j.mu.Unlock()
+
+ var err error
+ if previous != nil {
+ err = previous.Stop()
+ }
+
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ if !j.stopped && ctx.Err() == nil {
+ initialSync(ctx)
+ }
+ return err
}
-func (j *FilesystemWatcherJob) startProjectsWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {
+func (j *FilesystemWatcherJob) startProjectsWatcher(ctx context.Context) (*fswatch.Watcher, error) {
localSettings, err := j.settingsService.GetSettings(ctx)
if err != nil {
return nil, err
@@ -164,15 +202,10 @@ func (j *FilesystemWatcherJob) startProjectsWatcherInternal(ctx context.Context)
return nil, err
}
if startErr := watcher.Start(j.lifecycleCtx); startErr != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.
- return watcher, startErr
+ return nil, startErr
}
slog.InfoContext(ctx, "Projects filesystem watcher started", "path", projectsDirectory)
- if j.projectService != nil {
- if syncProjectsFromFileSystemErr := j.projectService.SyncProjectsFromFileSystem(ctx); syncProjectsFromFileSystemErr != nil {
- slog.ErrorContext(ctx, "Initial project sync after watcher start failed", "error", syncProjectsFromFileSystemErr)
- }
- }
return watcher, nil
}
@@ -198,35 +231,26 @@ func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlin
func (j *FilesystemWatcherJob) RestartTemplatesWatcher(ctx context.Context) error {
slog.InfoContext(ctx, "Restarting templates filesystem watcher")
- j.mu.Lock()
- defer j.mu.Unlock()
- if j.stopped {
- return errors.New("filesystem watcher stopped")
- }
if err := ctx.Err(); err != nil {
return err
}
- if j.templatesWatcher != nil {
- previous := j.templatesWatcher
- j.templatesWatcher = nil
- j.retired = append(j.retired, previous)
- // Callbacks need the domain mutex, so stop the watch loop separately from joining them.
- if err := previous.StopWatching(); err != nil {
- return err
- }
- }
- watcher, err := j.startTemplatesWatcherInternal(ctx)
+ j.restartMu.Lock()
+ defer j.restartMu.Unlock()
+ watcher, err := j.startTemplatesWatcher(ctx)
if err != nil {
- if watcher != nil {
- _ = watcher.StopWatching()
- }
return err
}
- j.templatesWatcher = watcher
- return nil
+ return j.replaceWatcher(ctx, &j.templatesWatcher, watcher, func(syncCtx context.Context) {
+ if j.templateService == nil {
+ return
+ }
+ if syncErr := j.templateService.SyncLocalTemplatesFromFilesystem(syncCtx); syncErr != nil {
+ slog.ErrorContext(syncCtx, "Initial template sync failed", "error", syncErr)
+ }
+ })
}
-func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {
+func (j *FilesystemWatcherJob) startTemplatesWatcher(ctx context.Context) (*fswatch.Watcher, error) {
if j.templateService == nil {
return nil, nil
}
@@ -244,10 +268,6 @@ func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Context
return nil, err
}
- if syncLocalTemplatesFromFilesystemErr := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); syncLocalTemplatesFromFilesystemErr != nil {
- slog.ErrorContext(ctx, "Initial template sync failed", "error", syncLocalTemplatesFromFilesystemErr)
- }
-
if directoriesOverlapInternal(projectsDirectory, templatesDir) {
slog.ErrorContext(ctx,
"Templates and projects directories overlap; templates watcher disabled",
@@ -265,7 +285,7 @@ func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Context
return nil, err
}
if startErr := watcher.Start(j.lifecycleCtx); startErr != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.
- return watcher, startErr
+ return nil, startErr
}
slog.InfoContext(ctx, "Templates filesystem watcher started", "path", templatesDir)
diff --git a/backend/pkg/scheduler/filesystem_watcher_job_test.go b/backend/pkg/scheduler/filesystem_watcher_job_test.go
index cadad8e85..280a00519 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job_test.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job_test.go
@@ -8,6 +8,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/fswatch"
)
func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *testing.T) {
@@ -57,3 +59,33 @@ func TestFilesystemWatcherJob_ConcurrentProjectRestartsAreSerializedInternal(t *
defer cancel()
require.NoError(t, job.Stop(stopCtx))
}
+
+func TestFilesystemWatcherJob_RequestRestartRearmsSupervisor(t *testing.T) {
+ _, settingsService, _ := setupAnalyticsStateServicesInternal(t)
+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "projectsDirectory", t.TempDir()))
+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "templatesDirectory", t.TempDir()))
+
+ job, err := NewFilesystemWatcherJob(t.Context(), nil, nil, settingsService, 2)
+ require.NoError(t, err)
+
+ superviseCtx, cancelSupervise := context.WithCancel(t.Context())
+ done := make(chan error, 1)
+ go func() { done <- job.Supervise(superviseCtx) }()
+
+ currentWatcher := func() *fswatch.Watcher {
+ job.mu.Lock()
+ defer job.mu.Unlock()
+ return job.projectsWatcher
+ }
+ require.Eventually(t, func() bool { return currentWatcher() != nil }, 5*time.Second, 10*time.Millisecond)
+ firstWatcher := currentWatcher()
+
+ job.RequestRestart()
+ require.Eventually(t, func() bool { return currentWatcher() != firstWatcher }, 5*time.Second, 10*time.Millisecond)
+
+ cancelSupervise()
+ require.NoError(t, <-done)
+ stopCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ require.NoError(t, job.Stop(stopCtx))
+}
diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.go
index 6a34989e9..7fa9fe37a 100644
--- a/backend/pkg/scheduler/image_update_watcher.go
+++ b/backend/pkg/scheduler/image_update_watcher.go
@@ -73,8 +73,6 @@ type ImageUpdateWatcher struct {
backfillRetry time.Duration
metadataReady chan struct{}
metadataReadyOnce sync.Once
- started chan struct{}
- startedOnce sync.Once
stopped chan struct{}
stoppedOnce sync.Once
}
@@ -102,7 +100,6 @@ func NewImageUpdateWatcher(cfg *config.Config,
debounce: imageUpdateWatcherDebounce,
backfillRetry: imageUpdateWatcherBackfillRetry,
metadataReady: make(chan struct{}),
- started: make(chan struct{}),
stopped: make(chan struct{}),
trigger: make(chan struct{},
1),
@@ -136,18 +133,12 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
w.runMu.Unlock()
defer cancel()
defer close(done)
- return w.runEventLoopInternal(runCtx)
+ return w.runEventLoop(runCtx)
}
-func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error {
- eventBus := w.dockerService.EventBus()
- if eventBus == nil {
- return errors.New("docker event bus unavailable")
- }
- eventCh, unsubscribe := eventBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
+func (w *ImageUpdateWatcher) runEventLoop(runCtx context.Context) error {
+ eventCh, unsubscribe := w.dockerService.EventBus().Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
defer unsubscribe()
- reconnect := time.NewTicker(5 * time.Second)
- defer reconnect.Stop()
timer := time.NewTimer(time.Hour)
timer.Stop()
defer timer.Stop()
@@ -157,7 +148,6 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error
submitted := make(chan error, 1)
submitting := false
workers.Go(func() { w.runMetadataBackfillInternal(runCtx, metadata) })
- w.startedOnce.Do(func() { close(w.started) })
w.RefreshSchedule()
ready := false
for {
@@ -169,7 +159,7 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error
w.metadataReadyOnce.Do(func() { close(w.metadataReady) })
w.Trigger()
case <-w.trigger:
- if ready && !submitting {
+ if ready && !submitting && w.settingsService.GetBoolSetting(runCtx, "pollingEnabled", true) {
timer.Reset(w.debounce)
}
case <-timer.C:
@@ -188,11 +178,9 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error
timer.Reset(w.debounce)
}
case <-w.scheduleRefresh:
- if err := w.refreshScheduleInternal(runCtx); err != nil {
+ if err := w.refreshSchedule(runCtx); err != nil {
slog.ErrorContext(runCtx, "Failed to checkpoint image polling schedule", "error", err)
}
- case <-reconnect.C:
- eventCh, unsubscribe = w.reconnectEventsInternal(eventCh, unsubscribe)
case _, ok := <-eventCh:
if !ok {
eventCh = nil
@@ -206,20 +194,6 @@ func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error
}
}
-func (w *ImageUpdateWatcher) reconnectEventsInternal(eventCh <-chan events.Message, unsubscribe func()) (<-chan events.Message, func()) {
- if eventCh != nil {
- return eventCh, unsubscribe
- }
- currentBus := w.dockerService.EventBus()
- if currentBus == nil {
- return eventCh, unsubscribe
- }
- unsubscribe()
- eventCh, unsubscribe = currentBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
- w.eventDegraded.Store(false)
- return eventCh, unsubscribe
-}
-
func (w *ImageUpdateWatcher) runMetadataBackfillInternal(ctx context.Context, metadata chan<- struct{}) {
for attempt := 1; ctx.Err() == nil; attempt++ {
var err error
@@ -316,13 +290,6 @@ func (w *ImageUpdateWatcher) RefreshSchedule() {
}
func (w *ImageUpdateWatcher) RunNow(ctx context.Context) (err error) {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-w.stopped:
- return errors.New("image update watcher is not running")
- case <-w.started:
- }
select {
case <-ctx.Done():
return ctx.Err()
@@ -408,7 +375,16 @@ func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Contex
return err
}
-func (w *ImageUpdateWatcher) refreshScheduleInternal(ctx context.Context) error {
+func (w *ImageUpdateWatcher) refreshSchedule(ctx context.Context) error {
+ w.dispatchMu.RLock()
+ coordinator := w.coordinator
+ w.dispatchMu.RUnlock()
+ if coordinator == nil {
+ return errors.New("durable job coordinator unavailable")
+ }
+ if !w.settingsService.GetBoolSetting(ctx, "pollingEnabled", true) {
+ return coordinator.Checkpoint(ctx, w.Name(), "", time.Time{})
+ }
spec := w.settingsService.GetStringSetting(ctx, "pollingInterval", imageUpdateWatcherDefaultSchedule)
schedule, err := cronScheduleParser.Parse(spec)
if err != nil {
@@ -419,12 +395,6 @@ func (w *ImageUpdateWatcher) refreshScheduleInternal(ctx context.Context) error
return err
}
}
- w.dispatchMu.RLock()
- coordinator := w.coordinator
- w.dispatchMu.RUnlock()
- if coordinator == nil {
- return errors.New("durable job coordinator unavailable")
- }
return coordinator.Checkpoint(ctx, w.Name(), spec, schedule.Next(time.Now().In(w.location)))
}
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index 2b27b58f4..5fb7fd1c7 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -177,7 +177,7 @@ func (b *lockedBufferInternal) stringInternal() string {
return b.String()
}
-func newImageUpdateWatcherForTestInternal(t *testing.T,
+func newImageUpdateWatcherForTest(t *testing.T,
scanner imageUpdateScannerInternal,
settings pollingSettingReaderInternal,
eventBus *bus.DockerEventBus,
@@ -198,7 +198,6 @@ func newImageUpdateWatcherForTestInternal(t *testing.T,
debounce: 10 * time.Millisecond,
backfillRetry: 10 * time.Millisecond,
metadataReady: make(chan struct{}),
- started: make(chan struct{}),
stopped: make(chan struct{}),
}
coordinator, _ := newTestCoordinatorInternal(t, t.Context(), time.UTC, func(ctx context.Context, run schedulertypes.Run) (schedulertypes.Outcome, error) {
@@ -229,7 +228,7 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tes
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -261,7 +260,7 @@ func TestImageUpdateWatcher_EventTriggersAreOptIn(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -294,7 +293,7 @@ func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testin
}
settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Equal(t, 1, <-scanner.startedCh)
@@ -313,7 +312,7 @@ func TestImageUpdateWatcher_DisabledTriggersAreSkippedUntilEnabled(t *testing.T)
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: false}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
time.Sleep(30 * time.Millisecond)
@@ -332,7 +331,7 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{errors: []error{errors.New("registry unavailable")}}
settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -347,7 +346,7 @@ func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *test
releaseCh: releaseCh,
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Equal(t, 1, <-scanner.startedCh)
@@ -365,7 +364,7 @@ func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *test
func TestImageUpdateWatcher_RunNowReturnsCompletedResultInternal(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)
@@ -380,7 +379,7 @@ func TestImageUpdateWatcher_RunNowReturnsCompletedResultInternal(t *testing.T) {
func TestImageUpdateWatcher_RunNowReturnsContainedScanPanicInternal(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{panics: []bool{false, true}}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)
startImageUpdateWatcherForTestInternal(t, watcher)
// Wait for durable startup admission and the startup scan itself to finish,
// not just for the scanner to start; RunNow rejects a scan still running.
@@ -402,7 +401,7 @@ func TestImageUpdateWatcher_TriggeredScanRetriesAfterManualScanFinishes(t *testi
}
settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
// Let the startup scan finish so the gate is free.
@@ -462,7 +461,7 @@ func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *test
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true, eventWatcherEnabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, backfiller)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, backfiller)
startImageUpdateWatcherForTestInternal(t, watcher)
<-backfillStarted
@@ -495,7 +494,7 @@ func TestImageUpdateWatcher_BackfillRetriesContainedPanicInternal(t *testing.T)
}
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return backfiller.countInternal() == 2 }, time.Second, time.Millisecond)
@@ -521,7 +520,7 @@ func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {
}
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
startImageUpdateWatcherForTestInternal(t, watcher)
select {
@@ -547,7 +546,7 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.
}
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
ctx, cancel := context.WithCancel(t.Context())
errCh := make(chan error, 1)
go func() { errCh <- watcher.Start(ctx) }()
@@ -566,7 +565,7 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.
func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true, schedule: "* * * * * *"}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)
jobScheduler, err := NewJobScheduler(t.Context(), watcher.coordinator, time.UTC)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, jobScheduler.Stop(context.WithoutCancel(t.Context()))) })
@@ -601,7 +600,7 @@ func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsWatcherRunning(t *testin
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -614,7 +613,7 @@ func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsWatcherRunning(t *testin
func TestImageUpdateWatcher_SupervisorRestartsAfterCancellationInternal(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, &pollingSettingReaderFakeInternal{enabled: true}, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, &pollingSettingReaderFakeInternal{enabled: true}, bus.NewDockerEventBus(), nil)
supervisor := newWatcherSupervisorInternal(watcher)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
@@ -633,7 +632,7 @@ func TestImageUpdateWatcher_SupervisorRestartsAfterCancellationInternal(t *testi
func TestImageUpdateWatcher_RunNowWaitsForMetadataReadiness(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTest(t, scanner, settings, bus.NewDockerEventBus(), nil)
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
diff --git a/backend/pkg/scheduler/runs/activity.go b/backend/pkg/scheduler/runs/activity.go
index 3bdd04d89..0dbeac6c1 100644
--- a/backend/pkg/scheduler/runs/activity.go
+++ b/backend/pkg/scheduler/runs/activity.go
@@ -6,7 +6,6 @@ import (
"time"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/italypaleale/francis/actor"
kit "go.getarcane.app/kit/pkg"
)
@@ -22,27 +21,25 @@ func (q *Coordinator) associateActivity(run *scheduler.Run) {
}
}
-func (q *Coordinator) observeRun(ctx context.Context, run scheduler.Run) {
+func (q *Coordinator) observeRun(ctx context.Context, run scheduler.Run) bool {
if q.observer == nil {
- return
+ return false
}
- q.observerMu.Lock()
- defer q.observerMu.Unlock()
- q.syncActivity(ctx, run)
- q.signal()
+ defer q.observers.Lock(run.ID)()
+ return q.syncActivity(ctx, run)
}
-func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) {
+func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) bool {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var state scheduler.CoordinatorState
id := kit.SHA256Hex(run.EnvironmentID + "\x00" + run.JobID)
if err := q.service.GetState(ctx, coordinatorType, id, &state); err != nil {
- return
+ return true
}
version, pending := state.PendingActivitySync[run.ID]
if !pending {
- return
+ return false
}
current, err := q.Get(ctx, run.EnvironmentID, run.JobID, run.ID)
if err == nil {
@@ -53,60 +50,20 @@ func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) {
}
if err != nil {
slog.ErrorContext(ctx, "Job activity synchronization failed; will retry", "runId", run.ID, "error", err)
+ return true
}
+ return false
}
-func (q *Coordinator) retryActivitySync(ctx context.Context) {
- if q.observer == nil {
- return
- }
- cursor := ""
- for {
- page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
- if err != nil {
- slog.ErrorContext(ctx, "List pending job activities", "error", err)
- return
- }
- for _, entry := range page.States {
- var state scheduler.CoordinatorState
- if decodeErr := entry.Data.Decode(&state); decodeErr != nil {
- slog.ErrorContext(ctx, "Decode pending job activities", "error", decodeErr)
- continue
- }
- for runID := range state.PendingActivitySync {
- if ctx.Err() != nil {
- return
- }
- q.observerMu.Lock()
- q.syncActivity(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})
- q.observerMu.Unlock()
- }
- }
- cursor = page.AfterID()
- if cursor == "" {
- return
- }
- }
-}
-
-func (q *Coordinator) hasPendingActivitySync(ctx context.Context) bool {
- ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
- defer cancel()
- cursor := ""
- for {
- page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
- if err != nil {
- return true
- }
- for _, entry := range page.States {
- var state scheduler.CoordinatorState
- if entry.Data.Decode(&state) != nil || len(state.PendingActivitySync) > 0 {
+func (q *Coordinator) retryActivitySync(ctx context.Context, states []scheduler.CoordinatorState) bool {
+ pending := false
+ for _, state := range states {
+ for runID := range state.PendingActivitySync {
+ if ctx.Err() != nil {
return true
}
- }
- cursor = page.AfterID()
- if cursor == "" {
- return false
+ pending = q.observeRun(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID}) || pending
}
}
+ return pending
}
diff --git a/backend/pkg/scheduler/runs/activity_test.go b/backend/pkg/scheduler/runs/activity_test.go
index 8113b4c1c..5b9d948cc 100644
--- a/backend/pkg/scheduler/runs/activity_test.go
+++ b/backend/pkg/scheduler/runs/activity_test.go
@@ -3,6 +3,7 @@ package runs
import (
"context"
"errors"
+ "slices"
"sync/atomic"
"testing"
"time"
@@ -32,6 +33,18 @@ func (o *activityObserverFake) SyncRunActivity(context.Context, st.Run) error {
return nil
}
+func coordinatorStates(t *testing.T, q *Coordinator) []st.CoordinatorState {
+ t.Helper()
+ states, err := q.states(t.Context())
+ require.NoError(t, err)
+ return states
+}
+
+func pendingActivitySync(t *testing.T, q *Coordinator) bool {
+ t.Helper()
+ return slices.ContainsFunc(coordinatorStates(t, q), func(state st.CoordinatorState) bool { return len(state.PendingActivitySync) > 0 })
+}
+
func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {
q := newResolutionQueue(t)
observer := &activityObserverFake{}
@@ -56,10 +69,10 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {
persisted, getErr := q.Get(t.Context(), "0", run.JobID, run.ID)
return getErr == nil && persisted.Status == st.Succeeded
}, 5*time.Second, 10*time.Millisecond)
- require.True(t, q.hasPendingActivitySync(t.Context()))
+ require.True(t, pendingActivitySync(t, q))
observer.unavailable.Store(false)
- q.retryActivitySync(t.Context())
- require.False(t, q.hasPendingActivitySync(t.Context()))
+ require.False(t, q.retryActivitySync(t.Context(), coordinatorStates(t, q)))
+ require.False(t, pendingActivitySync(t, q))
require.Positive(t, observer.observed.Load())
duplicate, err := q.Submit(t.Context(), st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "manual"})
require.NoError(t, err)
diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.go
index 0f1560899..bb6d2333a 100644
--- a/backend/pkg/scheduler/runs/actors.go
+++ b/backend/pkg/scheduler/runs/actors.go
@@ -102,6 +102,14 @@ func (a *coordinatorActor) Invoke(ctx context.Context, method string, data actor
return nil, a.q.service.SetState(ctx, coordinatorType, a.id, state, nil)
case "repair":
return nil, a.flush(ctx, &state)
+ case "remove":
+ if state.Record.Schedule != "" || len(state.PendingActivitySync) > 0 || len(state.Record.Runs) > 0 || len(state.Record.Receipts) > 0 {
+ return false, nil
+ }
+ if err := a.q.service.DeleteState(ctx, coordinatorType, a.id); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return nil, err
+ }
+ return true, nil
case "import":
var command st.CoordinatorImport
if err := data.Decode(&command); err != nil {
@@ -352,23 +360,19 @@ func (a *coordinatorActor) Alarm(ctx context.Context, name string, data actor.En
// repair flushes coordinators so schedule alarms and dispatches are restored.
// Unless all is set, it skips records without unfinished runs so idle coordinators can deactivate.
-func (q *Coordinator) repair(ctx context.Context, all bool) (busy bool, err error) {
- records, err := q.Records(ctx)
- if err != nil {
- return false, err
- }
- unfinished := func(record st.QueueRecord) bool {
- return slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention })
+func (q *Coordinator) repair(ctx context.Context, states []st.CoordinatorState, all bool) (busy bool, err error) {
+ unfinished := func(state st.CoordinatorState) bool {
+ return slices.ContainsFunc(state.Record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention })
}
- for _, record := range records {
- if !all && !unfinished(record) {
+ for _, state := range states {
+ if !all && !unfinished(state) {
continue
}
- if _, invokeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {
+ if _, invokeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(state.Record.EnvironmentID+"\x00"+state.Record.JobID), "repair", nil); invokeErr != nil {
return false, invokeErr
}
}
- return slices.ContainsFunc(records, unfinished), nil
+ return slices.ContainsFunc(states, unfinished), nil
}
func (q *Coordinator) importLegacy(ctx context.Context) error {
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
index 978b6d7f3..4009abb70 100644
--- a/backend/pkg/scheduler/runs/actors_test.go
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -91,7 +91,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
run.Outcome.Targets = []st.TargetOutcome{{ID: "artifact", Status: st.Succeeded, RecoveryData: jsontext.Value(`{"backupId":"artifact"}`)}}
return nil
}))
- require.True(t, q.hasPendingActivitySync(t.Context()))
+ require.True(t, pendingActivitySync(t, q))
q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {
t.Error("startup must not execute jobs before activation")
return st.Outcome{Status: st.Failed}, nil
@@ -128,8 +128,8 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
require.Equal(t, 1, history.Total)
require.Equal(t, time.UTC, history.Runs[0].CreatedAt.Location())
observer.unavailable.Store(false)
- restored.retryActivitySync(t.Context())
- require.False(t, restored.hasPendingActivitySync(t.Context()))
+ require.False(t, restored.retryActivitySync(t.Context(), coordinatorStates(t, restored)))
+ require.False(t, pendingActivitySync(t, restored))
require.Positive(t, observer.observed.Load())
_, exists, err := store.Get(t.Context(), key)
require.NoError(t, err)
@@ -268,10 +268,10 @@ func TestReconciliationRepairsMissingAlarm(t *testing.T) {
record, err := q.ScheduleState(t.Context(), "repair")
require.NoError(t, err)
id := kit.SHA256Hex("0\x00repair")
- _, err = q.repair(t.Context(), true)
+ _, err = q.repair(t.Context(), coordinatorStates(t, q), true)
require.NoError(t, err)
require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
- _, err = q.repair(t.Context(), true)
+ _, err = q.repair(t.Context(), coordinatorStates(t, q), true)
require.NoError(t, err)
require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
after, err := q.ScheduleState(t.Context(), "repair")
@@ -358,7 +358,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
state.Dispatches[key] = intent
state.Revision++
require.NoError(t, q.service.SetState(t.Context(), coordinatorType, id, state, nil))
- _, err = q.repair(t.Context(), false)
+ _, err = q.repair(t.Context(), coordinatorStates(t, q), false)
require.NoError(t, err)
require.NoError(t, q.service.GetState(t.Context(), coordinatorType, id, &state))
require.Equal(t, original, state.Dispatches[key].JobID)
diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.go
index d86dc9e7d..dd0b0710e 100644
--- a/backend/pkg/scheduler/runs/coordinator.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -44,7 +44,7 @@ type Coordinator struct {
execute func(context.Context, st.Run) (st.Outcome, error)
reconcile func(context.Context, st.Run) (st.Outcome, error)
observer st.RunObserver
- observerMu sync.Mutex
+ observers utils.KeyedMutex
location *time.Location
}
@@ -54,8 +54,9 @@ type Coordinator struct {
// Cancellation of ctx affects admission only, not an accepted run's lifetime.
func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, error) {
q.mu.Lock()
- defer q.mu.Unlock()
- if q.stopping {
+ stopping := q.stopping
+ q.mu.Unlock()
+ if stopping {
return st.Run{}, errors.New("job queue is stopping")
}
request.EnvironmentID = cmp.Or(request.EnvironmentID, "0")
@@ -110,7 +111,9 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, e
RemoteOutcome: remoteOutcome,
}
q.associateActivity(&result)
+ fresh := result
err := q.mutate(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {
+ result = fresh
if request.Trigger == "scheduled" || request.Trigger == "recovery" {
record.LastEnqueuedAt = now
}
@@ -157,8 +160,8 @@ func separateTrigger(trigger string) bool {
// Checkpoint records the next due occurrence atomically with overdue recovery.
func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {
q.mu.Lock()
- defer q.mu.Unlock()
first := !q.checkpointed[jobID]
+ q.mu.Unlock()
recovered := false
var recoveredRun st.Run
err := q.mutate(ctx, "0", jobID, func(record *st.QueueRecord) error {
@@ -190,7 +193,9 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, ne
return nil
})
if err == nil {
+ q.mu.Lock()
q.checkpointed[jobID] = true
+ q.mu.Unlock()
if recovered {
q.observeRun(ctx, recoveredRun)
q.signal()
@@ -199,6 +204,17 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, ne
return err
}
+func (q *Coordinator) Remove(ctx context.Context, jobID string) error {
+ if err := q.Checkpoint(ctx, jobID, "", time.Time{}); err != nil {
+ return err
+ }
+ q.mu.Lock()
+ delete(q.checkpointed, jobID)
+ q.mu.Unlock()
+ _, err := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex("0\x00"+jobID), "remove", nil)
+ return err
+}
+
// Start validates persisted state on ctx and starts dispatching on appCtx. Repeated calls
// are harmless, but a stopped queue cannot be restarted; create a new instance.
func (q *Coordinator) Start(ctx, appCtx context.Context) error {
@@ -220,17 +236,15 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {
return fmt.Errorf("import legacy job history: %w", err)
}
// Fail startup if persisted state cannot be read. Never replace it with empty state.
- records, err := q.Records(ctx)
+ states, err := q.states(ctx)
if err != nil {
return fmt.Errorf("read startup job history: %w", err)
}
- for _, record := range records {
+ for index := range states {
+ record := &states[index].Record
changed := false
- for _, run := range record.Runs {
- if normalizeLegacyRun(&run) {
- changed = true
- break
- }
+ for i := range record.Runs {
+ changed = normalizeLegacyRun(&record.Runs[i]) || changed
}
if !changed {
continue
@@ -244,7 +258,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {
return fmt.Errorf("normalize startup job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)
}
}
- if pruneErr := q.prune(ctx); pruneErr != nil {
+ if pruneErr := q.prune(ctx, states); pruneErr != nil {
return fmt.Errorf("prune startup job history: %w", pruneErr)
}
if errErr := ctx.Err(); errErr != nil {
@@ -287,25 +301,16 @@ func (q *Coordinator) Stop(ctx context.Context) error {
func (q *Coordinator) dispatch(ctx context.Context) {
defer close(q.done)
- initialRetention := true
// Flush every coordinator once while enabled to restore schedule alarms; later passes only cover unfinished work.
fullRepair := true
nextRetention := time.Now()
for ctx.Err() == nil {
- q.retryActivitySync(ctx)
- if initialRetention || !time.Now().Before(nextRetention) {
- if err := q.prune(ctx); err != nil {
- if ctx.Err() == nil {
- slog.ErrorContext(ctx, "Job retention failed", "error", err)
- }
- } else {
- initialRetention = q.hasPendingActivitySync(ctx)
- }
- nextRetention = time.Now().Add(time.Hour)
- }
enabled := q.enabled.Load()
- busy, err := q.repair(ctx, fullRepair)
- pendingActivity := q.hasPendingActivitySync(ctx)
+ retain := !time.Now().Before(nextRetention)
+ busy, pendingActivity, pruned, err := q.pass(ctx, fullRepair, retain)
+ if retain {
+ nextRetention = time.Now().Add(kit.Ternary(pruned, time.Hour, 5*time.Second))
+ }
nextWake := time.Now().Add(kit.Ternary(busy || pendingActivity, 15*time.Second, coordinatorIdleWake))
if err != nil {
if ctx.Err() == nil {
@@ -336,6 +341,23 @@ func (q *Coordinator) dispatch(ctx context.Context) {
}
}
+func (q *Coordinator) pass(ctx context.Context, all, retain bool) (busy, pendingActivity, pruned bool, err error) {
+ states, err := q.states(ctx)
+ if err != nil {
+ return false, true, false, err
+ }
+ pendingActivity = q.retryActivitySync(ctx, states)
+ if retain {
+ pruneErr := q.prune(ctx, states)
+ if pruneErr != nil && ctx.Err() == nil {
+ slog.ErrorContext(ctx, "Job retention failed", "error", pruneErr)
+ }
+ pruned = pruneErr == nil
+ }
+ busy, err = q.repair(ctx, states, all)
+ return busy, pendingActivity, pruned, err
+}
+
func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) error {
// Keep the execution slot while retrying, but release it after a bounded backoff.
wait := time.Second
@@ -348,18 +370,21 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st
return ErrRunConflict
}
now := time.Now().UTC()
- current.Status = outcome.Status
- current.UpdatedAt = now
+ merged := st.Outcome{
+ Status: outcome.Status,
+ Message: outcome.Message,
+ ActivityID: cmp.Or(outcome.ActivityID, current.Outcome.ActivityID),
+ Targets: slices.Clone(outcome.Targets),
+ Steps: kit.Ternary(len(outcome.Steps) == 0, current.Outcome.Steps, outcome.Steps),
+ }
for _, target := range current.Outcome.Targets {
if !slices.ContainsFunc(outcome.Targets, func(updated st.TargetOutcome) bool { return updated.ID == target.ID }) {
- outcome.Targets = append(outcome.Targets, target)
+ merged.Targets = append(merged.Targets, target)
}
}
- outcome.ActivityID = cmp.Or(outcome.ActivityID, current.Outcome.ActivityID)
- if len(outcome.Steps) == 0 {
- outcome.Steps = current.Outcome.Steps
- }
- current.Outcome = outcome
+ current.Status = outcome.Status
+ current.UpdatedAt = now
+ current.Outcome = merged
if outcome.Status == st.Retrying || outcome.Status == st.Waiting {
delay := 5 * time.Second * time.Duration(1<<min(run.AttemptCount-1, 6))
delay = min(delay, 5*time.Minute)
@@ -381,7 +406,7 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st
if len(current.Attempts) > 0 {
last := &current.Attempts[len(current.Attempts)-1]
last.FinishedAt = &now
- last.Outcome = outcome
+ last.Outcome = merged
last.Outcome.Steps = nil
}
return nil
diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.go
index 6cbe0b069..a6c759b9b 100644
--- a/backend/pkg/scheduler/runs/coordinator_store.go
+++ b/backend/pkg/scheduler/runs/coordinator_store.go
@@ -6,6 +6,8 @@ import (
"encoding/json/v2"
"errors"
"fmt"
+ "maps"
+ "math/rand/v2"
"slices"
"sort"
"time"
@@ -16,6 +18,7 @@ import (
"github.com/italypaleale/francis/actor"
"go.getarcane.app/kit/pkg"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
)
@@ -24,7 +27,17 @@ const queuePrefix = "jobs."
func (q *Coordinator) mutate(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {
id := kit.SHA256Hex(environmentID + "\x00" + jobID)
q.runtime.NameActor(id, jobID+"@"+environmentID)
- for range 20 {
+ for attempt := range 20 {
+ if attempt > 0 {
+ delay := 5 * time.Millisecond << min(attempt-1, 5)
+ timer := time.NewTimer(delay/2 + rand.N(delay/2))
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return ctx.Err()
+ case <-timer.C:
+ }
+ }
var state st.CoordinatorState
err := q.service.GetState(ctx, coordinatorType, id, &state)
if err != nil && !errors.Is(err, actor.ErrStateNotFound) {
@@ -45,7 +58,7 @@ func (q *Coordinator) mutate(ctx context.Context, environmentID, jobID string, c
return nil
}
}
- return errors.New("job coordinator state is busy; retry mutation")
+ return common.Classify(common.ErrUnavailable, errors.New("job coordinator state is busy; retry mutation"))
}
func (q *Coordinator) prepareMutation(state *st.CoordinatorState, environmentID, jobID string, change func(*st.QueueRecord) error) error {
@@ -147,7 +160,19 @@ func pruneDispatches(state *st.CoordinatorState) {
}
func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
- records := []st.QueueRecord{}
+ states, err := q.states(ctx)
+ if err != nil {
+ return nil, err
+ }
+ records := make([]st.QueueRecord, 0, len(states))
+ for _, state := range states {
+ records = append(records, state.Record)
+ }
+ return records, nil
+}
+
+func (q *Coordinator) states(ctx context.Context) ([]st.CoordinatorState, error) {
+ states := []st.CoordinatorState{}
cursor := ""
for {
page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
@@ -161,11 +186,11 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
}
normalizeRecordTimes(&state.Record)
q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)
- records = append(records, state.Record)
+ states = append(states, state)
}
cursor = page.AfterID()
if cursor == "" {
- return records, nil
+ return states, nil
}
}
}
@@ -256,21 +281,23 @@ func (q *Coordinator) UpdateRun(ctx context.Context, run st.Run, change func(*st
return err
}
-func (q *Coordinator) prune(ctx context.Context) error {
- records, err := q.Records(ctx)
- if err != nil {
- return err
- }
+func (q *Coordinator) prune(ctx context.Context, states []st.CoordinatorState) error {
now := time.Now().UTC()
- for _, record := range records {
- if !pruneRuns(&record, now) {
+ for _, state := range states {
+ record := st.QueueRecord{Runs: slices.Clone(state.Record.Runs), Receipts: maps.Clone(state.Record.Receipts)}
+ if pruneRuns(&record, now) {
+ if mutateErr := q.mutate(ctx, state.Record.EnvironmentID, state.Record.JobID, func(current *st.QueueRecord) error {
+ pruneRuns(current, now)
+ return nil
+ }); mutateErr != nil {
+ return fmt.Errorf("prune job %q in environment %q: %w", state.Record.JobID, state.Record.EnvironmentID, mutateErr)
+ }
+ }
+ if state.Record.Schedule != "" || len(state.PendingActivitySync) > 0 || len(record.Runs) > 0 || len(record.Receipts) > 0 {
continue
}
- if mutateErr := q.mutate(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
- pruneRuns(current, now)
- return nil
- }); mutateErr != nil {
- return fmt.Errorf("prune job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)
+ if _, removeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(state.Record.EnvironmentID+"\x00"+state.Record.JobID), "remove", nil); removeErr != nil {
+ return fmt.Errorf("remove job %q in environment %q: %w", state.Record.JobID, state.Record.EnvironmentID, removeErr)
}
}
return nil
diff --git a/backend/pkg/scheduler/runs/coordinator_test.go b/backend/pkg/scheduler/runs/coordinator_test.go
new file mode 100644
index 000000000..8dbf84094
--- /dev/null
+++ b/backend/pkg/scheduler/runs/coordinator_test.go
@@ -0,0 +1,63 @@
+package runs
+
+import (
+ "context"
+ "sync"
+ "testing"
+
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/stretchr/testify/require"
+)
+
+type claimingObserverFake struct {
+ once sync.Once
+ target string
+ claim func(st.Run)
+ claimed bool
+}
+
+func (o *claimingObserverFake) ActivityID(run st.Run) string {
+ if o.target != "" && run.ID == o.target && run.Status == st.Queued {
+ o.once.Do(func() {
+ o.claimed = true
+ o.claim(run)
+ })
+ }
+ return ""
+}
+
+func (o *claimingObserverFake) SyncRunActivity(context.Context, st.Run) error { return nil }
+
+func TestSubmitRetryAfterClaimAppendsFreshRun(t *testing.T) {
+ q := newResolutionQueue(t)
+ observer := &claimingObserverFake{}
+ q.SetObserver(observer)
+ queued, err := q.Submit(t.Context(), st.Request{JobID: "coalesce", Trigger: "scheduled"})
+ require.NoError(t, err)
+ observer.target = queued.ID
+ observer.claim = func(run st.Run) {
+ claim, claimErr := q.claimExecution(t.Context(), st.ExecutionCommand{
+ EnvironmentID: run.EnvironmentID,
+ JobID: run.JobID,
+ RunID: run.ID,
+ Attempt: 1,
+ })
+ require.NoError(t, claimErr)
+ require.True(t, claim.Claimed)
+ }
+
+ submitted, err := q.Submit(t.Context(), st.Request{JobID: "coalesce", Trigger: "scheduled"})
+ require.NoError(t, err)
+ require.True(t, observer.claimed)
+ require.NotEqual(t, queued.ID, submitted.ID)
+ require.Equal(t, st.Queued, submitted.Status)
+
+ history, err := q.List(t.Context(), "0", "coalesce", 1, 10)
+ require.NoError(t, err)
+ require.Equal(t, 2, history.Total)
+ statuses := map[string]st.RunStatus{}
+ for _, run := range history.Runs {
+ statuses[run.ID] = run.Status
+ }
+ require.Equal(t, map[string]st.RunStatus{queued.ID: st.Running, submitted.ID: st.Queued}, statuses)
+}
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index 87105f39c..a8b3e7113 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -145,8 +145,8 @@ func (js *jobSchedulerInternal) StartScheduler(ctx context.Context) error {
_, watching := js.watchers[record.JobID]
js.mu.RUnlock()
if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) && !watching {
- if checkpointErr := js.coordinator.Checkpoint(ctx, record.JobID, "", time.Time{}); checkpointErr != nil {
- return checkpointErr
+ if removeErr := js.coordinator.Remove(ctx, record.JobID); removeErr != nil {
+ return removeErr
}
}
}
@@ -202,7 +202,7 @@ func (js *jobSchedulerInternal) RemoveJob(ctx context.Context, id string) {
started := js.started
js.mu.Unlock()
if started {
- if err := js.coordinator.Checkpoint(ctx, id, "", time.Time{}); err != nil {
+ if err := js.coordinator.Remove(ctx, id); err != nil {
slog.ErrorContext(ctx, "Unschedule failed", "job", id, "error", err)
}
}
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index 2dfd9e26e..52684364d 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -8,6 +8,7 @@ import (
"time"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
"github.com/stretchr/testify/require"
)
@@ -303,10 +304,8 @@ func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "0 0 * * * *", polling.Schedule)
require.True(t, nextPolling.Equal(polling.NextRun))
- removed, err := js.coordinator.ScheduleState(t.Context(), "removed-job")
- require.NoError(t, err)
- require.Empty(t, removed.Schedule)
- require.True(t, removed.NextRun.IsZero())
+ _, err = js.coordinator.ScheduleState(t.Context(), "removed-job")
+ require.ErrorIs(t, err, actor.ErrStateNotFound)
select {
case <-executed:
case <-time.After(5 * time.Second):
@@ -447,9 +446,8 @@ func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
js.RemoveJob(t.Context(), job.Name())
require.False(t, js.HasJob(job.Name()))
- record, err := js.coordinator.ScheduleState(t.Context(), job.Name())
- require.NoError(t, err)
- require.True(t, record.NextRun.IsZero())
+ _, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.ErrorIs(t, err, actor.ErrStateNotFound)
}
func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {
diff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.go
index 8c6c49243..5a27b942e 100644
--- a/backend/pkg/utils/francis/provider.go
+++ b/backend/pkg/utils/francis/provider.go
@@ -72,6 +72,9 @@ func StoreURL(databaseURL string) (string, error) {
if !sqliteutil.HasPragma(query, "synchronous") {
query.Add("_pragma", "synchronous(NORMAL)")
}
+ if !sqliteutil.HasPragma(query, "journal_mode") {
+ query.Add("_pragma", "journal_mode(WAL)")
+ }
return "file:" + (&url.URL{Path: StorePath(databasePath)}).EscapedPath() + "?" + query.Encode(), nil
case strings.HasPrefix(databaseURL, "postgres"):
return databaseURL, nil
diff --git a/backend/pkg/utils/francis/provider_test.go b/backend/pkg/utils/francis/provider_test.go
index dd5ecef1a..e76893935 100644
--- a/backend/pkg/utils/francis/provider_test.go
+++ b/backend/pkg/utils/francis/provider_test.go
@@ -18,15 +18,16 @@ func TestMigrateLegacyStore(t *testing.T) {
databaseURL := "file:" + mainPath + "?_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate"
storeURL, err := StoreURL(databaseURL)
require.NoError(t, err)
- const defaults = "_pragma=busy_timeout%2810000%29&_pragma=synchronous%28NORMAL%29"
- require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+defaults+"&_txlock=immediate", storeURL)
+ const tuned = "_pragma=busy_timeout%2810000%29&_pragma=synchronous%28NORMAL%29"
+ const defaults = tuned + "&_pragma=journal_mode%28WAL%29"
+ require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+tuned+"&_txlock=immediate", storeURL)
for input, expected := range map[string]string{
"file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db?" + defaults + "&_txlock=immediate",
"file:/data/arcane%2Edb": "file:/data/arcane.francis.db?" + defaults,
"file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db?" + defaults,
"file:data/arcane": "file:data/arcane.francis.db?" + defaults,
- "file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29",
- "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%2810000%29",
+ "file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29&_pragma=journal_mode%28WAL%29",
+ "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%2810000%29&_pragma=journal_mode%28WAL%29",
"file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%2810000%29",
"postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",
} {