Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
refactor: move automation logic to use actors (#3458)
Kyle Mendell3bdb4e67ff48
diff --git a/Justfile b/Justfileindex 086a56934..250e3d6e0 100644--- a/Justfile+++ b/Justfile@@ -316,21 +316,25 @@ _lint-js:@just _lint-tests@just _lint-email-templates-# Lint Go backend+# Build golangci-lint with the custom linters enabled by the shared config[group('quality')]-_lint-backend:+_build-golangci-lint:golangci-lint custom++# Lint Go backend+[group('quality')]+_lint-backend: _build-golangci-lintcd backend && ../.bin/golangci-lint-custom run -c ../.github/.golangci.yml ./...# Lint Go CLI[group('quality')]-_lint-cli:- cd cli && golangci-lint run -c ../.github/.golangci.yml ./...+_lint-cli: _build-golangci-lint+ cd cli && ../.bin/golangci-lint-custom run -c ../.github/.golangci.yml ./...# Lint Types[group('quality')]-_lint-types:- cd types && golangci-lint run -c ../.github/.golangci.yml ./...+_lint-types: _build-golangci-lint+ cd types && ../.bin/golangci-lint-custom run -c ../.github/.golangci.yml ./...# Lint edge tunnel protobuf definitions.[group('quality')]diff --git a/backend/api/handlers/activities_test.go b/backend/api/handlers/activities_test.goindex 93665c272..8111fb1ab 100644--- a/backend/api/handlers/activities_test.go+++ b/backend/api/handlers/activities_test.go@@ -83,7 +83,7 @@ func TestActivityHandlerClearHistoryDeletesSelectedEnvironmentOnlyInternal(t *tefunc TestActivityHandlerClearHistoryProxiesRemoteEnvironmentInternal(t *testing.T) {ctx := context.Background()db := setupActivityHandlerTestDBInternal(t)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)token := "remote-token"@@ -190,7 +190,7 @@ func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testingdb := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)activityService := services.NewActivityService(db, settingsService)@@ -236,7 +236,7 @@ func TestActivityHandlerStreamAllReusesRemoteEnvironmentAfterInitialPollInternaldb := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)activityService := services.NewActivityService(db, settingsService)@@ -302,7 +302,7 @@ func TestActivityHandlerStreamAllRemoteFailureEmitsErrorAndKeepsStreamingInternadb := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)activityService := services.NewActivityService(db, settingsService)@@ -339,7 +339,7 @@ func TestActivityHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *testdb := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)activityService := services.NewActivityService(db, settingsService)_, err = activityService.StartActivity(ctx, services.StartActivityRequest{EnvironmentID: "0", Type: models.ActivityTypeResourceAction})diff --git a/backend/api/handlers/dashboard_test.go b/backend/api/handlers/dashboard_test.goindex 9aca9e1b8..6212340d7 100644--- a/backend/api/handlers/dashboard_test.go+++ b/backend/api/handlers/dashboard_test.go@@ -34,7 +34,7 @@ func setupDashboardHandlerTestDB(t *testing.T) (*database.DB, *services.Settingsrequire.NoError(t, db.AutoMigrate(&models.ApiKey{}, &models.Environment{}, &models.ImageUpdateRecord{}, &models.Project{}, &models.SettingVariable{}))databaseDB := &database.DB{DB: db}- settingsSvc, err := services.NewSettingsService(context.Background(), databaseDB)+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), databaseDB)require.NoError(t, err)return databaseDB, settingsSvc@@ -187,7 +187,7 @@ func TestDashboardHandlerStreamAllEmitsRemoteSnapshotInternal(t *testing.T) {db := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)token := "remote-token"@@ -231,7 +231,7 @@ func TestDashboardHandlerStreamAllLegacyAgentComposesSnapshotFromGranularEndpoindb := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)token := "remote-token"@@ -283,7 +283,7 @@ func TestDashboardHandlerStreamAllLegacyAgent404EmitsIncompatibleErrorInternal(tdb := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)token := "remote-token"@@ -326,7 +326,7 @@ func TestDashboardHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *tesdb := setupActivityHandlerTestDBInternal(t)limitStreamTestDBToSingleConnInternal(t, db)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)allowedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {@@ -370,7 +370,7 @@ func TestDashboardHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *tesfunc TestDashboardHandlerRemoteFetchReusesLoadedEnvironmentInternal(t *testing.T) {ctx := context.Background()db := setupActivityHandlerTestDBInternal(t)- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {diff --git a/backend/api/handlers/remenv_handlers_test.go b/backend/api/handlers/remenv_handlers_test.goindex a218db91c..526d13d57 100644--- a/backend/api/handlers/remenv_handlers_test.go+++ b/backend/api/handlers/remenv_handlers_test.go@@ -141,7 +141,7 @@ func TestSettingsHandler_GetSettings_RemoteFiltersNonAdminVisibility(t *testing.defer server.Close()db := setupActivityHandlerTestDBInternal(t)- settingsService, err := services.NewSettingsService(context.Background(), db)+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), db)require.NoError(t, err)handler := &SettingsHandler{settingsService: settingsService,@@ -167,7 +167,7 @@ func TestSettingsHandler_GetSettings_RemotePreservesAdminResponse(t *testing.T)defer server.Close()db := setupActivityHandlerTestDBInternal(t)- settingsService, err := services.NewSettingsService(context.Background(), db)+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), db)require.NoError(t, err)handler := &SettingsHandler{settingsService: settingsService,diff --git a/backend/api/handlers/settings_actor_test.go b/backend/api/handlers/settings_actor_test.gonew file mode 100644index 000000000..a06452800--- /dev/null+++ b/backend/api/handlers/settings_actor_test.go@@ -0,0 +1,32 @@+package handlers++import (+ "context"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*services.SettingsService, error) {+ t.Helper()+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)+ require.NoError(t, err)+ effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, executor.Stop(stopCtx))+ require.NoError(t, effects.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+ })+ return services.NewSettingsService(ctx, db, executor, effects)+}diff --git a/backend/api/handlers/settings_test.go b/backend/api/handlers/settings_test.goindex c749b03c1..c4b4ca100 100644--- a/backend/api/handlers/settings_test.go+++ b/backend/api/handlers/settings_test.go@@ -12,7 +12,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models"- "github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"apitypes "github.com/getarcaneapp/arcane/types/v2/settings""github.com/libtnb/sqlite"@@ -76,7 +75,7 @@ func TestSettingsHandler_UpdateLocalEnvironment_RejectsUnreadableProjectsDirectorequire.NoError(t, err)require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))- settingsSvc, err := services.NewSettingsService(ctx, &database.DB{DB: db})+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, &database.DB{DB: db})require.NoError(t, err)originalDir := settingsSvc.GetSettingsConfig().ProjectsDirectory.Valuediff --git a/backend/api/handlers/variables_test.go b/backend/api/handlers/variables_test.goindex 996da4165..e9e3aaa4a 100644--- a/backend/api/handlers/variables_test.go+++ b/backend/api/handlers/variables_test.go@@ -123,7 +123,7 @@ func setupVariableHandlerServicesInternal(t *testing.T) (*services.VariableServi&models.SettingVariable{},))databaseDB := &database.DB{DB: databaseConnection}- settingsService, err := services.NewSettingsService(context.Background(), databaseDB)+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), databaseDB)require.NoError(t, err)require.NoError(t, settingsService.UpdateSetting(context.Background(), "projectsDirectory", t.TempDir()))environmentService := services.NewEnvironmentService(databaseDB, nil, nil, nil, settingsService, nil)diff --git a/backend/go.mod b/backend/go.modindex 9dd1464a6..661102656 100644--- a/backend/go.mod+++ b/backend/go.mod@@ -10,6 +10,7 @@ replace (require (emperror.dev/emperror v0.33.0emperror.dev/errors v0.8.1+ github.com/anthdm/hollywood v1.0.5github.com/aws/aws-sdk-go-v2/config v1.32.31github.com/aws/aws-sdk-go-v2/credentials v1.19.30github.com/aws/aws-sdk-go-v2/service/ecr v1.60.0@@ -81,6 +82,7 @@ require (connectrpc.com/connect v1.19.2 // indirectdario.cat/mergo v1.0.2 // indirectgithub.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect+ github.com/DataDog/gostackparse v0.7.0 // indirectgithub.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679 // indirectgithub.com/DmitriyVTitov/size v1.5.0 // indirectgithub.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect@@ -218,6 +220,7 @@ require (github.com/xhit/go-str2duration/v2 v2.1.0 // indirectgithub.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirectgithub.com/yusufpapurcu/wmi v1.2.4 // indirect+ github.com/zeebo/xxh3 v1.0.2 // indirectgo.opentelemetry.io/auto/sdk v1.2.1 // indirectgo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirectgo.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 // indirectdiff --git a/backend/go.sum b/backend/go.sumindex 4263599cf..f615a547b 100644--- a/backend/go.sum+++ b/backend/go.sum@@ -11,6 +11,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgithub.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=+github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=+github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679 h1:qNT7R4qrN+5u5ajSbqSW1opHP4LA8lzA+ASyw5MQZjs=github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=@@ -30,6 +32,8 @@ github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA=github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=+github.com/anthdm/hollywood v1.0.5 h1:SuCTVRRFqx0MZ4E0RijHl4+Xt56PF0C7aqYKguBU6j8=+github.com/anthdm/hollywood v1.0.5/go.mod h1:wU4WxIRVs++E2PuiVXc8dA2An/Wlom4AhzwQ7e3tDzI=github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=@@ -549,6 +553,10 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUgithub.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=go.getarcane.app/builds v0.3.0 h1:6fCgHy6PaEr3LpCtiF1iCdT4nuV3D3xvUHnV+E/0cqk=go.getarcane.app/builds v0.3.0/go.mod h1:yZJDSh9XX/sgZlwx7lghHEucgZeQ74Gr/Fx+XdR2bdk=go.getarcane.app/docker/convert v0.1.0 h1:prlIOs6dMx6CdnetsJFZ6ak0vcistM3DcH5lrZiFHBU=diff --git a/backend/internal/actors/actor.go b/backend/internal/actors/actor.gonew file mode 100644index 000000000..22cc40ef8--- /dev/null+++ b/backend/internal/actors/actor.go@@ -0,0 +1,171 @@+package actors++import (+ "context"+ "sync"++ "emperror.dev/errors"+ "github.com/anthdm/hollywood/actor"+)++var errActorRequestNoResponseInternal = errors.Sentinel("actor request completed without response")++// Context exposes the operations available while handling one actor message+// without leaking Hollywood into actor consumers.+type Context struct {+ context context.Context+ engine *actor.Engine+ target *actor.PID+ respond func(any)+ workers *sync.WaitGroup+}++// Context returns the current receiver lifetime.+func (c *Context) Context() context.Context {+ return c.context+}++// Respond sends exactly one response to the current request.+func (c *Context) Respond(message any) {+ c.respond(message)+}++// Behavior defines the application behavior hosted by one shared actor.+type Behavior struct {+ Initialize func(*Context)+ Handle func(*Context, any)+ Cleanup func(*Context)+}++// Binding connects an ingress to an actor for the duration of its lifetime.+type Binding interface {+ bindInternal(handle *handleInternal)+ unbindInternal(handle *handleInternal)+ replayInternal()+}++// Actor owns one Hollywood process, its ingress bindings, and all workers+// started from its behavior.+type Actor struct {+ handle *handleInternal+ bindings []Binding+ unbindOnce sync.Once+}++type behaviorActorInternal struct {+ behavior Behavior+ bindings []Binding+ lifetime lifetimeInternal+ workers sync.WaitGroup+}++// NewActor starts one application behavior on the shared actor runtime.+func NewActor(ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, producer func() Behavior, bindings ...Binding) (*Actor, error) {+ if ctx == nil {+ return nil, errors.New("actor context unavailable")+ }+ if runtime == nil {+ return nil, errors.New("actor runtime unavailable")+ }+ if producer == nil {+ return nil, errors.New("actor behavior unavailable")+ }++ handle, err := runtime.startInternal(+ func() actor.Receiver {+ return &behaviorActorInternal{behavior: producer(), bindings: bindings}+ },+ kind,+ id,+ maxRestarts,+ actor.WithContext(ctx),+ )+ if err != nil {+ return nil, err+ }++ applicationActor := &Actor{handle: handle, bindings: bindings}+ for _, binding := range bindings {+ binding.bindInternal(handle)+ }+ return applicationActor, nil+}++// Send queues a message for the actor.+func (a *Actor) Send(message any) error {+ return a.handle.Send(message)+}++// Done closes after the actor and its workers stop.+func (a *Actor) Done() <-chan struct{} {+ return a.handle.Done()+}++// Stop disconnects ingress, drains the mailbox, cancels workers, and joins+// them before the actor terminates.+func (a *Actor) Stop(ctx context.Context) error {+ a.unbindOnce.Do(func() {+ for _, binding := range a.bindings {+ binding.unbindInternal(a.handle)+ }+ })+ return a.handle.Stop(ctx)+}++// Request sends one typed request and consumes its response.+func Request[K comparable, Q, R any](ctx context.Context, a *Actor, request Message[K, Q]) (Message[K, R], error) {+ if a == nil {+ var zero Message[K, R]+ return zero, errors.New("actor unavailable")+ }+ return requestHandleInternal[K, Q, R](ctx, a.handle, request)+}++func (a *behaviorActorInternal) Receive(ctx *actor.Context) {+ switch message := ctx.Message().(type) {+ case actor.Initialized:+ return+ case actor.Started:+ lifetimeCtx := a.lifetime.startInternal(ctx.Context())+ for _, binding := range a.bindings {+ binding.replayInternal()+ }+ if a.behavior.Initialize != nil {+ a.behavior.Initialize(a.contextInternal(lifetimeCtx, ctx))+ }+ case actor.Stopped:+ a.lifetime.stopInternal()+ if a.behavior.Cleanup != nil {+ a.behavior.Cleanup(a.contextInternal(ctx.Context(), ctx))+ }+ // Strict joining preserves actor serialization across receiver restarts:+ // a replacement must never run while an old worker can still own state.+ // Behavior workers are therefore required to honor their lifetime context.+ a.workers.Wait()+ default:+ if request, ok := message.(requestInternal); ok {+ defer request.reply.Resolve(requestResultInternal{err: errActorRequestNoResponseInternal})+ if a.behavior.Handle != nil {+ requestCtx := a.contextInternal(a.lifetime.contextInternal(), ctx)+ requestCtx.respond = func(message any) {+ request.reply.Resolve(requestResultInternal{message: message})+ }+ a.behavior.Handle(requestCtx, request.message)+ }+ return+ }+ if a.behavior.Handle != nil {+ a.behavior.Handle(a.contextInternal(a.lifetime.contextInternal(), ctx), message)+ }+ }+}++func (a *behaviorActorInternal) contextInternal(lifetimeCtx context.Context, ctx *actor.Context) *Context {+ return &Context{+ context: lifetimeCtx,+ engine: ctx.Engine(),+ target: ctx.PID().CloneVT(),+ respond: ctx.Respond,+ workers: &a.workers,+ }+}diff --git a/backend/internal/actors/actor_test.go b/backend/internal/actors/actor_test.gonew file mode 100644index 000000000..0738b5bd5--- /dev/null+++ b/backend/internal/actors/actor_test.go@@ -0,0 +1,71 @@+package actors++import (+ "context"+ "sync/atomic"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++type actorBindingFakeInternal struct {+ binds atomic.Int32+ unbinds atomic.Int32+}++func (b *actorBindingFakeInternal) bindInternal(*handleInternal) {+ b.binds.Add(1)+}++func (b *actorBindingFakeInternal) unbindInternal(*handleInternal) {+ b.unbinds.Add(1)+}++func (b *actorBindingFakeInternal) replayInternal() {}++func TestActorOwnsLifecycleBindingsAndWorkerJoinInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ binding := &actorBindingFakeInternal{}+ initialized := make(chan struct{})+ workerExited := make(chan struct{})+ applicationActor, err := NewActor(+ t.Context(),+ runtime,+ "actor-test",+ "lifecycle",+ 1,+ func() Behavior {+ return Behavior{+ Initialize: func(ctx *Context) {+ close(initialized)+ Worker[uint8, NoPayload]{+ Actor: ctx,+ WorkContext: ctx.Context(),+ Label: "actor lifecycle test worker",+ CompletionKind: 1,+ }.Start(func(workerCtx context.Context) (NoPayload, error) {+ defer close(workerExited)+ <-workerCtx.Done()+ return NoPayload{}, workerCtx.Err()+ })+ },+ }+ },+ binding,+ )+ require.NoError(t, err)+ require.Equal(t, int32(1), binding.binds.Load())+ <-initialized++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, applicationActor.Stop(stopCtx))+ <-workerExited+ require.Equal(t, int32(1), binding.unbinds.Load())+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/executor.go b/backend/internal/actors/executor.gonew file mode 100644index 000000000..6f1c7e443--- /dev/null+++ b/backend/internal/actors/executor.go@@ -0,0 +1,179 @@+package actors++import (+ "context"+ "sync"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+)++// Executor serializes heterogeneous blocking tasks through one actor mailbox.+// A task must not synchronously submit another task to the same Executor.+type Executor struct {+ actor *Actor+}++// Task is one submitted executor operation whose result may be awaited with a+// context independent from the operation's own lifetime.+type Task[T any] struct {+ result *Promise[executorResultInternal[T]]+ actor *Actor+}++type executorTaskInternal interface {+ executeInternal(actorCtx context.Context)+}++type executorResultInternal[T any] struct {+ value T+ err error+}++type executorTaskValueInternal[T any] struct {+ ctx context.Context+ label string+ work func(context.Context) (T, error)+ after func(T, error)+ result *Promise[executorResultInternal[T]]+ started *sync.Once+}++// NewExecutor creates a serial task executor on the shared actor runtime.+func NewExecutor(ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32) (*Executor, error) {+ if runtime == nil {+ return nil, errors.New("actor runtime unavailable")+ }+ if ctx == nil {+ return nil, errors.New("executor context unavailable")+ }+ applicationActor, err := NewActor(+ ctx,+ runtime,+ kind,+ id,+ maxRestarts,+ func() Behavior {+ return Behavior{Handle: receiveExecutorInternal}+ },+ )+ if err != nil {+ return nil, err+ }+ return &Executor{actor: applicationActor}, nil+}++// Execute queues work, waits for its typed result, and runs after only once the+// result is available to the caller. Work and after are both panic-contained.+func Execute[T any](ctx context.Context, executor *Executor, label string, work func(context.Context) (T, error), after func(T, error)) (T, error) {+ var zero T+ task, err := Submit(ctx, executor, label, work, after)+ if err != nil {+ return zero, err+ }+ return task.Wait(ctx)+}++// Submit queues work synchronously and returns a handle that can be awaited+// separately. This is useful for terminal cleanup that must stay queued even+// when the shutdown wait context expires.+func Submit[T any](ctx context.Context, executor *Executor, label string, work func(context.Context) (T, error), after func(T, error)) (*Task[T], error) {+ if executor == nil || executor.actor == nil {+ return nil, errors.New("actor executor unavailable")+ }+ if ctx == nil {+ return nil, errors.New("task context unavailable")+ }+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ if work == nil {+ return nil, errors.New("actor task unavailable")+ }++ result := NewPromise[executorResultInternal[T]]()+ if err := executor.actor.Send(executorTaskValueInternal[T]{+ ctx: ctx,+ label: label,+ work: work,+ after: after,+ result: result,+ started: &sync.Once{},+ }); err != nil {+ return nil, err+ }+ return &Task[T]{result: result, actor: executor.actor}, nil+}++// Wait waits for the submitted task result without changing the task lifetime.+func (t *Task[T]) Wait(ctx context.Context) (T, error) {+ var zero T+ if t == nil || t.result == nil || t.actor == nil {+ return zero, errors.New("actor task unavailable")+ }+ if ctx == nil {+ return zero, errors.New("task wait context unavailable")+ }+ completed, err := awaitInternal(ctx, t.actor.handle, t.result, errors.New("actor executor stopped"))+ if err != nil {+ return zero, err+ }+ return completed.value, completed.err+}++// Stop drains the executor mailbox and waits for the actor to terminate.+func (e *Executor) Stop(ctx context.Context) error {+ if e == nil || e.actor == nil {+ return nil+ }+ return e.actor.Stop(ctx)+}++// Done closes after the executor actor terminates.+func (e *Executor) Done() <-chan struct{} {+ if e == nil || e.actor == nil {+ return nil+ }+ return e.actor.Done()+}++func receiveExecutorInternal(ctx *Context, rawMessage any) {+ task, ok := rawMessage.(executorTaskInternal)+ if !ok {+ return+ }+ task.executeInternal(ctx.Context())+}++func (t executorTaskValueInternal[T]) executeInternal(actorCtx context.Context) {+ if t.result == nil || t.started == nil {+ return+ }+ t.started.Do(func() {+ workCtx, cancel := context.WithCancel(actorCtx)+ stopTaskCancel := context.AfterFunc(t.ctx, cancel) //nolint:contextcheck // task execution combines caller cancellation with the actor lifetime.+ defer cancel()+ defer stopTaskCancel()++ var (+ value T+ err error+ )+ func() {+ defer utils.RecoverToError(&err, t.label)+ if err = t.ctx.Err(); err == nil {+ value, err = t.work(workCtx)+ }+ }()+ t.result.Resolve(executorResultInternal[T]{value: value, err: err})++ if t.after == nil {+ return+ }+ func() {+ var afterErr error+ defer utils.RecoverToError(&afterErr, t.label+" completion")+ t.after(value, err)+ }()+ })+}diff --git a/backend/internal/actors/executor_test.go b/backend/internal/actors/executor_test.gonew file mode 100644index 000000000..10cd1a2e9--- /dev/null+++ b/backend/internal/actors/executor_test.go@@ -0,0 +1,109 @@+package actors++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++func TestExecutorSerializesHeterogeneousTasksInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ executor, err := NewExecutor(t.Context(), runtime, "executor-test", "serial", 1)+ require.NoError(t, err)++ firstStarted := make(chan struct{})+ releaseFirst := make(chan struct{})+ firstResult := make(chan error, 1)+ go func() {+ _, executeErr := Execute(t.Context(), executor, "first", func(context.Context) (int, error) {+ close(firstStarted)+ <-releaseFirst+ return 1, nil+ }, nil)+ firstResult <- executeErr+ }()+ <-firstStarted++ secondStarted := make(chan struct{})+ secondResult := make(chan string, 1)+ go func() {+ value, executeErr := Execute(t.Context(), executor, "second", func(context.Context) (string, error) {+ close(secondStarted)+ return "two", nil+ }, nil)+ if executeErr != nil {+ secondResult <- executeErr.Error()+ return+ }+ secondResult <- value+ }()++ require.Never(t, func() bool { return len(secondStarted) > 0 }, 20*time.Millisecond, time.Millisecond)+ close(releaseFirst)+ require.NoError(t, <-firstResult)+ require.Equal(t, "two", <-secondResult)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, executor.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestExecutorPublishesResultBeforeCompletionCallbackInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ executor, err := NewExecutor(t.Context(), runtime, "executor-test", "completion-order", 1)+ require.NoError(t, err)++ afterStarted := make(chan struct{})+ releaseAfter := make(chan struct{})+ result := make(chan int, 1)+ go func() {+ value, _ := Execute(t.Context(), executor, "ordered", func(context.Context) (int, error) {+ return 42, nil+ }, func(int, error) {+ close(afterStarted)+ <-releaseAfter+ })+ result <- value+ }()++ <-afterStarted+ require.Equal(t, 42, <-result)+ close(releaseAfter)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, executor.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestExecutorContainsTaskPanicAndContinuesInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ executor, err := NewExecutor(t.Context(), runtime, "executor-test", "panic", 1)+ require.NoError(t, err)++ _, err = Execute(t.Context(), executor, "panic task", func(context.Context) (NoPayload, error) {+ panic("deliberate executor panic")+ }, nil)+ require.ErrorContains(t, err, "panic task panicked")++ value, err := Execute(t.Context(), executor, "following task", func(context.Context) (int, error) {+ return 7, nil+ }, nil)+ require.NoError(t, err)+ require.Equal(t, 7, value)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, executor.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/gate.go b/backend/internal/actors/gate.gonew file mode 100644index 000000000..50015a7cc--- /dev/null+++ b/backend/internal/actors/gate.go@@ -0,0 +1,161 @@+package actors++import (+ "context"+ "sync"++ "emperror.dev/errors"+)++type gateOperationInternal uint8++const (+ gateAcquireInternal gateOperationInternal = iota+ gateReleaseInternal+)++// AdmissionKey namespaces a string identifier shared by the application gate.+type AdmissionKey struct {+ Scope string+ ID string+}++// Gate admits at most one active lease for each key through one actor-owned map.+type Gate[K comparable] struct {+ actor *Actor+}++// Lease represents one admitted key. Release is idempotent and confirms that+// the actor processed the completion before returning.+type Lease[K comparable] struct {+ gate *Gate[K]+ key K+ once sync.Once+}++type gateCommandInternal[K comparable] struct {+ key K+ reply *Promise[bool]+}++type gateActorInternal[K comparable] struct {+ active map[K]struct{}+}++// NewGate creates a keyed admission gate on the shared actor runtime.+func NewGate[K comparable](ctx context.Context, runtime *Runtime, kind, id string) (*Gate[K], error) {+ if ctx == nil {+ return nil, errors.New("gate context unavailable")+ }+ if runtime == nil {+ return nil, errors.New("actor runtime unavailable")+ }++ active := make(map[K]struct{})+ applicationActor, err := NewActor(+ ctx,+ runtime,+ kind,+ id,+ // Shared gates tolerate transient receiver failures without forgetting+ // active leases, whose map intentionally survives receiver replacement.+ 3,+ func() Behavior {+ // The map intentionally survives receiver replacement so an actor restart+ // cannot forget leases held by work that is still running.+ state := &gateActorInternal[K]{active: active}+ return Behavior{Handle: state.receiveInternal}+ },+ )+ if err != nil {+ return nil, err+ }+ return &Gate[K]{actor: applicationActor}, nil+}++// TryAcquire returns a lease when key is idle and refuses immediately when it+// is already active. Actor failure is reported separately from contention.+func (g *Gate[K]) TryAcquire(ctx context.Context, key K) (*Lease[K], bool, error) {+ if ctx == nil {+ return nil, false, errors.New("gate request context unavailable")+ }+ if err := ctx.Err(); err != nil {+ return nil, false, err+ }+ if g == nil || g.actor == nil {+ return nil, false, errors.New("actor gate unavailable")+ }++ reply := NewPromise[bool]()+ if err := g.actor.Send(Message[gateOperationInternal, gateCommandInternal[K]]{+ Kind: gateAcquireInternal,+ Value: gateCommandInternal[K]{+ key: key,+ reply: reply,+ },+ }); err != nil {+ return nil, false, err+ }++ // Admission must resolve after it is queued; abandoning an admitted reply on+ // caller cancellation would leak the key without a lease to release it.+ admitted, err := awaitInternal(context.WithoutCancel(ctx), g.actor.handle, reply, errors.New("actor gate stopped"))+ if err != nil {+ return nil, false, err+ }+ if !admitted {+ return nil, false, nil+ }+ return &Lease[K]{gate: g, key: key}, true, nil+}++// Release returns the lease to its gate exactly once.+func (l *Lease[K]) Release() {+ if l == nil || l.gate == nil {+ return+ }+ l.once.Do(func() {+ reply := NewPromise[bool]()+ if err := l.gate.actor.Send(Message[gateOperationInternal, gateCommandInternal[K]]{+ Kind: gateReleaseInternal,+ Value: gateCommandInternal[K]{+ key: l.key,+ reply: reply,+ },+ }); err != nil {+ return+ }+ _, _ = awaitInternal(context.Background(), l.gate.actor.handle, reply, errors.New("actor gate stopped"))+ })+}++// Stop drains the gate mailbox and waits for the actor to terminate.+func (g *Gate[K]) Stop(ctx context.Context) error {+ if g == nil || g.actor == nil {+ return nil+ }+ return g.actor.Stop(ctx)+}++func (g *gateActorInternal[K]) receiveInternal(_ *Context, rawMessage any) {+ message, ok := rawMessage.(Message[gateOperationInternal, gateCommandInternal[K]])+ if !ok || message.Value.reply == nil {+ return+ }+ // Resolve a refusal if future gate logic panics before producing a result.+ // Promise resolution is idempotent, so successful paths win first.+ defer message.Value.reply.Resolve(false)++ switch message.Kind {+ case gateAcquireInternal:+ if _, active := g.active[message.Value.key]; active {+ message.Value.reply.Resolve(false)+ return+ }+ g.active[message.Value.key] = struct{}{}+ message.Value.reply.Resolve(true)+ case gateReleaseInternal:+ delete(g.active, message.Value.key)+ message.Value.reply.Resolve(true)+ }+}diff --git a/backend/internal/actors/gate_test.go b/backend/internal/actors/gate_test.gonew file mode 100644index 000000000..b15f214e8--- /dev/null+++ b/backend/internal/actors/gate_test.go@@ -0,0 +1,67 @@+package actors++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++func TestGateSerializesEachKeyAndReleasesIndependentlyInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ gate, err := NewGate[string](t.Context(), runtime, "gate-test", "keyed")+ require.NoError(t, err)++ first, admitted, err := gate.TryAcquire(t.Context(), "first")+ require.NoError(t, err)+ require.True(t, admitted)++ _, admitted, err = gate.TryAcquire(t.Context(), "first")+ require.NoError(t, err)+ require.False(t, admitted)++ second, admitted, err := gate.TryAcquire(t.Context(), "second")+ require.NoError(t, err)+ require.True(t, admitted)+ second.Release()++ first.Release()+ first.Release()+ replacement, admitted, err := gate.TryAcquire(t.Context(), "first")+ require.NoError(t, err)+ require.True(t, admitted)+ replacement.Release()++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestGateReturnsContextCancellationWithoutLeakingKeyInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ gate, err := NewGate[string](t.Context(), runtime, "gate-test", "cancellation")+ require.NoError(t, err)++ canceled, cancel := context.WithCancel(context.Background())+ cancel()+ _, admitted, err := gate.TryAcquire(canceled, "key")+ require.ErrorIs(t, err, context.Canceled)+ require.False(t, admitted)++ lease, admitted, err := gate.TryAcquire(t.Context(), "key")+ require.NoError(t, err)+ require.True(t, admitted)+ lease.Release()++ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)+ defer stopCancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/message.go b/backend/internal/actors/message.gonew file mode 100644index 000000000..3429b9948--- /dev/null+++ b/backend/internal/actors/message.go@@ -0,0 +1,375 @@+package actors++import (+ "context"+ "fmt"+ "sync"+ "sync/atomic"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+)++// NoPayload is used by typed actor messages that only carry a kind.+type NoPayload struct{}++// Message is the shared typed envelope for actor commands, events, and replies.+// Worker completion handlers must call Acknowledge before returning.+type Message[K comparable, V any] struct {+ Kind K+ Value V+ Err error+ ack chan struct{}+}++// Acknowledge confirms that an actor processed a worker completion.+func (m Message[K, V]) Acknowledge() {+ if m.ack == nil {+ return+ }+ select {+ case m.ack <- struct{}{}:+ default:+ }+}++// Promise is an idempotent one-result completion primitive.+type Promise[T any] struct {+ once sync.Once+ done chan T+}++// NewPromise creates a promise that accepts exactly one result.+func NewPromise[T any]() *Promise[T] {+ return &Promise[T]{done: make(chan T, 1)}+}++// Resolve publishes value once; later calls are harmless.+func (p *Promise[T]) Resolve(value T) {+ if p == nil {+ return+ }+ p.once.Do(func() { p.done <- value })+}++// Done returns the channel that receives the promised result.+func (p *Promise[T]) Done() <-chan T {+ if p == nil {+ return nil+ }+ return p.done+}++// awaitInternal waits for a promise while preserving a result that races with caller+// cancellation or actor termination.+func awaitInternal[T any](ctx context.Context, handle *handleInternal, promise *Promise[T], stoppedErr error) (T, error) {+ var zero T+ if ctx == nil || handle == nil || promise == nil {+ return zero, errors.New("actor wait unavailable")+ }+ select {+ case result := <-promise.Done():+ return result, nil+ case <-ctx.Done():+ select {+ case result := <-promise.Done():+ return result, nil+ default:+ return zero, ctx.Err()+ }+ case <-handle.Done():+ select {+ case result := <-promise.Done():+ return result, nil+ default:+ return zero, stoppedErr+ }+ }+}++// lifetimeInternal is an actor-receiver lifecycle context.+type lifetimeInternal struct {+ mu sync.Mutex+ ctx context.Context+ cancel context.CancelFunc+}++func (l *lifetimeInternal) startInternal(parent context.Context) context.Context {+ l.mu.Lock()+ defer l.mu.Unlock()+ if l.cancel != nil {+ l.cancel()+ }+ l.ctx, l.cancel = context.WithCancel(parent)+ return l.ctx+}++func (l *lifetimeInternal) contextInternal() context.Context {+ l.mu.Lock()+ defer l.mu.Unlock()+ return l.ctx+}++func (l *lifetimeInternal) stopInternal() {+ l.mu.Lock()+ defer l.mu.Unlock()+ if l.cancel != nil {+ l.cancel()+ l.cancel = nil+ }+}++// Ingress coalesces concurrent producers into one pending mailbox signal while+// retaining the newest value. Bind may happen after Send, which lets services+// accept startup triggers before their actor exists.+type Ingress[K comparable, V any] struct {+ mu sync.Mutex+ kind K+ value V+ generation uint64+ acknowledged uint64+ queued bool+ handle *handleInternal+}++// NewIngress creates an unbound coalescing actor ingress.+func NewIngress[K comparable, V any](kind K) *Ingress[K, V] {+ return &Ingress[K, V]{kind: kind}+}++// Bind delivers future signals to handle and flushes one signal queued before startup.+func (i *Ingress[K, V]) bindInternal(handle *handleInternal) {+ if i == nil {+ return+ }+ i.mu.Lock()+ defer i.mu.Unlock()+ i.handle = handle+ if i.queued && handle != nil {+ _ = handle.Send(Message[K, NoPayload]{Kind: i.kind})+ }+}++// Unbind stops delivery to handle without discarding a pending signal.+func (i *Ingress[K, V]) unbindInternal(handle *handleInternal) {+ if i == nil {+ return+ }+ i.mu.Lock()+ defer i.mu.Unlock()+ if i.handle == handle {+ i.handle = nil+ }+}++func (i *Ingress[K, V]) replayInternal() {+ if i == nil {+ return+ }+ i.mu.Lock()+ defer i.mu.Unlock()+ if i.generation == i.acknowledged || i.queued || i.handle == nil {+ return+ }+ i.queued = true+ _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})+}++// Send records value and queues at most one mailbox signal.+func (i *Ingress[K, V]) Send(value V) {+ if i == nil {+ return+ }+ i.mu.Lock()+ defer i.mu.Unlock()+ i.value = value+ i.generation+++ if i.queued {+ return+ }+ i.queued = true+ if i.handle != nil {+ _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})+ }+}++// Begin returns the newest value and its generation while retaining the work+// as pending until Acknowledge commits it.+func (i *Ingress[K, V]) Begin() (V, uint64) {+ i.mu.Lock()+ defer i.mu.Unlock()+ i.queued = false+ return i.value, i.generation+}++// Acknowledge commits work through generation and replays a newer pending value.+func (i *Ingress[K, V]) Acknowledge(generation uint64) {+ i.mu.Lock()+ defer i.mu.Unlock()+ if generation > i.acknowledged {+ i.acknowledged = generation+ }+ if i.generation == i.acknowledged || i.queued || i.handle == nil {+ return+ }+ i.queued = true+ _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})+}++// Take returns and immediately acknowledges the newest value.+func (i *Ingress[K, V]) Take() V {+ value, generation := i.Begin()+ i.Acknowledge(generation)+ return value+}++// Pending reports whether an unacknowledged value remains.+func (i *Ingress[K, V]) Pending() bool {+ i.mu.Lock()+ defer i.mu.Unlock()+ return i.generation != i.acknowledged+}++// Latest returns the newest value without changing admission state.+func (i *Ingress[K, V]) Latest() V {+ i.mu.Lock()+ defer i.mu.Unlock()+ return i.value+}++// Timer sends generation-fenced messages to an actor. It is actor-owned and+// must be stopped when the receiver handles actor.Stopped.+type Timer[K comparable] struct {+ timer *time.Timer+ generation uint64+}++var timerGenerationInternal atomic.Uint64++// Reset replaces the active timer and returns its generation.+func (t *Timer[K]) Reset(ctx *Context, kind K, delay time.Duration) uint64 {+ if t.timer != nil {+ t.timer.Stop()+ }+ generation := timerGenerationInternal.Add(1)+ t.generation = generation+ engine := ctx.engine+ target := ctx.target+ actorCtx := ctx.Context()+ t.timer = time.AfterFunc(delay, func() {+ if actorCtx.Err() == nil {+ engine.Send(target, Message[K, uint64]{Kind: kind, Value: generation})+ }+ })+ return generation+}++// Current reports whether generation belongs to the latest Reset call.+func (t *Timer[K]) Current(generation uint64) bool {+ return generation == t.generation+}++// Stop cancels the timer if one is active.+func (t *Timer[K]) Stop() {+ t.generation = timerGenerationInternal.Add(1)+ if t.timer != nil {+ t.timer.Stop()+ t.timer = nil+ }+}++// requestHandleInternal sends a typed request through Arcane's promise envelope.+type requestInternal struct {+ message any+ reply *Promise[requestResultInternal]+}++type requestResultInternal struct {+ message any+ err error+}++func requestHandleInternal[K comparable, Q, R any](ctx context.Context, handle *handleInternal, request Message[K, Q]) (Message[K, R], error) {+ var zero Message[K, R]+ if ctx == nil || handle == nil {+ return zero, errors.New("actor request target unavailable")+ }+ reply := NewPromise[requestResultInternal]()+ if err := handle.Send(requestInternal{message: request, reply: reply}); err != nil {+ return zero, err+ }+ result, err := awaitInternal(ctx, handle, reply, errors.New("actor stopped"))+ if err != nil {+ return zero, err+ }+ if result.err != nil {+ return zero, result.err+ }+ typed, ok := result.message.(Message[K, R])+ if !ok {+ return zero, fmt.Errorf("unexpected actor response type %T", result.message)+ }+ return typed, nil+}++// Worker describes blocking work owned by an actor but executed outside Receive.+type Worker[K comparable, V any] struct {+ Actor *Context+ WorkContext context.Context+ Label string+ CompletionKind K+ PanicValue V+ RetryDelay time.Duration+ ActorStopped func(V, error)+}++// Start contains panics, joins the work to WaitGroup, and posts one typed+// completion back to the actor. Its completion handler must call Acknowledge.+func (w Worker[K, V]) Start(work func(context.Context) (V, error)) {+ actorCtx := w.Actor.Context()+ engine := w.Actor.engine+ target := w.Actor.target+ workerCtx, cancel := context.WithCancel(w.WorkContext)+ stopActorCancel := context.AfterFunc(actorCtx, cancel)+ w.Actor.workers.Go(func() {+ var (+ value = w.PanicValue+ err error+ )+ defer cancel()+ defer stopActorCancel()+ defer func() {+ if actorCtx.Err() == nil {+ ack := make(chan struct{}, 1)+ engine.Send(target, Message[K, V]{Kind: w.CompletionKind, Value: value, Err: err, ack: ack})+ select {+ case <-ack:+ return+ case <-actorCtx.Done():+ }+ }+ if w.ActorStopped != nil {+ w.ActorStopped(value, err)+ }+ }()+ for {+ value = w.PanicValue+ err = nil+ func() {+ defer utils.RecoverToError(&err, w.Label)+ value, err = work(workerCtx)+ }()+ if err == nil || w.RetryDelay <= 0 || workerCtx.Err() != nil {+ return+ }+ timer := time.NewTimer(w.RetryDelay)+ select {+ case <-workerCtx.Done():+ timer.Stop()+ return+ case <-timer.C:+ }+ }+ })+}diff --git a/backend/internal/actors/message_test.go b/backend/internal/actors/message_test.gonew file mode 100644index 000000000..d930b1050--- /dev/null+++ b/backend/internal/actors/message_test.go@@ -0,0 +1,415 @@+package actors++import (+ "context"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/anthdm/hollywood/actor"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++type messageKindInternal uint8++const (+ ingressMessageInternal messageKindInternal = iota+ timerStartMessageInternal+ timerElapsedMessageInternal+ requestMessageInternal+ responseMessageInternal+ workerCompletedMessageInternal+)++type ingressReceiverInternal struct {+ ingress *Ingress[messageKindInternal, int]+ values chan<- int+}++func (r *ingressReceiverInternal) Receive(ctx *actor.Context) {+ if message, ok := ctx.Message().(Message[messageKindInternal, NoPayload]); ok && message.Kind == ingressMessageInternal {+ r.values <- r.ingress.Take()+ }+}++func TestIngressCoalescesBeforeBindingAndRetainsLatestValueInternal(t *testing.T) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ require.NoError(t, err)+ ingress := NewIngress[messageKindInternal, int](ingressMessageInternal)+ values := make(chan int, 2)+ pid := engine.Spawn(+ func() actor.Receiver { return &ingressReceiverInternal{ingress: ingress, values: values} },+ "ingress-test",+ actor.WithID("coalesced"),+ actor.WithMaxRestarts(1),+ )++ ingress.Send(1)+ ingress.Send(2)+ handle := &handleInternal{engine: engine, pid: pid, done: make(chan struct{})}+ ingress.bindInternal(handle)+ require.Equal(t, 2, <-values)++ ingress.Send(3)+ require.Equal(t, 3, <-values)+ ingress.unbindInternal(handle)+ <-engine.Poison(pid).Done()+}++type timerReceiverInternal struct {+ timer Timer[messageKindInternal]+ elapsed chan<- uint64+}++func (r *timerReceiverInternal) Receive(ctx *actor.Context) {+ switch message := ctx.Message().(type) {+ case Message[messageKindInternal, NoPayload]:+ if message.Kind == timerStartMessageInternal {+ actorCtx := &Context{+ context: ctx.Context(),+ engine: ctx.Engine(),+ target: ctx.PID().CloneVT(),+ respond: ctx.Respond,+ workers: &sync.WaitGroup{},+ }+ r.timer.Reset(actorCtx, timerElapsedMessageInternal, time.Hour)+ r.timer.Reset(actorCtx, timerElapsedMessageInternal, time.Millisecond)+ }+ case Message[messageKindInternal, uint64]:+ if message.Kind == timerElapsedMessageInternal && r.timer.Current(message.Value) {+ r.elapsed <- message.Value+ }+ case actor.Stopped:+ r.timer.Stop()+ }+}++func TestTimerOnlyDeliversLatestGenerationInternal(t *testing.T) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ require.NoError(t, err)+ elapsed := make(chan uint64, 2)+ pid := engine.Spawn(+ func() actor.Receiver { return &timerReceiverInternal{elapsed: elapsed} },+ "timer-test",+ actor.WithID("latest"),+ actor.WithMaxRestarts(1),+ )+ engine.Send(pid, Message[messageKindInternal, NoPayload]{Kind: timerStartMessageInternal})+ require.Equal(t, uint64(2), <-elapsed)+ require.Never(t, func() bool { return len(elapsed) > 0 }, 20*time.Millisecond, time.Millisecond)+ <-engine.Poison(pid).Done()+}++func TestRequestReturnsTypedResponseInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ applicationActor, err := NewActor(+ t.Context(),+ runtime,+ "request-test",+ "typed",+ 1,+ func() Behavior {+ return Behavior{Handle: func(ctx *Context, rawMessage any) {+ if message, ok := rawMessage.(Message[messageKindInternal, int]); ok && message.Kind == requestMessageInternal {+ ctx.Respond(Message[messageKindInternal, string]{Kind: responseMessageInternal, Value: "ok"})+ }+ }}+ },+ )+ require.NoError(t, err)+ for range 10 {+ response, requestErr := Request[messageKindInternal, int, string](t.Context(), applicationActor, Message[messageKindInternal, int]{Kind: requestMessageInternal, Value: 1})+ require.NoError(t, requestErr)+ require.Equal(t, responseMessageInternal, response.Kind)+ require.Equal(t, "ok", response.Value)+ }+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, applicationActor.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestRequestFailsWhenHandlerReturnsWithoutResponseInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ applicationActor, err := NewActor(+ t.Context(),+ runtime,+ "request-test",+ "missing-response",+ 1,+ func() Behavior {+ return Behavior{Handle: func(*Context, any) {}}+ },+ )+ require.NoError(t, err)++ _, err = Request[messageKindInternal, int, string](context.Background(), applicationActor, Message[messageKindInternal, int]{Kind: requestMessageInternal})+ require.ErrorContains(t, err, "without response")++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, applicationActor.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++type workerReceiverInternal struct {+ errors chan<- error+ values chan<- string+}++func (r workerReceiverInternal) Receive(ctx *actor.Context) {+ if message, ok := ctx.Message().(Message[messageKindInternal, string]); ok && message.Kind == workerCompletedMessageInternal {+ defer message.Acknowledge()+ if r.errors != nil {+ r.errors <- message.Err+ }+ if r.values != nil {+ r.values <- message.Value+ }+ }+}++func TestWorkerContainsPanicAndDeliversCompletionInternal(t *testing.T) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ require.NoError(t, err)+ workerErrors := make(chan error, 1)+ workerValues := make(chan string, 1)+ pid := engine.Spawn(+ func() actor.Receiver { return workerReceiverInternal{errors: workerErrors, values: workerValues} },+ "worker-test",+ actor.WithID("panic"),+ actor.WithMaxRestarts(1),+ )++ var wg sync.WaitGroup+ Worker[messageKindInternal, string]{+ Actor: &Context{+ context: context.Background(),+ engine: engine,+ target: pid,+ workers: &wg,+ },+ WorkContext: context.Background(),+ Label: "deliberate worker test",+ CompletionKind: workerCompletedMessageInternal,+ PanicValue: "panic fallback",+ }.Start(func(context.Context) (string, error) {+ panic("deliberate worker panic")+ })++ require.ErrorContains(t, <-workerErrors, "deliberate worker test panicked")+ require.Equal(t, "panic fallback", <-workerValues)+ wg.Wait()+ <-engine.Poison(pid).Done()+}++func TestWorkerRetriesContainedPanicInternal(t *testing.T) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ require.NoError(t, err)+ workerErrors := make(chan error, 1)+ workerValues := make(chan string, 1)+ pid := engine.Spawn(+ func() actor.Receiver { return workerReceiverInternal{errors: workerErrors, values: workerValues} },+ "worker-test",+ actor.WithID("retry"),+ actor.WithMaxRestarts(1),+ )++ var (+ wg sync.WaitGroup+ attempts int+ )+ Worker[messageKindInternal, string]{+ Actor: &Context{+ context: context.Background(),+ engine: engine,+ target: pid,+ workers: &wg,+ },+ WorkContext: context.Background(),+ Label: "retrying worker test",+ CompletionKind: workerCompletedMessageInternal,+ PanicValue: "panic fallback",+ RetryDelay: time.Millisecond,+ }.Start(func(context.Context) (string, error) {+ attempts+++ if attempts == 1 {+ panic("deliberate retryable panic")+ }+ return "completed", nil+ })++ require.NoError(t, <-workerErrors)+ require.Equal(t, "completed", <-workerValues)+ require.Equal(t, 2, attempts)+ wg.Wait()+ <-engine.Poison(pid).Done()+}++type lifetimeReceiverInternal struct {+ lifetime lifetimeInternal+ started chan<- context.Context+}++func (r *lifetimeReceiverInternal) Receive(ctx *actor.Context) {+ switch ctx.Message().(type) {+ case actor.Started:+ r.started <- r.lifetime.startInternal(ctx.Context())+ case panicActorMessageInternal:+ panic("deliberate lifetime restart")+ case actor.Stopped:+ r.lifetime.stopInternal()+ }+}++func TestLifetimeCancelsOldReceiverBeforeHollywoodRestartInternal(t *testing.T) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ require.NoError(t, err)+ started := make(chan context.Context, 2)+ pid := engine.Spawn(+ func() actor.Receiver { return &lifetimeReceiverInternal{started: started} },+ "lifetime-test",+ actor.WithID("restart"),+ actor.WithMaxRestarts(1),+ actor.WithRestartDelay(0),+ )++ first := <-started+ engine.Send(pid, panicActorMessageInternal{})+ require.Eventually(t, func() bool { return first.Err() != nil }, time.Second, time.Millisecond)+ second := <-started+ require.NoError(t, second.Err())+ <-engine.Poison(pid).Done()+}++func TestWorkerResolvesFallbackWhenActorStopsBeforeCompletionInternal(t *testing.T) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ require.NoError(t, err)+ pid := engine.Spawn(+ func() actor.Receiver { return workerReceiverInternal{errors: make(chan error, 1)} },+ "worker-test",+ actor.WithID("stopped"),+ actor.WithMaxRestarts(1),+ )++ actorCtx, stopActor := context.WithCancel(context.Background())+ result := NewPromise[error]()+ var wg sync.WaitGroup+ Worker[messageKindInternal, string]{+ Actor: &Context{+ context: actorCtx,+ engine: engine,+ target: pid,+ workers: &wg,+ },+ WorkContext: context.Background(),+ Label: "stopped worker test",+ CompletionKind: workerCompletedMessageInternal,+ ActorStopped: func(_ string, err error) {+ result.Resolve(err)+ },+ }.Start(func(ctx context.Context) (string, error) {+ <-ctx.Done()+ return "", ctx.Err()+ })++ stopActor()+ require.ErrorIs(t, <-result.Done(), context.Canceled)+ wg.Wait()+ <-engine.Poison(pid).Done()+}++type noAcknowledgementReceiverInternal struct {+ received chan<- struct{}+}++func (r noAcknowledgementReceiverInternal) Receive(ctx *actor.Context) {+ if message, ok := ctx.Message().(Message[messageKindInternal, string]); ok && message.Kind == workerCompletedMessageInternal {+ r.received <- struct{}{}+ }+}++func TestWorkerWithoutAcknowledgementUnblocksWhenActorLifetimeEndsInternal(t *testing.T) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ require.NoError(t, err)+ received := make(chan struct{}, 1)+ pid := engine.Spawn(+ func() actor.Receiver { return noAcknowledgementReceiverInternal{received: received} },+ "worker-test",+ actor.WithID("missing-ack"),+ )+ actorCtx, stopActor := context.WithCancel(context.Background())+ fallback := make(chan error, 1)+ var workers sync.WaitGroup+ Worker[messageKindInternal, string]{+ Actor: &Context{+ context: actorCtx,+ engine: engine,+ target: pid,+ workers: &workers,+ },+ WorkContext: actorCtx,+ Label: "missing acknowledgement worker",+ CompletionKind: workerCompletedMessageInternal,+ ActorStopped: func(_ string, err error) {+ fallback <- err+ },+ }.Start(func(context.Context) (string, error) {+ return "done", nil+ })++ <-received+ stopActor()+ require.NoError(t, <-fallback)+ require.NoError(t, utils.WaitGroup(t.Context(), &workers))+ <-engine.Poison(pid).Done()+}++func TestIngressReplaysUnacknowledgedValueAfterActorRestartInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ ingress := NewIngress[messageKindInternal, int](ingressMessageInternal)+ processed := make(chan int, 1)+ var attempts atomic.Int32+ applicationActor, err := NewActor(+ t.Context(),+ runtime,+ "ingress-test",+ "restart-replay",+ 2,+ func() Behavior {+ return Behavior{Handle: func(_ *Context, rawMessage any) {+ message, ok := rawMessage.(Message[messageKindInternal, NoPayload])+ if !ok || message.Kind != ingressMessageInternal {+ return+ }+ value, generation := ingress.Begin()+ if attempts.Add(1) == 1 {+ panic("restart after ingress begin")+ }+ ingress.Acknowledge(generation)+ processed <- value+ }}+ },+ ingress,+ )+ require.NoError(t, err)++ ingress.Send(42)+ require.Equal(t, 42, <-processed)+ require.False(t, ingress.Pending())+ require.Equal(t, int32(2), attempts.Load())++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, applicationActor.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/resource.go b/backend/internal/actors/resource.gonew file mode 100644index 000000000..08f526751--- /dev/null+++ b/backend/internal/actors/resource.go@@ -0,0 +1,128 @@+package actors++import (+ "context"+ "log/slog"+ "sync/atomic"++ "emperror.dev/errors"+)++// ErrResourceStopped reports work submitted after terminal resource shutdown.+var ErrResourceStopped = errors.New("actor resource stopped")++// Resource serializes the lifecycle and use of one replaceable value through+// an actor executor. Its value is never accessed outside that mailbox, and its+// zero value represents the absence of a resource.+type Resource[T comparable] struct {+ executor *Executor+ value T+ terminal atomic.Bool+ stop func(T) error+}++// NewResource creates an actor-owned replaceable resource.+func NewResource[T comparable](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, stop func(T) error) (*Resource[T], error) {+ if stop == nil {+ return nil, errors.New("resource stop function unavailable")+ }+ executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)+ if err != nil {+ return nil, err+ }+ return &Resource[T]{executor: executor, stop: stop}, nil+}++// Restart stops the current value before building and publishing its replacement.+// A non-zero failed build is cleaned up.+func (r *Resource[T]) Restart(ctx context.Context, label string, build func(context.Context) (T, error)) error {+ if build == nil {+ return errors.New("resource build function unavailable")+ }+ if r.terminal.Load() {+ return ErrResourceStopped+ }+ _, err := Execute(ctx, r.executor, label, func(workCtx context.Context) (NoPayload, error) {+ if r.terminal.Load() {+ return NoPayload{}, ErrResourceStopped+ }+ stopErr := r.clearInternal()+ next, buildErr := build(workCtx)+ if buildErr != nil {+ var zero T+ if next != zero {+ buildErr = errors.Combine(buildErr, r.stop(next))+ }+ return NoPayload{}, errors.Combine(stopErr, buildErr)+ }+ r.value = next+ if stopErr != nil {+ slog.WarnContext(workCtx, "failed to stop replaced actor resource", "label", label, "error", stopErr)+ }+ return NoPayload{}, nil+ }, nil)+ return err+}++// Do serializes work with Restart and Stop. Work is skipped after terminal stop.+func (r *Resource[T]) Do(ctx context.Context, label string, work func(context.Context, T) error) error {+ if work == nil {+ return errors.New("resource work function unavailable")+ }+ if r.terminal.Load() {+ return ErrResourceStopped+ }+ _, err := Execute(ctx, r.executor, label, func(workCtx context.Context) (NoPayload, error) {+ if r.terminal.Load() {+ return NoPayload{}, ErrResourceStopped+ }+ return NoPayload{}, work(workCtx, r.value)+ }, nil)+ return err+}++// Clear stops and forgets the current value while keeping the resource reusable.+func (r *Resource[T]) Clear(ctx context.Context, label string) error {+ if r.terminal.Load() {+ return ErrResourceStopped+ }+ _, err := Execute(ctx, r.executor, label, func(context.Context) (NoPayload, error) {+ if r.terminal.Load() {+ return NoPayload{}, ErrResourceStopped+ }+ return NoPayload{}, r.clearInternal()+ }, nil)+ return err+}++// Stop permanently fences replacement, stops the current value, and joins the actor.+func (r *Resource[T]) Stop(ctx context.Context) error {+ if ctx == nil {+ return errors.New("resource stop context unavailable")+ }+ r.terminal.Store(true)+ task, submitErr := Submit(context.WithoutCancel(ctx), r.executor, "stop actor resource", func(context.Context) (NoPayload, error) {+ return NoPayload{}, r.clearInternal()+ }, nil)+ var clearErr error+ if submitErr == nil {+ _, clearErr = task.Wait(ctx)+ }+ actorErr := r.executor.Stop(ctx)+ if submitErr != nil {+ // Submit only fails after the executor can no longer own the value, so+ // terminal cleanup must stop it directly instead of leaking it.+ clearErr = r.clearInternal()+ }+ return errors.Combine(submitErr, clearErr, actorErr)+}++func (r *Resource[T]) clearInternal() error {+ var zero T+ if r.value == zero {+ return nil+ }+ current := r.value+ r.value = zero+ return r.stop(current)+}diff --git a/backend/internal/actors/resource_test.go b/backend/internal/actors/resource_test.gonew file mode 100644index 000000000..081ba426e--- /dev/null+++ b/backend/internal/actors/resource_test.go@@ -0,0 +1,99 @@+package actors++import (+ "context"+ "errors"+ "sync"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++func TestResourceSerializesRestartWorkAndStopInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ var (+ mu sync.Mutex+ stopped []int+ )+ resource, err := NewResource(t.Context(), runtime, "resource-test", "lifecycle", 1, func(value int) error {+ mu.Lock()+ stopped = append(stopped, value)+ mu.Unlock()+ return nil+ })+ require.NoError(t, err)+ require.NoError(t, resource.Restart(t.Context(), "start first", func(context.Context) (int, error) { return 1, nil }))+ require.NoError(t, resource.Restart(t.Context(), "start second", func(context.Context) (int, error) { return 2, nil }))++ var current int+ require.NoError(t, resource.Do(t.Context(), "read current", func(_ context.Context, value int) error {+ current = value+ return nil+ }))+ require.Equal(t, 2, current)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, resource.Stop(stopCtx))+ mu.Lock()+ require.Equal(t, []int{1, 2}, stopped)+ mu.Unlock()+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestResourceFailedReplacementCleansPartialValueInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ var stopped []int+ resource, err := NewResource(t.Context(), runtime, "resource-test", "failed-replacement", 1, func(value int) error {+ stopped = append(stopped, value)+ return nil+ })+ require.NoError(t, err)+ require.NoError(t, resource.Restart(t.Context(), "start", func(context.Context) (int, error) { return 1, nil }))+ require.Error(t, resource.Restart(t.Context(), "replace", func(context.Context) (int, error) {+ return 2, errors.New("build failed")+ }))+ require.Equal(t, []int{1, 2}, stopped)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, resource.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestResourceCanceledStopStillCleansAndFencesInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ stopped := make(chan struct{})+ resource, err := NewResource(t.Context(), runtime, "resource-test", "canceled-stop", 1, func(int) error {+ close(stopped)+ return nil+ })+ require.NoError(t, err)+ require.NoError(t, resource.Restart(t.Context(), "start", func(context.Context) (int, error) { return 1, nil }))++ canceled, cancel := context.WithCancel(context.Background())+ cancel()+ require.ErrorIs(t, resource.Stop(canceled), context.Canceled)+ select {+ case <-stopped:+ case <-time.After(time.Second):+ t.Fatal("resource cleanup was abandoned after stop wait cancellation")+ }+ require.ErrorIs(t, resource.Restart(t.Context(), "restart", func(context.Context) (int, error) { return 2, nil }), ErrResourceStopped)++ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)+ defer stopCancel()+ require.NoError(t, resource.executor.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/runner.go b/backend/internal/actors/runner.gonew file mode 100644index 000000000..c454145ce--- /dev/null+++ b/backend/internal/actors/runner.go@@ -0,0 +1,84 @@+package actors++import (+ "context"++ "emperror.dev/errors"+)++// Runner owns one long-lived background function and joins it before stopping.+type Runner struct {+ actor *Actor+}++type runnerStateInternal struct {+ run func(context.Context) error+ label string+}++type runnerEventInternal uint8++const runnerExitedInternal runnerEventInternal = iota++// NewRunner starts a long-lived function on the shared actor runtime.+func NewRunner(ctx context.Context, runtime *Runtime, kind, id, label string, maxRestarts int32, run func(context.Context) error) (*Runner, error) {+ if ctx == nil {+ return nil, errors.New("runner context unavailable")+ }+ if runtime == nil {+ return nil, errors.New("actor runtime unavailable")+ }+ if run == nil {+ return nil, errors.New("runner function unavailable")+ }++ applicationActor, err := NewActor(+ ctx,+ runtime,+ kind,+ id,+ maxRestarts,+ func() Behavior {+ state := &runnerStateInternal{run: run, label: label}+ return Behavior{+ Initialize: state.initializeInternal,+ Handle: state.receiveInternal,+ }+ },+ )+ if err != nil {+ return nil, err+ }+ return &Runner{actor: applicationActor}, nil+}++// Stop cancels the background function and waits for it and its actor to exit.+func (r *Runner) Stop(ctx context.Context) error {+ if r == nil || r.actor == nil {+ return nil+ }+ return r.actor.Stop(ctx)+}++func (r *runnerStateInternal) initializeInternal(ctx *Context) {+ Worker[runnerEventInternal, NoPayload]{+ Actor: ctx,+ WorkContext: ctx.Context(),+ Label: r.label,+ CompletionKind: runnerExitedInternal,+ }.Start(func(workerCtx context.Context) (NoPayload, error) {+ return NoPayload{}, r.run(workerCtx)+ })+}++func (r *runnerStateInternal) receiveInternal(_ *Context, rawMessage any) {+ message, ok := rawMessage.(Message[runnerEventInternal, NoPayload])+ if !ok {+ return+ }+ defer message.Acknowledge()+ if message.Err != nil {+ panic(message.Err)+ }+ panic(errors.New(r.label + " exited unexpectedly"))+}diff --git a/backend/internal/actors/runner_test.go b/backend/internal/actors/runner_test.gonew file mode 100644index 000000000..9da4cab18--- /dev/null+++ b/backend/internal/actors/runner_test.go@@ -0,0 +1,41 @@+package actors++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++func TestRunnerCancelsAndJoinsBackgroundFunctionInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ started := make(chan struct{})+ exited := make(chan struct{})+ runner, err := NewRunner(t.Context(), runtime, "runner-test", "lifecycle", "test runner", 1, func(ctx context.Context) error {+ close(started)+ <-ctx.Done()+ close(exited)+ return nil+ })+ require.NoError(t, err)+ select {+ case <-started:+ case <-time.After(time.Second):+ t.Fatal("runner did not start")+ }++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, runner.Stop(stopCtx))+ select {+ case <-exited:+ default:+ t.Fatal("runner stop returned before background function exited")+ }+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/runtime.go b/backend/internal/actors/runtime.gonew file mode 100644index 000000000..f8c9c5714--- /dev/null+++ b/backend/internal/actors/runtime.go@@ -0,0 +1,290 @@+// Package actors owns Arcane's in-process actor runtime and its shared safety rails.+package actors++import (+ "context"+ "fmt"+ "log/slog"+ "math"+ "sync"+ "sync/atomic"++ "emperror.dev/errors"+ "github.com/anthdm/hollywood/actor"+ "go.uber.org/fx"+)++const actorMonitorID = "arcane-monitor"++type monitorReadyInternal struct{}++// Runtime owns the shared Hollywood engine and observes actor failures.+type Runtime struct {+ engine *actor.Engine+ monitorPID *actor.PID+ deadLetters atomic.Uint64+ restarts atomic.Uint64+ maxRestartsExceeded atomic.Uint64+ duplicateIDs atomic.Uint64+ policyMu sync.Mutex+ policies map[string]actorPolicyInternal+}++type actorPolicyInternal struct {+ maxRestarts int32+ handle *handleInternal+}++// handleInternal identifies a spawned actor and reports when its process terminates.+type handleInternal struct {+ engine *actor.Engine+ pid *actor.PID+ done chan struct{}+ doneOnce sync.Once+ stopOnce sync.Once+}++// Done closes when the actor process permanently stops.+func (h *handleInternal) Done() <-chan struct{} {+ if h == nil {+ return nil+ }+ return h.done+}++// Send queues a message for the actor while hiding Hollywood's engine and PID+// from services that only need to communicate with their owned process.+func (h *handleInternal) Send(message any) error {+ if h == nil || h.engine == nil || h.pid == nil {+ return errors.New("actor handle unavailable")+ }+ select {+ case <-h.done:+ return errors.New("actor stopped")+ default:+ h.engine.Send(h.pid, message)+ return nil+ }+}++// Stop drains the actor mailbox, initiates termination exactly once, and waits+// for the process to exit. A canceled wait does not cancel the queued stop.+func (h *handleInternal) Stop(ctx context.Context) error {+ if h == nil || h.engine == nil || h.pid == nil {+ return nil+ }+ if ctx == nil {+ return errors.New("actor stop context unavailable")+ }+ h.stopOnce.Do(func() { h.engine.Poison(h.pid) })+ select {+ case <-h.done:+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+}++func (h *handleInternal) stopInternal() {+ if h != nil {+ h.doneOnce.Do(func() { close(h.done) })+ }+}++// NewRuntime constructs the shared actor engine and registers its lifecycle monitor.+func NewRuntime(appCtx context.Context, lc fx.Lifecycle) (*Runtime, error) {+ engine, err := actor.NewEngine(actor.NewEngineConfig())+ if err != nil {+ return nil, err+ }++ runtime := &Runtime{+ engine: engine,+ policies: make(map[string]actorPolicyInternal),+ }+ ready := make(chan struct{})+ var readyOnce sync.Once+ runtime.monitorPID = engine.Spawn(+ func() actor.Receiver {+ return &runtimeMonitorInternal{+ runtime: runtime,+ ready: ready,+ readyOnce: &readyOnce,+ }+ },+ "runtime-monitor",+ actor.WithID(actorMonitorID),+ actor.WithContext(appCtx),+ actor.WithMaxRestarts(math.MaxInt32),+ )+ engine.Subscribe(runtime.monitorPID)+ engine.BroadcastEvent(monitorReadyInternal{})++ select {+ case <-ready:+ case <-appCtx.Done():+ <-engine.Poison(runtime.monitorPID).Done()+ return nil, appCtx.Err()+ }++ lc.Append(fx.Hook{+ OnStop: func(ctx context.Context) error {+ engine.Unsubscribe(runtime.monitorPID)+ stopped := engine.PoisonCtx(ctx, runtime.monitorPID)+ select {+ case <-stopped.Done():+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+ },+ })++ return runtime, nil+}++// startInternal starts a Hollywood receiver with a deterministic ID and an+// Arcane-enforced restart limit.+// Hollywood v1.0.5 panics while cleaning up its native max-restart path, so the+// shared monitor enforces the same ceiling and stops the actor safely instead.+func (r *Runtime) startInternal(producer actor.Producer, kind, id string, maxRestarts int32, opts ...actor.OptFunc) (*handleInternal, error) {+ if r == nil || r.engine == nil {+ return nil, errors.New("actor runtime unavailable")+ }+ if producer == nil {+ return nil, errors.New("actor producer unavailable")+ }++ policyKey := actor.NewPID(r.engine.Address(), kind+"/"+id).String()+ r.policyMu.Lock()+ if _, exists := r.policies[policyKey]; exists || r.engine.Registry.GetPID(kind, id) != nil {+ r.policyMu.Unlock()+ return nil, fmt.Errorf("actor ID already in use: %s/%s", kind, id)+ }+ handle := &handleInternal{+ engine: r.engine,+ pid: actor.NewPID(r.engine.Address(), kind+"/"+id),+ done: make(chan struct{}),+ }+ r.policies[policyKey] = actorPolicyInternal{maxRestarts: maxRestarts, handle: handle}+ r.policyMu.Unlock()++ options := append([]actor.OptFunc{}, opts...)+ options = append(options, actor.WithID(id), actor.WithMaxRestarts(math.MaxInt32))+ handle.pid = r.engine.Spawn(producer, kind, options...)+ if handle.pid.String() != policyKey {+ r.policyMu.Lock()+ delete(r.policies, policyKey)+ r.policyMu.Unlock()+ r.engine.Stop(handle.pid)+ return nil, fmt.Errorf("actor PID mismatch: got %s, expected %s", handle.pid.String(), policyKey)+ }+ return handle, nil+}++// DeadLetterCount returns the number of undeliverable actor messages observed.+func (r *Runtime) DeadLetterCount() uint64 {+ if r == nil {+ return 0+ }+ return r.deadLetters.Load()+}++// RestartCount returns the number of actor restarts observed.+func (r *Runtime) RestartCount() uint64 {+ if r == nil {+ return 0+ }+ return r.restarts.Load()+}++// MaxRestartsExceededCount returns the number of permanently stopped actors observed.+func (r *Runtime) MaxRestartsExceededCount() uint64 {+ if r == nil {+ return 0+ }+ return r.maxRestartsExceeded.Load()+}++// DuplicateIDCount returns the number of duplicate actor registrations observed.+func (r *Runtime) DuplicateIDCount() uint64 {+ if r == nil {+ return 0+ }+ return r.duplicateIDs.Load()+}++type runtimeMonitorInternal struct {+ runtime *Runtime+ ready chan struct{}+ readyOnce *sync.Once+}++func (m *runtimeMonitorInternal) Receive(ctx *actor.Context) {+ switch event := ctx.Message().(type) {+ case actor.Started:+ ctx.Engine().Subscribe(ctx.PID())++ case monitorReadyInternal:+ m.readyOnce.Do(func() { close(m.ready) })++ case actor.ActorRestartedEvent:+ if event.PID == nil {+ slog.ErrorContext(ctx.Context(), "actor restarted event omitted PID")+ return+ }+ m.runtime.policyMu.Lock()+ policy, tracked := m.runtime.policies[event.PID.String()]+ m.runtime.policyMu.Unlock()+ if tracked && event.Restarts > policy.maxRestarts {+ m.runtime.maxRestartsExceeded.Add(1)+ slog.ErrorContext(ctx.Context(), "actor permanently stopped after exceeding restart limit", "pid", event.PID.String(), "reason", event.Reason, "restarts", event.Restarts, "maxRestarts", policy.maxRestarts, "stack", string(event.Stacktrace))+ ctx.Engine().Stop(event.PID)+ return+ }+ m.runtime.restarts.Add(1)+ slog.ErrorContext(ctx.Context(), "actor crashed and restarted", "pid", event.PID.String(), "reason", event.Reason, "restarts", event.Restarts, "stack", string(event.Stacktrace))++ case actor.ActorMaxRestartsExceededEvent:+ if event.PID == nil {+ slog.ErrorContext(ctx.Context(), "actor max-restarts event omitted PID")+ return+ }+ m.runtime.maxRestartsExceeded.Add(1)+ slog.ErrorContext(ctx.Context(), "actor permanently stopped after exceeding restart limit", "pid", event.PID.String())++ case actor.ActorStoppedEvent:+ if event.PID == nil {+ slog.ErrorContext(ctx.Context(), "actor stopped event omitted PID")+ return+ }+ m.runtime.policyMu.Lock()+ policy, tracked := m.runtime.policies[event.PID.String()]+ if tracked {+ delete(m.runtime.policies, event.PID.String())+ }+ m.runtime.policyMu.Unlock()+ if tracked {+ policy.handle.stopInternal()+ }++ case actor.ActorDuplicateIdEvent:+ m.runtime.duplicateIDs.Add(1)+ pid := "<nil>"+ if event.PID != nil {+ pid = event.PID.String()+ }+ slog.ErrorContext(ctx.Context(), "duplicate actor ID rejected", "pid", pid)++ case actor.DeadLetterEvent:+ m.runtime.deadLetters.Add(1)+ target := "<nil>"+ if event.Target != nil {+ target = event.Target.String()+ }+ slog.ErrorContext(ctx.Context(), "actor message sent to dead letter", "target", target, "messageType", fmt.Sprintf("%T", event.Message))++ case actor.Stopped:+ ctx.Engine().Unsubscribe(ctx.PID())+ }+}diff --git a/backend/internal/actors/runtime_test.go b/backend/internal/actors/runtime_test.gonew file mode 100644index 000000000..ec01dccae--- /dev/null+++ b/backend/internal/actors/runtime_test.go@@ -0,0 +1,154 @@+package actors++import (+ "context"+ "testing"+ "time"++ "github.com/anthdm/hollywood/actor"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++type panicActorInternal struct{}++func (panicActorInternal) Receive(ctx *actor.Context) {+ if _, ok := ctx.Message().(panicActorMessageInternal); ok {+ panic("deliberate actor test panic")+ }+}++type panicActorMessageInternal struct{}++type stopBlockingActorInternal struct {+ release <-chan struct{}+}++func (a stopBlockingActorInternal) Receive(ctx *actor.Context) {+ if _, ok := ctx.Message().(actor.Stopped); ok {+ <-a.release+ }+}++func TestRuntimeMonitorObservesRestartsMaxRestartsAndDeadLettersInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ handle, err := runtime.startInternal(+ func() actor.Receiver { return panicActorInternal{} },+ "panic-test",+ "restart-limit",+ 3,+ actor.WithContext(t.Context()),+ actor.WithRestartDelay(0),+ )+ require.NoError(t, err)+ pid := handle.pid++ for expectedRestarts := uint64(1); expectedRestarts <= 3; expectedRestarts++ {+ require.NoError(t, handle.Send(panicActorMessageInternal{}))+ require.Eventually(t, func() bool {+ return runtime.RestartCount() == expectedRestarts+ }, time.Second, time.Millisecond)+ }++ require.NoError(t, handle.Send(panicActorMessageInternal{}))+ require.Eventually(t, func() bool {+ return runtime.MaxRestartsExceededCount() == 1+ }, time.Second, time.Millisecond)+ require.Eventually(t, func() bool {+ select {+ case <-handle.Done():+ return true+ default:+ return false+ }+ }, time.Second, time.Millisecond)++ runtime.engine.Send(pid, panicActorMessageInternal{})+ require.Eventually(t, func() bool {+ return runtime.DeadLetterCount() == 1+ }, time.Second, time.Millisecond)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestHandleCanceledStopStillTerminatesAndCanBeJoinedInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ release := make(chan struct{})+ handle, err := runtime.startInternal(+ func() actor.Receiver { return stopBlockingActorInternal{release: release} },+ "handle-test",+ "stop",+ 1,+ actor.WithContext(t.Context()),+ )+ require.NoError(t, err)+ require.NotNil(t, handle)++ canceled, cancel := context.WithCancel(context.Background())+ cancel()+ require.ErrorIs(t, handle.Stop(canceled), context.Canceled)+ close(release)++ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)+ defer stopCancel()+ require.NoError(t, handle.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestRuntimeRejectsDuplicateActorIDWithoutOverwritingOwnerInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ first, err := runtime.startInternal(+ func() actor.Receiver { return panicActorInternal{} },+ "duplicate-test",+ "same-id",+ 1,+ actor.WithContext(t.Context()),+ )+ require.NoError(t, err)++ second, err := runtime.startInternal(+ func() actor.Receiver { return panicActorInternal{} },+ "duplicate-test",+ "same-id",+ 1,+ actor.WithContext(t.Context()),+ )+ require.Nil(t, second)+ require.ErrorContains(t, err, "actor ID already in use")+ require.Equal(t, actor.NewPID(runtime.engine.Address(), "duplicate-test/same-id").String(), first.pid.String())++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, first.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestRuntimeMonitorResubscribesOnStartedInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)++ runtime.engine.Unsubscribe(runtime.monitorPID)+ time.Sleep(10 * time.Millisecond)+ runtime.engine.Send(runtime.monitorPID, actor.Started{})+ time.Sleep(10 * time.Millisecond)+ runtime.engine.Send(actor.NewPID(runtime.engine.Address(), "missing/actor"), "dead letter")+ require.Eventually(t, func() bool {+ return runtime.DeadLetterCount() == 1+ }, time.Second, time.Millisecond)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/signal.go b/backend/internal/actors/signal.gonew file mode 100644index 000000000..2fc9c49b4--- /dev/null+++ b/backend/internal/actors/signal.go@@ -0,0 +1,62 @@+package actors++import (+ "sync"+ "sync/atomic"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+)++// Signal delivers each published value once to every current subscriber.+// Subscribers run sequentially on the publishing goroutine.+type Signal[T any] struct {+ mu sync.RWMutex+ nextID atomic.Uint64+ subscribers map[uint64]func(T)+}++// NewSignal creates an empty typed signal.+func NewSignal[T any]() *Signal[T] {+ return &Signal[T]{subscribers: make(map[uint64]func(T))}+}++// Subscribe registers a callback and returns an idempotent unsubscribe function.+func (s *Signal[T]) Subscribe(callback func(T)) func() {+ if s == nil || callback == nil {+ return func() {}+ }+ id := s.nextID.Add(1)+ s.mu.Lock()+ s.subscribers[id] = callback+ s.mu.Unlock()++ var once sync.Once+ return func() {+ once.Do(func() {+ s.mu.Lock()+ delete(s.subscribers, id)+ s.mu.Unlock()+ })+ }+}++// Publish synchronously delivers value without holding the subscriber lock.+func (s *Signal[T]) Publish(value T) {+ if s == nil {+ return+ }+ s.mu.RLock()+ subscribers := make([]func(T), 0, len(s.subscribers))+ for _, callback := range s.subscribers {+ subscribers = append(subscribers, callback)+ }+ s.mu.RUnlock()++ for _, callback := range subscribers {+ func() {+ var err error+ defer utils.RecoverToError(&err, "actor signal subscriber")+ callback(value)+ }()+ }+}diff --git a/backend/internal/actors/signal_test.go b/backend/internal/actors/signal_test.gonew file mode 100644index 000000000..ea9d72f16--- /dev/null+++ b/backend/internal/actors/signal_test.go@@ -0,0 +1,38 @@+package actors++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestSignalPublishesOnceAndUnsubscribesInternal(t *testing.T) {+ signal := NewSignal[int]()+ firstCalls := 0+ secondCalls := 0+ unsubscribeFirst := signal.Subscribe(func(value int) {+ firstCalls += value+ })+ signal.Subscribe(func(value int) {+ secondCalls += value+ })++ signal.Publish(1)+ unsubscribeFirst()+ unsubscribeFirst()+ signal.Publish(2)++ require.Equal(t, 1, firstCalls)+ require.Equal(t, 3, secondCalls)+}++func TestSignalContainsSubscriberPanicInternal(t *testing.T) {+ signal := NewSignal[int]()+ called := false+ signal.Subscribe(func(int) { panic("deliberate signal panic") })+ signal.Subscribe(func(int) { called = true })++ signal.Publish(1)++ require.True(t, called)+}diff --git a/backend/internal/actors/snapshot.go b/backend/internal/actors/snapshot.gonew file mode 100644index 000000000..4ce5c0c4f--- /dev/null+++ b/backend/internal/actors/snapshot.go@@ -0,0 +1,29 @@+package actors++import "sync/atomic"++// Snapshot publishes immutable copies of a value for lock-free readers.+type Snapshot[T any] struct {+ value atomic.Pointer[T]+}++// Store replaces the published value with an immutable copy.+func (s *Snapshot[T]) Store(value T) {+ if s == nil {+ return+ }+ s.value.Store(&value)+}++// Load returns a copy of the latest published value.+func (s *Snapshot[T]) Load() (T, bool) {+ var zero T+ if s == nil {+ return zero, false+ }+ value := s.value.Load()+ if value == nil {+ return zero, false+ }+ return *value, true+}diff --git a/backend/internal/actors/state.go b/backend/internal/actors/state.gonew file mode 100644index 000000000..74e969715--- /dev/null+++ b/backend/internal/actors/state.go@@ -0,0 +1,75 @@+package actors++import (+ "context"++ "emperror.dev/errors"+)++// State serializes mutations through an actor and publishes immutable snapshots.+type State[T any] struct {+ executor *Executor+ value T+ clone func(T) T+ snapshot Snapshot[T]+}++// NewState creates actor-owned state with a caller-defined snapshot clone.+func NewState[T any](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, initial T, clone func(T) T) (*State[T], error) {+ if clone == nil {+ return nil, errors.New("actor state clone unavailable")+ }+ executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)+ if err != nil {+ return nil, err+ }+ state := &State[T]{+ executor: executor,+ value: initial,+ clone: clone,+ }+ state.snapshot.Store(clone(initial))+ return state, nil+}++// Apply serializes one mutation and publishes the resulting state.+func (s *State[T]) Apply(ctx context.Context, label string, mutate func(context.Context, *T) error) error {+ if s == nil || s.executor == nil {+ return errors.New("actor state unavailable")+ }+ if mutate == nil {+ return errors.New("actor state mutation unavailable")+ }+ _, err := Execute(ctx, s.executor, label, func(actorCtx context.Context) (NoPayload, error) {+ defer func() {+ s.snapshot.Store(s.clone(s.value))+ }()+ return NoPayload{}, mutate(actorCtx, &s.value)+ }, nil)+ return err+}++// Load returns the latest immutable state snapshot.+func (s *State[T]) Load() (T, bool) {+ if s == nil {+ var zero T+ return zero, false+ }+ return s.snapshot.Load()+}++// Stop drains pending mutations and joins the state actor.+func (s *State[T]) Stop(ctx context.Context) error {+ if s == nil || s.executor == nil {+ return nil+ }+ return s.executor.Stop(ctx)+}++// Done closes after the state actor terminates.+func (s *State[T]) Done() <-chan struct{} {+ if s == nil || s.executor == nil {+ return nil+ }+ return s.executor.Done()+}diff --git a/backend/internal/actors/state_map.go b/backend/internal/actors/state_map.gonew file mode 100644index 000000000..e9863be7a--- /dev/null+++ b/backend/internal/actors/state_map.go@@ -0,0 +1,178 @@+package actors++import (+ "context"+ "maps"+ "sync"+ "sync/atomic"++ "emperror.dev/errors"+)++// StateMap owns mutable keyed state while publishing immutable snapshots for readers.+type StateMap[K comparable, V any] struct {+ executor *Executor+ values map[K]V+ snapshot atomic.Pointer[map[K]V]+ mu sync.Mutex+}++type stateMapValueInternal[V any] struct {+ value V+ found bool+}++// NewStateMap creates a locally serialized state map for callers without an actor runtime.+func NewStateMap[K comparable, V any]() *StateMap[K, V] {+ state := &StateMap[K, V]{values: make(map[K]V)}+ state.publishInternal()+ return state+}++// NewActorStateMap creates a state map whose mutations use the shared actor runtime.+func NewActorStateMap[K comparable, V any](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32) (*StateMap[K, V], error) {+ executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)+ if err != nil {+ return nil, err+ }+ state := &StateMap[K, V]{executor: executor, values: make(map[K]V)}+ state.publishInternal()+ return state, nil+}++// Get reads one value from the latest immutable snapshot without entering the mailbox.+func (s *StateMap[K, V]) Get(key K) (V, bool) {+ var zero V+ if s == nil {+ return zero, false+ }+ snapshot := s.snapshot.Load()+ if snapshot == nil {+ return zero, false+ }+ value, ok := (*snapshot)[key]+ return value, ok+}++// Values reads every value from the latest immutable snapshot.+func (s *StateMap[K, V]) Values() []V {+ if s == nil {+ return nil+ }+ snapshot := s.snapshot.Load()+ if snapshot == nil {+ return nil+ }+ values := make([]V, 0, len(*snapshot))+ for _, value := range *snapshot {+ values = append(values, value)+ }+ return values+}++// Apply serializes a mutation and publishes it atomically when changed is true.+func (s *StateMap[K, V]) Apply(ctx context.Context, label string, mutate func(map[K]V) (changed bool, err error)) error {+ if mutate == nil {+ return errors.New("state map mutation unavailable")+ }+ _, err := ApplyStateMap(ctx, s, label, func(values map[K]V) (NoPayload, bool, error) {+ changed, err := mutate(values)+ return NoPayload{}, changed, err+ })+ return err+}++// ApplyStateMap serializes a typed mutation and publishes its resulting state.+func ApplyStateMap[K comparable, V, R any](+ ctx context.Context,+ s *StateMap[K, V],+ label string,+ mutate func(map[K]V) (result R, changed bool, err error),+) (R, error) {+ var zero R+ if s == nil {+ return zero, errors.New("state map unavailable")+ }+ if mutate == nil {+ return zero, errors.New("state map mutation unavailable")+ }+ work := func(context.Context) (R, error) {+ completed := false+ defer func() {+ if !completed {+ s.publishInternal()+ }+ }()+ result, changed, err := mutate(s.values)+ completed = true+ if changed || err != nil {+ s.publishInternal()+ }+ return result, err+ }+ if s.executor != nil {+ return Execute(ctx, s.executor, label, work, nil)+ }++ s.mu.Lock()+ defer s.mu.Unlock()+ return work(ctx)+}++// Store publishes a value and returns the value it replaced, if any.+func (s *StateMap[K, V]) Store(ctx context.Context, label string, key K, value V) (V, bool, error) {+ result, err := ApplyStateMap(ctx, s, label, func(values map[K]V) (stateMapValueInternal[V], bool, error) {+ previous, replaced := values[key]+ values[key] = value+ return stateMapValueInternal[V]{value: previous, found: replaced}, true, nil+ })+ return result.value, result.found, err+}++// Remove deletes and returns one value.+func (s *StateMap[K, V]) Remove(ctx context.Context, label string, key K) (V, bool, error) {+ result, err := ApplyStateMap(ctx, s, label, func(values map[K]V) (stateMapValueInternal[V], bool, error) {+ value, removed := values[key]+ if removed {+ delete(values, key)+ }+ return stateMapValueInternal[V]{value: value, found: removed}, removed, nil+ })+ return result.value, result.found, err+}++// RemoveWhere atomically removes every value selected by predicate.+func (s *StateMap[K, V]) RemoveWhere(ctx context.Context, label string, predicate func(K, V) bool) ([]V, error) {+ if predicate == nil {+ return nil, errors.New("state map predicate unavailable")+ }+ return ApplyStateMap(ctx, s, label, func(values map[K]V) ([]V, bool, error) {+ removed := make([]V, 0)+ for key, value := range values {+ if predicate(key, value) {+ delete(values, key)+ removed = append(removed, value)+ }+ }+ return removed, len(removed) > 0, nil+ })+}++// Drain atomically removes and returns every value.+func (s *StateMap[K, V]) Drain(ctx context.Context, label string) ([]V, error) {+ return s.RemoveWhere(ctx, label, func(K, V) bool { return true })+}++// Stop drains actor-backed mutations and joins the shared executor.+func (s *StateMap[K, V]) Stop(ctx context.Context) error {+ if s == nil || s.executor == nil {+ return nil+ }+ return s.executor.Stop(ctx)+}++func (s *StateMap[K, V]) publishInternal() {+ snapshot := make(map[K]V, len(s.values))+ maps.Copy(snapshot, s.values)+ s.snapshot.Store(&snapshot)+}diff --git a/backend/internal/actors/state_map_test.go b/backend/internal/actors/state_map_test.gonew file mode 100644index 000000000..fb792bffc--- /dev/null+++ b/backend/internal/actors/state_map_test.go@@ -0,0 +1,114 @@+package actors++import (+ "context"+ "sync"+ "testing"+ "time"++ "emperror.dev/errors"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++func TestActorStateMapSerializesMutationsAndPublishesSnapshotsInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ state, err := NewActorStateMap[string, int](t.Context(), runtime, "state-map-test", "values", 1)+ require.NoError(t, err)++ require.NoError(t, state.Apply(t.Context(), "store value", func(values map[string]int) (bool, error) {+ values["answer"] = 42+ return true, nil+ }))+ value, ok := state.Get("answer")+ require.True(t, ok)+ require.Equal(t, 42, value)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, state.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestActorStateMapPublishesPartialMutationBeforeReturningErrorInternal(t *testing.T) {+ state := NewStateMap[string, int]()+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {+ values["removed"] = 1+ values["retained"] = 2+ return true, nil+ }))++ expectedErr := errors.New("mutation failed after change")+ err := state.Apply(t.Context(), "partial mutation", func(values map[string]int) (bool, error) {+ delete(values, "removed")+ return true, expectedErr+ })+ require.ErrorIs(t, err, expectedErr)+ _, found := state.Get("removed")+ require.False(t, found)+ retained, found := state.Get("retained")+ require.True(t, found)+ require.Equal(t, 2, retained)+}++func TestStateMapPublishesPartialMutationBeforePropagatingPanicInternal(t *testing.T) {+ state := NewStateMap[string, int]()+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {+ values["removed"] = 1+ return true, nil+ }))++ require.Panics(t, func() {+ _ = state.Apply(t.Context(), "panic after mutation", func(values map[string]int) (bool, error) {+ delete(values, "removed")+ panic("mutation panic")+ })+ })+ _, found := state.Get("removed")+ require.False(t, found)+}++func TestActorStateMapCanceledMutationDoesNotShareResultStorageInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ state, err := NewActorStateMap[string, int](t.Context(), runtime, "state-map-test", "canceled-result", 1)+ require.NoError(t, err)+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {+ values["first"] = 1+ values["second"] = 2+ return true, nil+ }))++ mutationStarted := make(chan struct{})+ releaseMutation := make(chan struct{})+ var started sync.Once+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)+ defer cancel()+ result := make(chan []int, 1)+ resultErr := make(chan error, 1)+ go func() {+ removed, removeErr := state.RemoveWhere(ctx, "blocked removal", func(string, int) bool {+ started.Do(func() { close(mutationStarted) })+ <-releaseMutation+ return true+ })+ result <- removed+ resultErr <- removeErr+ }()++ <-mutationStarted+ require.ErrorIs(t, <-resultErr, context.DeadlineExceeded)+ require.Empty(t, <-result)+ close(releaseMutation)+ require.Eventually(t, func() bool {+ return len(state.Values()) == 0+ }, time.Second, time.Millisecond)++ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)+ defer stopCancel()+ require.NoError(t, state.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/actors/state_test.go b/backend/internal/actors/state_test.gonew file mode 100644index 000000000..f57629ec0--- /dev/null+++ b/backend/internal/actors/state_test.go@@ -0,0 +1,50 @@+package actors++import (+ "context"+ "maps"+ "testing"+ "time"++ "emperror.dev/errors"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++type stateTestValueInternal struct {+ values map[string]int+}++func TestStateSerializesMutationsAndPublishesClonedSnapshotsInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ state, err := NewState(+ t.Context(),+ runtime,+ "state-test",+ "values",+ 1,+ stateTestValueInternal{values: make(map[string]int)},+ func(value stateTestValueInternal) stateTestValueInternal {+ value.values = maps.Clone(value.values)+ return value+ },+ )+ require.NoError(t, err)++ expectedErr := errors.New("mutation failed")+ err = state.Apply(t.Context(), "mutate state", func(_ context.Context, value *stateTestValueInternal) error {+ value.values["answer"] = 42+ return expectedErr+ })+ require.ErrorIs(t, err, expectedErr)+ snapshot, ok := state.Load()+ require.True(t, ok)+ require.Equal(t, 42, snapshot.values["answer"])++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, state.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex 0ac4597fe..38e875e3b 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -15,6 +15,7 @@ import ("github.com/subosito/gotenv""github.com/getarcaneapp/arcane/backend/v2/api/ws"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/di"@@ -103,15 +104,15 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *databasefunc() context.Context { return appCtx },newConfiguredHTTPClient,),+ di.ActorOptions,di.ServiceOptions,di.JobOptions,serverOptions,fx.Invoke(+ registerAppCancelHook,initializeStartupState,registerJobs,- registerAppRollbackCancelHook,startEdgeTunnelClient,- registerAppCancelHook,),fx.WithLogger(func() fxevent.Logger {logger := &fxevent.SlogLogger{Logger: slog.Default()}@@ -330,20 +331,22 @@ func runRoleStartupTasks(ctx context.Context, roleService *services.RoleService,}}-func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, router *echo.Echo, _ *http.Server) {+func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, cfg *config.Config, router *echo.Echo, _ *http.Server) {+ var stop func(context.Context) errorlc.Append(fx.Hook{OnStart: func(context.Context) error {- startEdgeTunnelClientIfConfigured(appCtx, cfg, router)+ var startErr error+ stop, startErr = startEdgeTunnelClientIfConfigured(appCtx, actorRuntime, cfg, router)+ if startErr != nil {+ slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", startErr)+ }return nil},- })-}--func registerAppRollbackCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {- lc.Append(fx.Hook{- OnStop: func(context.Context) error {- cancelApp()- return nil+ OnStop: func(ctx context.Context) error {+ if stop == nil {+ return nil+ }+ return stop(ctx)},})}@@ -357,10 +360,10 @@ func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {})}-func startEdgeTunnelClientIfConfigured(appCtx context.Context, cfg *config.Config, router http.Handler) {+func startEdgeTunnelClientIfConfigured(appCtx context.Context, actorRuntime *actors.Runtime, cfg *config.Config, router http.Handler) (func(context.Context) error, error) {managerEndpointConfigured := cfg.ManagerApiUrl != ""if !cfg.EdgeAgent || !managerEndpointConfigured || cfg.AgentToken == "" {- return+ return nil, nil}edgeCfg := &edge.Config{@@ -381,18 +384,13 @@ func startEdgeTunnelClientIfConfigured(appCtx context.Context, cfg *config.Confi}slog.InfoContext(appCtx, "Starting edge agent session client", edge.StartupLogAttrs(edgeCfg)...)- errCh, err := edge.StartTunnelClientWithErrors(appCtx, edgeCfg, router)+ stop, err := edge.StartTunnelClient(appCtx, actorRuntime, edgeCfg, router)if err != nil {- slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", err)- return+ return nil, errors.WrapIf(err, "failed to start edge tunnel client")}slog.InfoContext(appCtx, "Edge tunnel client started", "manager_url", cfg.ManagerApiUrl)- go func() {- for err := range errCh {- slog.ErrorContext(appCtx, "Edge tunnel client error", "error", err)- }- }()+ return stop, nil}func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpClient *http.Client) error {diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 61025c28c..928dd3d10 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -3,7 +3,6 @@ package bootstrapimport ("context""crypto/tls"- "errors""io""net""net/http"@@ -13,9 +12,11 @@ import ("time""github.com/getarcaneapp/arcane/backend/v2/api"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1""github.com/labstack/echo/v5""github.com/stretchr/testify/assert"@@ -182,6 +183,7 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {Config: cfg,HandlerDeps: api.HandlerDeps{},AuthMiddleware: middleware.NewAuthMiddleware(nil, cfg),+ TunnelRegistry: edge.NewTunnelRegistry(),})handler, protocols := configureHTTPProtocolsInternal(false, router)@@ -293,60 +295,37 @@ func TestNewHTTPServerRejectsInvalidTLSCertificateInternal(t *testing.T) {require.Error(t, err)}-func TestRegisterAppCancelHookRunsBeforeEarlierStopHooks(t *testing.T) {+func TestRegisterAppCancelHookRunsAfterLaterStopHooks(t *testing.T) {appCtx, cancelApp := context.WithCancel(context.Background())lifecycle := fxtest.NewLifecycle(t)- appCanceledBeforeDependencyStop := false+ registerAppCancelHook(lifecycle, cancelApp)+ appActiveDuringDependencyStop := falselifecycle.Append(fx.Hook{OnStop: func(context.Context) error {- appCanceledBeforeDependencyStop = errors.Is(appCtx.Err(), context.Canceled)+ appActiveDuringDependencyStop = appCtx.Err() == nilreturn nil},})- registerAppCancelHook(lifecycle, cancelApp)lifecycle.RequireStart()lifecycle.RequireStop()- require.True(t, appCanceledBeforeDependencyStop)-}--func TestRollbackCancelHookRunsBeforeEarlierStopsInternal(t *testing.T) {- appCtx, cancelApp := context.WithCancel(context.Background())- appCanceledBeforeDependencyStop := false- app := fxtest.New(t,- fx.Supply(cancelApp),- fx.Invoke(func(lc fx.Lifecycle) {- lc.Append(fx.Hook{- OnStop: func(context.Context) error {- appCanceledBeforeDependencyStop = errors.Is(appCtx.Err(), context.Canceled)- return nil- },- })- }),- fx.Invoke(registerAppRollbackCancelHook),- fx.Invoke(func(lc fx.Lifecycle) {- lc.Append(fx.Hook{- OnStart: func(context.Context) error {- return errors.New("listen failed")- },- })- }),- )-- require.Error(t, app.Start(context.Background()))- require.True(t, appCanceledBeforeDependencyStop)+ require.True(t, appActiveDuringDependencyStop)+ require.ErrorIs(t, appCtx.Err(), context.Canceled)}func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {appCtx := t.Context()lifecycle := fxtest.NewLifecycle(t)- jobScheduler := newJobScheduler(appCtx, lifecycle, &config.Config{}, nil, nil, nil)+ runtime, err := actors.NewRuntime(appCtx, lifecycle)+ require.NoError(t, err)+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, runtime, nil, nil, nil, nil)+ require.NoError(t, err)watcher := &blockingBusWatcherInternal{started: make(chan struct{}),stopped: make(chan struct{}),}- jobScheduler.RegisterBusWatcher(watcher, false)+ require.NoError(t, jobScheduler.RegisterBusWatcher(watcher, false))lifecycle.RequireStart()select {diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.goindex fdce88495..8816c56cc 100644--- a/backend/internal/bootstrap/edge_bootstrap.go+++ b/backend/internal/bootstrap/edge_bootstrap.go@@ -8,12 +8,14 @@ import ("emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/models""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/labstack/echo/v5"+ "go.uber.org/fx")// registerEdgeTunnelRoutes configures the manager-side edge tunnel server.@@ -21,10 +23,13 @@ import (// Returns the TunnelServer for graceful shutdown.func registerEdgeTunnelRoutes(ctx context.Context,+ lifecycle fx.Lifecycle,+ actorRuntime *actors.Runtime,cfg *config.Config,apiGroup *echo.Group,environmentService *services.EnvironmentService,eventService *services.EventService,+ registry *edge.TunnelRegistry,) *edge.TunnelServer {// Resolver that validates API key and returns the environment IDresolver := func(ctx context.Context, token string) (string, error) {@@ -90,7 +95,7 @@ func registerEdgeTunnelRoutes(return nil}- server := edge.NewTunnelServerWithRegistry(edge.GetRegistry(), resolver, statusCallback)+ server := edge.NewTunnelServerWithRegistry(registry, resolver, statusCallback)server.SetConfig(&edge.Config{EdgeMTLSMode: cfg.EdgeMTLSMode,EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,@@ -135,7 +140,20 @@ func registerEdgeTunnelRoutes(})createEdgeMTLSIssueEventsInternal(ctx, eventService, envIDCopy, envNameCopy, remoteAddr, certIssued, caGenerated, reenrolled)})- go server.StartCleanupLoop(ctx)+ var cleanupRunner *actors.Runner+ lifecycle.Append(fx.Hook{+ OnStart: func(context.Context) error {+ var err error+ cleanupRunner, err = actors.NewRunner(ctx, actorRuntime, "edge-tunnel", "cleanup", "edge tunnel cleanup", 3, func(runCtx context.Context) error {+ server.StartCleanupLoop(runCtx)+ return nil+ })+ return err+ },+ OnStop: func(stopCtx context.Context) error {+ return cleanupRunner.Stop(stopCtx)+ },+ })apiGroup.POST("/tunnel/poll", server.HandlePoll)// Rate-limit agent mTLS enrollment per-IP. Enrollment is authenticated// only by the agent token, so we cap bursts to mitigate brute-force ordiff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 18d2b4fc6..141df9fe6 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -5,22 +5,36 @@ import ("encoding/json/v2""log/slog""net/http"+ "slices"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/models""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""go.uber.org/fx")-func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *services.SystemUpgradeService) *scheduler.JobScheduler {+func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *actors.Runtime, _ *actors.Gate[actors.AdmissionKey], imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *services.SystemUpgradeService) (schedulertypes.JobScheduler, error) {schedulerCtx, cancelScheduler := context.WithCancel(appCtx)- jobScheduler := scheduler.NewJobScheduler(schedulerCtx, cfg.GetLocation())+ jobScheduler, err := scheduler.NewJobScheduler(schedulerCtx, runtime, cfg.GetLocation())+ if err != nil {+ cancelScheduler()+ return nil, err+ }lc.Append(fx.Hook{OnStart: func(context.Context) error {slog.InfoContext(appCtx, "Starting scheduler")- jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true)- jobScheduler.StartScheduler()+ if imageUpdateWatcher != nil {+ if err := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); err != nil {+ return err+ }+ }+ if err := jobScheduler.StartScheduler(); err != nil {+ return err+ }if analytics != nil {go analytics.Run(schedulerCtx)}@@ -30,8 +44,8 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Configreturn nil},OnStop: func(ctx context.Context) error {- cancelScheduler()err := jobScheduler.Stop(ctx)+ cancelScheduler()if err != nil {slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)return err@@ -40,21 +54,24 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Configreturn nil},})- return jobScheduler+ return jobScheduler, nil}type registerJobsParams struct {fx.In- AppCtx context.Context- Config *config.Config- Scheduler *scheduler.JobScheduler+ Lifecycle fx.Lifecycle+ AppCtx context.Context+ Config *config.Config+ Scheduler schedulertypes.JobScheduler+ ActorRuntime *actors.RuntimeActivity *services.ActivityServiceGitOpsSync *services.GitOpsSyncServiceEnvironment *services.EnvironmentServiceJobSchedule *services.JobServiceSettings *services.SettingsService+ Admission *actors.Gate[actors.AdmissionKey]AutoUpdate *scheduler.AutoUpdateJobImageUpdateWatcher *scheduler.ImageUpdateWatcher@@ -70,7 +87,7 @@ type registerJobsParams struct {ActivitySweep *scheduler.ActivitySweepJob}-func registerJobs(params registerJobsParams) {+func registerJobs(params registerJobsParams) error {params.JobSchedule.SetScheduler(params.AppCtx, params.Scheduler)// Bootstrap owns registration, agent-mode gating, and settings callbacks.@@ -97,36 +114,47 @@ func registerJobs(params registerJobsParams) {}}- params.Scheduler.RegisterJob(params.AutoUpdate)- params.Scheduler.RegisterJob(params.DockerClientRefresh)- params.Scheduler.RegisterJob(params.Analytics)- params.Scheduler.RegisterJob(params.EventCleanup)- params.Scheduler.RegisterJob(params.PruningVolumeHelper)- params.Scheduler.RegisterJob(params.ExpiredSessionsCleanup)- params.Scheduler.RegisterJob(params.ScheduledPrune)+ for _, job := range []schedulertypes.Job{+ params.AutoUpdate,+ params.DockerClientRefresh,+ params.Analytics,+ params.EventCleanup,+ params.PruningVolumeHelper,+ params.ExpiredSessionsCleanup,+ params.ScheduledPrune,+ params.VulnerabilityScan,+ params.AutoHeal,+ params.ActivitySweep,+ } {+ if err := params.Scheduler.RegisterJob(job); err != nil {+ return err+ }+ }// FilesystemWatcher is intentionally not scheduler-registered; it watches inline// and is only rebound on settings changes below.- params.Scheduler.RegisterJob(params.VulnerabilityScan)- params.Scheduler.RegisterJob(params.AutoHeal)// Internal self-healing sweep (managers and agents alike); intentionally// absent from job_metadata so it stays out of the Jobs UI.- params.Scheduler.RegisterJob(params.ActivitySweep)// GitOps sync and environment health are no longer single global jobs; each// entity registers its own dynamic job.- registerDynamicJobs(dynamicJobsParams{+ if err := registerDynamicJobs(dynamicJobsParams{AppCtx: params.AppCtx,Config: params.Config,Scheduler: params.Scheduler,GitOpsSync: params.GitOpsSync,Environment: params.Environment,JobSchedule: params.JobSchedule,- })+ Admission: params.Admission,+ }); err != nil {+ return err+ }- setupSettingsCallbacks(settingsCallbacksParams{+ if err := setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{+ Lifecycle: params.Lifecycle,LifecycleCtx: params.AppCtx,Config: params.Config,Scheduler: params.Scheduler,+ ActorRuntime: params.ActorRuntime,Settings: params.Settings,Environment: params.Environment,AutoUpdate: params.AutoUpdate,@@ -135,25 +163,31 @@ func registerJobs(params registerJobsParams) {ScheduledPrune: params.ScheduledPrune,VulnerabilityScan: params.VulnerabilityScan,AutoHeal: params.AutoHeal,- })+ }); err != nil {+ return err+ }+ return nil}type dynamicJobsParams struct {AppCtx context.ContextConfig *config.Config- Scheduler *scheduler.JobScheduler+ Scheduler schedulertypes.JobSchedulerGitOpsSync *services.GitOpsSyncServiceEnvironment *services.EnvironmentServiceJobSchedule *services.JobService+ Admission *actors.Gate[actors.AdmissionKey]}// registerDynamicJobs injects the scheduler into the services that own per-entity// jobs and registers the jobs for already-existing entities at startup. AddJob is// an idempotent upsert, so these run safely before the scheduler is started.-func registerDynamicJobs(params dynamicJobsParams) {+func registerDynamicJobs(params dynamicJobsParams) error {// GitOps: one job per auto-sync-enabled sync (runs on manager and agents).if params.GitOpsSync != nil {- params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler)+ if err := params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {+ return err+ }params.GitOpsSync.RegisterAutoSyncJobsOnStartup(params.AppCtx)}@@ -161,7 +195,9 @@ func registerDynamicJobs(params dynamicJobsParams) {// UI still addresses "environment-health" by ID, so bridge its reschedule and// run-now back to EnvironmentService.if !params.Config.AgentMode && params.Environment != nil {- params.Environment.SetScheduler(params.AppCtx, params.Scheduler)+ if err := params.Environment.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {+ return err+ }params.JobSchedule.OnEnvironmentHealthReschedule = func(ctx context.Context) {params.Environment.RescheduleHealthJobs(ctx)}@@ -170,14 +206,17 @@ func registerDynamicJobs(params dynamicJobsParams) {}params.Environment.RegisterHealthJobsOnStartup(params.AppCtx)}+ return nil}-type settingsCallbacksParams struct {+type settingsSubscriptionsParams struct {+ Lifecycle fx.LifecycleLifecycleCtx context.ContextConfig *config.Config- Scheduler *scheduler.JobScheduler- Settings *services.SettingsService- Environment *services.EnvironmentService+ Scheduler settingsEffectsSchedulerInternal+ ActorRuntime *actors.Runtime+ Settings settingsChangeSubscriberInternal+ Environment timeoutSettingsEnvironmentInternalAutoUpdate *scheduler.AutoUpdateJobImageUpdateWatcher *scheduler.ImageUpdateWatcher@@ -187,9 +226,30 @@ type settingsCallbacksParams struct {AutoHeal *scheduler.AutoHealJob}-//nolint:contextcheck // callbacks intentionally use the app lifecycle context so reschedules outlive the triggering request context.-func setupSettingsCallbacks(params settingsCallbacksParams) {- params.Settings.OnImagePollingSettingsChanged = func(_ context.Context) {+type settingsChangeSubscriberInternal interface {+ SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func()+}++type settingsEffectsSchedulerInternal interface {+ RescheduleJob(ctx context.Context, job schedulertypes.Job) error+}++type timeoutSettingsEnvironmentInternal interface {+ ListRemoteEnvironments(ctx context.Context) ([]models.Environment, error)+ ProxyRequest(ctx context.Context, envID string, method string, path string, body []byte) ([]byte, int, error)+}++func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) error {+ unsubscribes := make([]func(), 0, 8)+ subscribe := func(keys []string, callback func([]libarcane.SettingUpdate)) {+ unsubscribes = append(unsubscribes, params.Settings.SubscribeSettingsChanges(keys, callback))+ }+ timeoutSyncExecutor, cancelTimeoutSync, err := setupTimeoutSettingsSubscriptionInternal(params, subscribe)+ if err != nil {+ return err+ }++ subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {if params.ImageUpdateWatcher != nil {params.ImageUpdateWatcher.RefreshSchedule()params.ImageUpdateWatcher.Trigger()@@ -197,53 +257,95 @@ func setupSettingsCallbacks(params settingsCallbacksParams) {if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoUpdate); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-update job", "error", err)}- }- params.Settings.OnAutoUpdateSettingsChanged = func(ctx context.Context) {- slog.DebugContext(params.LifecycleCtx, "AutoUpdateSettingsChanged callback triggered", "triggerContextCanceled", ctx.Err() != nil)+ })++ subscribe([]string{"autoUpdate", "autoUpdateInterval"}, func(_ []libarcane.SettingUpdate) {if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoUpdate); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-update job", "error", err)}- }- params.Settings.OnProjectsDirectoryChanged = func(_ context.Context) {+ })++ subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {if params.FilesystemWatcher != nil {if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)}}- }- params.Settings.OnTemplatesDirectoryChanged = func(_ context.Context) {+ })++ subscribe([]string{"templatesDirectory"}, func(_ []libarcane.SettingUpdate) {if params.FilesystemWatcher != nil {if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)}}- }- params.Settings.OnScheduledPruneSettingsChanged = func(_ context.Context) {+ })++ subscribe([]string{"scheduledPruneEnabled", "scheduledPruneInterval"}, func(_ []libarcane.SettingUpdate) {if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.ScheduledPrune); err != nil {slog.WarnContext(params.LifecycleCtx, "Failed to reschedule scheduled-prune job", "error", err)}- }- params.Settings.OnVulnerabilityScanSettingsChanged = func(_ context.Context) {- if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.VulnerabilityScan); err != nil {- slog.WarnContext(params.LifecycleCtx, "Failed to reschedule vulnerability-scan job", "error", err)- }- }- params.Settings.OnAutoHealSettingsChanged = func(ctx context.Context) {- if err := params.Scheduler.RescheduleJob(ctx, params.AutoHeal); err != nil {- slog.WarnContext(ctx, "Failed to reschedule auto-heal job", "error", err)- }+ })++ subscribe(+ []string{+ "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyNetwork", "trivySecurityOpts", "trivyPrivileged",+ "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers",+ },+ func(_ []libarcane.SettingUpdate) {+ if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.VulnerabilityScan); err != nil {+ slog.WarnContext(params.LifecycleCtx, "Failed to reschedule vulnerability-scan job", "error", err)+ }+ },+ )++ subscribe(+ []string{"autoHealEnabled", "autoHealInterval", "autoHealExcludedContainers", "autoHealMaxRestarts", "autoHealRestartWindow"},+ func(_ []libarcane.SettingUpdate) {+ if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoHeal); err != nil {+ slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-heal job", "error", err)+ }+ },+ )++ params.Lifecycle.Append(fx.Hook{+ OnStop: func(ctx context.Context) error {+ for _, unsubscribe := range slices.Backward(unsubscribes) {+ unsubscribe()+ }+ if cancelTimeoutSync != nil {+ cancelTimeoutSync()+ }+ return timeoutSyncExecutor.Stop(ctx)+ },+ })+ return nil+}++func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*actors.Executor, context.CancelFunc, error) {+ if params.Config.AgentMode {+ return nil, nil, nil}- // Only set up timeout sync callback on main instance (not in agent mode)- if !params.Config.AgentMode {- params.Settings.OnTimeoutSettingsChanged = func(ctx context.Context, timeoutSettings []libarcane.SettingUpdate) {- go syncTimeoutSettingsToAgentsInternal(context.WithoutCancel(ctx), params.Environment, timeoutSettings)- }+ timeoutSyncExecutor, err := actors.NewExecutor(params.LifecycleCtx, params.ActorRuntime, "services", "settings-timeout-sync", 3)+ if err != nil {+ return nil, nil, err}+ timeoutSyncContext, cancelTimeoutSync := context.WithCancel(params.LifecycleCtx)+ subscribe(libarcane.TimeoutSettingKeys(), func(updates []libarcane.SettingUpdate) {+ _, err := actors.Submit(timeoutSyncContext, timeoutSyncExecutor, "sync timeout settings to remote environments", func(ctx context.Context) (actors.NoPayload, error) {+ syncTimeoutSettingsToAgentsInternal(ctx, params.Environment, updates)+ return actors.NoPayload{}, nil+ }, nil)+ if err != nil && timeoutSyncContext.Err() == nil {+ slog.ErrorContext(timeoutSyncContext, "Failed to queue timeout settings sync", "error", err)+ }+ })+ return timeoutSyncExecutor, cancelTimeoutSync, nil}// syncTimeoutSettingsToAgentsInternal syncs timeout settings to all connected remote environments-func syncTimeoutSettingsToAgentsInternal(ctx context.Context, environment *services.EnvironmentService, timeoutSettings []libarcane.SettingUpdate) {+func syncTimeoutSettingsToAgentsInternal(ctx context.Context, environment timeoutSettingsEnvironmentInternal, timeoutSettings []libarcane.SettingUpdate) {envs, err := environment.ListRemoteEnvironments(ctx)if err != nil {slog.WarnContext(ctx, "Failed to list remote environments for timeout sync", "error", err)@@ -270,13 +372,13 @@ func syncTimeoutSettingsToAgentsInternal(ctx context.Context, environment *servislog.InfoContext(ctx, "Syncing environment settings to remote environments", "count", len(envs), "keys", keys)for _, env := range envs {- resp, err := environment.ExecuteRemoteRequest(ctx, env.ID, http.MethodPut, "/api/environments/0/settings", body)+ responseBody, statusCode, err := environment.ProxyRequest(ctx, env.ID, http.MethodPut, "/api/environments/0/settings", body)if err != nil {slog.WarnContext(ctx, "Failed to sync timeout settings to environment", "environmentID", env.ID, "environmentName", env.Name, "error", err)continue}- if err := resp.RequireSuccess(); err != nil {- slog.WarnContext(ctx, "Environment returned non-OK status for timeout sync", "environmentID", env.ID, "environmentName", env.Name, "statusCode", resp.StatusCode, "response", string(resp.Body))+ if statusCode < http.StatusOK || statusCode >= http.StatusMultipleChoices {+ slog.WarnContext(ctx, "Environment returned non-OK status for timeout sync", "environmentID", env.ID, "environmentName", env.Name, "statusCode", statusCode, "response", string(responseBody))continue}slog.DebugContext(ctx, "Successfully synced timeout settings to environment", "environmentID", env.ID, "environmentName", env.Name)diff --git a/backend/internal/bootstrap/jobs_bootstrap_test.go b/backend/internal/bootstrap/jobs_bootstrap_test.gonew file mode 100644index 000000000..5466912e7--- /dev/null+++ b/backend/internal/bootstrap/jobs_bootstrap_test.go@@ -0,0 +1,112 @@+package bootstrap++import (+ "context"+ "slices"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/models"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++type settingsSubscriptionStubInternal struct {+ pollingCallback func([]libarcane.SettingUpdate)+ timeoutCallback func([]libarcane.SettingUpdate)+}++func (s *settingsSubscriptionStubInternal) SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func() {+ if slices.Contains(keys, "pollingEnabled") {+ s.pollingCallback = callback+ }+ if slices.Contains(keys, "dockerApiTimeout") {+ s.timeoutCallback = callback+ }+ return func() {}+}++type settingsSubscriptionSchedulerStubInternal struct {+ rescheduled chan struct{}+}++func (s *settingsSubscriptionSchedulerStubInternal) RescheduleJob(context.Context, schedulertypes.Job) error {+ select {+ case <-s.rescheduled:+ default:+ close(s.rescheduled)+ }+ return nil+}++type timeoutSyncEnvironmentStubInternal struct {+ started chan struct{}+}++func (s *timeoutSyncEnvironmentStubInternal) ListRemoteEnvironments(context.Context) ([]models.Environment, error) {+ return []models.Environment{{+ BaseModel: models.BaseModel{ID: "remote"},+ Name: "remote",+ }}, nil+}++func (s *timeoutSyncEnvironmentStubInternal) ProxyRequest(ctx context.Context, _ string, _ string, _ string, _ []byte) ([]byte, int, error) {+ select {+ case <-s.started:+ default:+ close(s.started)+ }+ <-ctx.Done()+ return nil, 0, ctx.Err()+}++func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(context.Background(), lifecycle)+ require.NoError(t, err)++ settings := &settingsSubscriptionStubInternal{}+ scheduler := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}+ environment := &timeoutSyncEnvironmentStubInternal{started: make(chan struct{})}+ require.NoError(t, setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{+ Lifecycle: lifecycle,+ LifecycleCtx: context.Background(),+ Config: &config.Config{},+ Scheduler: scheduler,+ ActorRuntime: runtime,+ Settings: settings,+ Environment: environment,+ }))++ timeoutCallbackDone := make(chan struct{})+ go func() {+ settings.timeoutCallback([]libarcane.SettingUpdate{{Key: "dockerApiTimeout", Value: "30"}})+ close(timeoutCallbackDone)+ }()++ select {+ case <-environment.started:+ case <-time.After(time.Second):+ t.Fatal("timeout sync did not start")+ }+ select {+ case <-timeoutCallbackDone:+ case <-time.After(time.Second):+ t.Fatal("timeout settings callback remained blocked by remote sync")+ }++ settings.pollingCallback([]libarcane.SettingUpdate{{Key: "pollingEnabled", Value: "true"}})+ select {+ case <-scheduler.rescheduled:+ case <-time.After(time.Second):+ t.Fatal("local settings effect was blocked by remote timeout sync")+ }++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.goindex 0cc697927..e58d84476 100644--- a/backend/internal/bootstrap/router_bootstrap.go+++ b/backend/internal/bootstrap/router_bootstrap.go@@ -18,6 +18,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/api/handlers""github.com/getarcaneapp/arcane/backend/v2/api/ws""github.com/getarcaneapp/arcane/backend/v2/frontend"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/models"@@ -160,9 +161,12 @@ type RouterParams struct {fx.InContext context.Context+ Lifecycle fx.Lifecycle+ ActorRuntime *actors.RuntimeConfig *config.ConfigHandlerDeps api.HandlerDepsAuthMiddleware *middleware.AuthMiddleware+ TunnelRegistry *edge.TunnelRegistry}func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {@@ -217,8 +221,6 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {))handlerAppCtx := handlers.NewActivityAppContext(ctx)- tunnelRegistry := edge.NewTunnelRegistry()- edge.SetDefaultRegistry(tunnelRegistry)envResolver := func(ctx context.Context, id string) (string, *string, bool, error) {env, err := deps.Environment.GetEnvironmentByID(ctx, id)if err != nil || env == nil {@@ -240,7 +242,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {envResolver,createAuthValidatorInternal(deps),permissionMatcher,- tunnelRegistry,+ p.TunnelRegistry,// Proxied WebSocket upgrades enforce the same Origin policy as the local// endpoints, so a cross-origin page cannot ride a session cookie into a// remote environment.@@ -268,7 +270,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {// This is only registered when NOT in agent mode (i.e., running as manager)var tunnelServer *edge.TunnelServerif !cfg.AgentMode {- tunnelServer = registerEdgeTunnelRoutes(ctx, cfg, apiGroup, deps.Environment, deps.Event)+ tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, p.ActorRuntime, cfg, apiGroup, deps.Environment, deps.Event, p.TunnelRegistry)}if cfg.Environment != "production" {diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.goindex 806c1abd4..120706299 100644--- a/backend/internal/bootstrap/server_bootstrap.go+++ b/backend/internal/bootstrap/server_bootstrap.go@@ -108,9 +108,6 @@ func NewHTTPServer(lc fx.Lifecycle, p HTTPServerParams) (*http.Server, error) {if grpcServer != nil {grpcServer.GracefulStop()}- if p.TunnelServer != nil {- p.TunnelServer.WaitForCleanupDone()- }if shutdownErr == nil {slog.InfoContext(ctx, "Server stopped gracefully")}diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex df25cb880..204608808 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -2,12 +2,20 @@package diimport (+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"arcanelogging "github.com/getarcaneapp/arcane/backend/v2/internal/logging""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler""go.uber.org/fx")+// ActorOptions provides the shared in-process actor runtime.+var ActorOptions = fx.Options(+ fx.Provide(actors.NewRuntime),+ fx.Provide(provideAdmissionGateInternal),+ fx.Provide(provideTunnelRegistryInternal),+)+// ServiceOptions provides the backend service graph.var ServiceOptions = fx.Options(fx.Provide(@@ -18,7 +26,7 @@ var ServiceOptions = fx.Options(// Services constructed directly through their public constructors.services.NewEventService,services.NewActivityService,- services.NewSettingsService,+ provideSettingsServiceInternal,services.NewKVService,services.NewJobService,services.NewSettingsSearchService,diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.goindex f20fed7f9..10f3153b1 100644--- a/backend/internal/di/di_test.go+++ b/backend/internal/di/di_test.go@@ -82,6 +82,7 @@ func TestOptionsValidate(t *testing.T) {&http.Client{},),fx.Provide(func() context.Context { return context.Background() }),+ ActorOptions,ServiceOptions,JobOptions,fx.Invoke(func(graphParams) {}),diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex 57d85560b..33362b35b 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -6,11 +6,14 @@ import ("log/slog""net/http"+ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"arcanelogging "github.com/getarcaneapp/arcane/backend/v2/internal/logging""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/services"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler""github.com/getarcaneapp/arcane/backend/v2/resources""go.uber.org/fx"@@ -20,16 +23,67 @@ func provideResourcesFSInternal() embed.FS {return resources.FS}-func provideDockerClientServiceInternal(ctx context.Context, lc fx.Lifecycle, db *database.DB, cfg *config.Config, settings *services.SettingsService) *services.DockerClientService {+func provideSettingsServiceInternal(ctx context.Context, lc fx.Lifecycle, db *database.DB, runtime *actors.Runtime) (*services.SettingsService, error) {+ executor, err := actors.NewExecutor(ctx, runtime, "services", "settings", 3)+ if err != nil {+ return nil, err+ }+ effects, err := actors.NewExecutor(ctx, runtime, "services", "settings-effects", 3)+ if err != nil {+ return nil, errors.Combine(err, executor.Stop(ctx))+ }+ service, err := services.NewSettingsService(ctx, db, executor, effects)+ if err != nil {+ return nil, errors.Combine(err, executor.Stop(ctx), effects.Stop(ctx))+ }+ lc.Append(fx.Hook{+ OnStop: func(ctx context.Context) error {+ return errors.Combine(executor.Stop(ctx), effects.Stop(ctx))+ },+ })+ return service, nil+}++func provideAdmissionGateInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*actors.Gate[actors.AdmissionKey], error) {+ gate, err := actors.NewGate[actors.AdmissionKey](ctx, runtime, "admission", "application")+ if err != nil {+ return nil, err+ }+ lc.Append(fx.Hook{OnStop: gate.Stop})+ return gate, nil+}++func provideTunnelRegistryInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*edge.TunnelRegistry, error) {+ registry, err := edge.NewActorTunnelRegistry(ctx, runtime)+ if err != nil {+ return nil, err+ }+ edge.SetDefaultRegistry(registry)+ lc.Append(fx.Hook{+ OnStop: func(ctx context.Context) error {+ defer edge.ClearDefaultRegistry(registry)+ return registry.Stop(ctx)+ },+ })+ return registry, nil+}++func provideDockerClientServiceInternal(ctx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, db *database.DB, cfg *config.Config, settings *services.SettingsService) *services.DockerClientService {service := services.NewDockerClientService(ctx, db, cfg, settings)+ var runner *actors.Runnerlc.Append(fx.Hook{OnStart: func(context.Context) error {- go service.WatchEvents(ctx)- return nil+ var err error+ runner, err = actors.NewRunner(ctx, actorRuntime, "docker-events", "stream", "Docker event watcher", 3, func(runCtx context.Context) error {+ service.WatchEvents(runCtx)+ return nil+ })+ return err},- OnStop: func(context.Context) error {+ OnStop: func(stopCtx context.Context) error {+ err := runner.Stop(stopCtx)service.Close()- return nil+ return err},})return service@@ -113,10 +167,30 @@ func provideAuthMiddlewareInternal(auth *services.AuthService, apiKey *services.WithPermissionResolver(role)}-func provideFilesystemWatcherJobInternal(ctx context.Context, project *services.ProjectService, template *services.TemplateService, settings *services.SettingsService, cfg *config.Config) *scheduler.FilesystemWatcherJob {- job, err := scheduler.RegisterFilesystemWatcherJob(ctx, project, template, settings, cfg.ProjectScanMaxDepth)+func provideFilesystemWatcherJobInternal(ctx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, project *services.ProjectService, template *services.TemplateService, settings *services.SettingsService, cfg *config.Config) (*scheduler.FilesystemWatcherJob, error) {+ job, err := scheduler.NewFilesystemWatcherJob(ctx, actorRuntime, project, template, settings, cfg.ProjectScanMaxDepth)if err != nil {- slog.ErrorContext(ctx, "Failed to register filesystem watcher job", "error", err)+ return nil, err}- return job+ var runner *actors.Runner+ lc.Append(fx.Hook{+ OnStart: func(context.Context) error {+ runner, err = actors.NewRunner(ctx, actorRuntime, "filesystem-watcher", "lifecycle", "filesystem watcher", 3, func(runCtx context.Context) error {+ if startErr := job.Start(runCtx); startErr != nil {+ return startErr+ }+ <-runCtx.Done()+ return nil+ })+ if err != nil {+ return err+ }+ slog.InfoContext(ctx, "Filesystem watcher job registered")+ return nil+ },+ OnStop: func(stopCtx context.Context) error {+ return errors.Combine(runner.Stop(stopCtx), job.Stop(stopCtx))+ },+ })+ return job, nil}diff --git a/backend/internal/services/activity_service_test.go b/backend/internal/services/activity_service_test.goindex 2a9ec0aac..322c026b2 100644--- a/backend/internal/services/activity_service_test.go+++ b/backend/internal/services/activity_service_test.go@@ -373,7 +373,7 @@ func setupQueuedActivityServiceInternal(t *testing.T) (*ActivityService, contextrequire.NoError(t, err)sqlDB.SetMaxOpenConns(1)- settings, err := NewSettingsService(ctx, wrapped)+ settings, err := newSettingsServiceForTestInternal(t, ctx, wrapped)require.NoError(t, err)require.NoError(t, settings.SetIntSetting(ctx, maxConcurrentActivitiesSettingKey, 1))return NewActivityService(wrapped, settings), ctxdiff --git a/backend/internal/services/auth_service_buildables_test.go b/backend/internal/services/auth_service_buildables_test.goindex 38acdf519..333f2795c 100644--- a/backend/internal/services/auth_service_buildables_test.go+++ b/backend/internal/services/auth_service_buildables_test.go@@ -25,7 +25,7 @@ func TestGetAutoLoginConfig_Enabled(t *testing.T) {ctx := context.Background()db := setupAuthServiceTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.SetBoolSetting(ctx, "authLocalEnabled", true))@@ -51,7 +51,7 @@ func TestGetAutoLoginConfig_DisabledWhenLocalAuthDisabled(t *testing.T) {ctx := context.Background()db := setupAuthServiceTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.SetBoolSetting(ctx, "authLocalEnabled", false))diff --git a/backend/internal/services/auth_service_test.go b/backend/internal/services/auth_service_test.goindex 3e56df30a..2f9685905 100644--- a/backend/internal/services/auth_service_test.go+++ b/backend/internal/services/auth_service_test.go@@ -324,7 +324,7 @@ func TestVerifyToken_VersionMismatch(t *testing.T) {func TestRefreshToken_Valid(t *testing.T) {db := setupAuthServiceTestDB(t)userSvc := NewUserService(db)- settingsSvc, err := NewSettingsService(context.Background(), db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)require.NoError(t, err)s := newTestAuthService("")s.userService = userSvc@@ -355,7 +355,7 @@ func TestRefreshToken_Valid(t *testing.T) {func TestRefreshToken_VersionMismatchRotates(t *testing.T) {db := setupAuthServiceTestDB(t)userSvc := NewUserService(db)- settingsSvc, err := NewSettingsService(context.Background(), db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)require.NoError(t, err)s := newTestAuthService("")s.userService = userSvc@@ -470,7 +470,7 @@ func TestRevokeSessionThenVerifyTokenFails(t *testing.T) {func TestRefreshToken_RotatesJTI(t *testing.T) {db := setupAuthServiceTestDB(t)userSvc := NewUserService(db)- settingsSvc, err := NewSettingsService(context.Background(), db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)require.NoError(t, err)s := newTestAuthService("")s.userService = userSvc@@ -499,7 +499,7 @@ func TestRefreshToken_RotatesJTI(t *testing.T) {func TestRefreshToken_RejectsRevokedSession(t *testing.T) {db := setupAuthServiceTestDB(t)userSvc := NewUserService(db)- settingsSvc, err := NewSettingsService(context.Background(), db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)require.NoError(t, err)s := newTestAuthService("")s.userService = userSvc@@ -658,7 +658,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailNotVerified_NoExistingUser_Creatctx := context.Background()db := setupAuthServiceTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))@@ -694,7 +694,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailNotVerified_WithExistingUser_Retctx := context.Background()db := setupAuthServiceTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))@@ -736,7 +736,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailVerificationMissing_WithExistingctx := context.Background()db := setupAuthServiceTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))diff --git a/backend/internal/services/build_workspace_service_test.go b/backend/internal/services/build_workspace_service_test.goindex 7426c5589..491874812 100644--- a/backend/internal/services/build_workspace_service_test.go+++ b/backend/internal/services/build_workspace_service_test.go@@ -12,7 +12,7 @@ import (func TestBuildWorkspaceServiceDeleteFileRejectsCurrentDirectoryInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)root := t.TempDir()diff --git a/backend/internal/services/dashboard_service_test.go b/backend/internal/services/dashboard_service_test.goindex 530d5a2be..727a09439 100644--- a/backend/internal/services/dashboard_service_test.go+++ b/backend/internal/services/dashboard_service_test.go@@ -34,7 +34,7 @@ func setupDashboardServiceTestDB(t *testing.T) (*database.DB, *SettingsService)require.NoError(t, db.AutoMigrate(&models.ApiKey{}, &models.Environment{}, &models.ImageUpdateRecord{}, &models.Project{}, &models.SettingVariable{}))databaseDB := &database.DB{DB: db}- settingsSvc, err := NewSettingsService(context.Background(), databaseDB)+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), databaseDB)require.NoError(t, err)return databaseDB, settingsSvcdiff --git a/backend/internal/services/docker_client_service_test.go b/backend/internal/services/docker_client_service_test.goindex e7ffb871e..f0923ad95 100644--- a/backend/internal/services/docker_client_service_test.go+++ b/backend/internal/services/docker_client_service_test.go@@ -10,6 +10,7 @@ import ("testing""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/events"@@ -17,6 +18,7 @@ import ("github.com/moby/moby/client""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest")func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {@@ -189,6 +191,68 @@ func TestDockerClientService_PublishImageStateResyncNotifiesSubscribers(t *testi}}+func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *testing.T) {+ streamStarted := make(chan struct{})+ streamStopped := make(chan struct{})+ server := newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {+ switch dockerTestPathInternal(r.URL.Path) {+ case "/_ping":+ w.Header().Set("Api-Version", "1.41")+ w.WriteHeader(http.StatusOK)+ case "/events":+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ if flusher, ok := w.(http.Flusher); ok {+ flusher.Flush()+ }+ close(streamStarted)+ <-r.Context().Done()+ close(streamStopped)+ default:+ http.NotFound(w, r)+ }+ })++ lifecycle := fxtest.NewLifecycle(t)+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ service := newDockerClientServiceForTestInternal(server.URL)+ t.Cleanup(service.Close)++ eventsCh, unsubscribe := service.EventBus().Subscribe(events.ImageEventType)+ t.Cleanup(unsubscribe)+ runner, err := actors.NewRunner(t.Context(), actorRuntime, "docker-events-test", "stream", "Docker event watcher", 3, func(ctx context.Context) error {+ service.WatchEvents(ctx)+ return nil+ })+ require.NoError(t, err)++ select {+ case <-streamStarted:+ case <-time.After(time.Second):+ t.Fatal("Docker event stream did not start")+ }+ select {+ case message := <-eventsCh:+ require.Equal(t, dockerImageStateResyncActionInternal, message.Action)+ case <-time.After(time.Second):+ t.Fatal("Docker event stream did not publish reconnect resync")+ }++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, runner.Stop(stopCtx))+ require.Eventually(t, func() bool {+ select {+ case <-streamStopped:+ return true+ default:+ return false+ }+ }, time.Second, time.Millisecond)+ require.NoError(t, lifecycle.Stop(stopCtx))+}+func TestCountImageUsage_UsesContainerImageIDs(t *testing.T) {images := []image.Summary{{ID: "sha256:image-a", Containers: -1},diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.goindex 260df93b1..8527231b5 100644--- a/backend/internal/services/environment_service.go+++ b/backend/internal/services/environment_service.go@@ -15,6 +15,7 @@ import ("emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models"@@ -54,11 +55,9 @@ type EnvironmentService struct {// scheduler and lifecycleCtx are injected post-construction via SetScheduler// (manager-only). Each enabled environment gets its own health-check job; this// replaces the single global environment-health job.- scheduler DynamicScheduler- lifecycleCtx context.Context- // runningHealthChecks guards against a per-environment health check overlapping- // with itself (replaces the old single job's atomic "running" guard).- runningHealthChecks sync.Map+ scheduler schedulertypes.DynamicScheduler+ lifecycleCtx context.Context+ admissionGate *actors.Gate[actors.AdmissionKey]// variableSyncer is injected post-construction via SetVariableSyncer// (manager-only) to avoid a wire cycle with VariableService.@@ -114,18 +113,24 @@ func NewEnvironmentService(db *database.DB, httpClient *http.Client, dockerServi}const environmentHealthJobPrefix = "environment-health:"+const environmentHealthAdmissionScopeInternal = "environment-health"func environmentHealthJobNameInternal(envID string) string { return environmentHealthJobPrefix + envID }// SetScheduler injects the job scheduler and app lifecycle context. Called during// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job// registration becomes a no-op).-func (s *EnvironmentService) SetScheduler(ctx context.Context, scheduler DynamicScheduler) { //nolint:contextcheck // health-check jobs must capture the app lifecycle context, not request contexts+func (s *EnvironmentService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error { //nolint:contextcheck // health-check jobs must capture the app lifecycle context, not request contexts+ if scheduler == nil || admissionGate == nil {+ return errors.New("environment scheduler dependencies unavailable")+ }if ctx == nil {ctx = context.Background()}s.lifecycleCtx = ctxs.scheduler = scheduler+ s.admissionGate = admissionGate+ return nil}// SetVariableSyncer injects the global-variable syncer. Called during@@ -236,22 +241,19 @@ func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {return nil}-func (s *EnvironmentService) acquireHealthLockInternal(envID string) mo.Option[func()] {- if _, loaded := s.runningHealthChecks.LoadOrStore(envID, struct{}{}); loaded {- return mo.None[func()]()- }- return mo.Some(func() { s.runningHealthChecks.Delete(envID) })-}-// runHealthCheckInternal tests one environment's connection (updating its DB status)// and, for online remotes, syncs registries and repositories to it.func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) {- release, ok := s.acquireHealthLockInternal(envID).Get()- if !ok {+ lease, admitted, err := s.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: envID})+ if err != nil {+ slog.ErrorContext(ctx, "environment health check admission failed", "environment_id", envID, "error", err)+ return+ }+ if !admitted {slog.WarnContext(ctx, "environment health check skipped; previous run still in progress", "environment_id", envID)return}- defer release()+ defer lease.Release()status, err := s.TestConnection(ctx, envID, nil)switch {diff --git a/backend/internal/services/environment_service_test.go b/backend/internal/services/environment_service_test.goindex 16fc3656c..b441f2ec7 100644--- a/backend/internal/services/environment_service_test.go+++ b/backend/internal/services/environment_service_test.go@@ -18,6 +18,7 @@ import ("gorm.io/gorm""github.com/coder/websocket"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -28,8 +29,31 @@ import ("github.com/getarcaneapp/arcane/types/v2/environment""github.com/getarcaneapp/arcane/types/v2/gitops""go.getarcane.app/sys/crypto"+ "go.uber.org/fx/fxtest")+func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), actorRuntime, "environment-test-admission", "overlap")+ require.NoError(t, err)++ key := actors.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: "environment-id"}+ lease, admitted, err := gate.TryAcquire(t.Context(), key)+ require.NoError(t, err)+ require.True(t, admitted)++ service := &EnvironmentService{admissionGate: gate}+ service.runHealthCheckInternal(t.Context(), "environment-id")+ lease.Release()++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}+func setupEnvironmentServiceTestDB(t *testing.T) *database.DB {t.Helper()@@ -149,7 +173,7 @@ func TestEnvironmentService_DeleteEnvironment_CascadesGitOpsSyncs(t *testing.T)scheduler := &gitOpsSyncTestSchedulerInternal{}svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)- svc.SetScheduler(ctx, scheduler)+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))require.NoError(t, svc.DeleteEnvironment(ctx, "env-delete-gitops", nil, nil))diff --git a/backend/internal/services/federated_credential_service_test.go b/backend/internal/services/federated_credential_service_test.goindex 18bbc81f1..a73644279 100644--- a/backend/internal/services/federated_credential_service_test.go+++ b/backend/internal/services/federated_credential_service_test.go@@ -133,7 +133,7 @@ func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTestroleSvc := NewRoleService(db)userSvc := NewUserService(db).WithRoleService(roleSvc)sessionSvc := NewSessionService(db)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventSvc := NewEventService(db, &config.Config{}, nil)authSvc := NewAuthService(userSvc, settingsSvc, eventSvc, sessionSvc, roleSvc, "test-federated-secret", &config.Config{diff --git a/backend/internal/services/git_repository_service_test.go b/backend/internal/services/git_repository_service_test.goindex 0df985319..0b9fe7e5c 100644--- a/backend/internal/services/git_repository_service_test.go+++ b/backend/internal/services/git_repository_service_test.go@@ -43,7 +43,7 @@ func setupGitRepositoryServiceTestInternal(t *testing.T) (*GitRepositoryService,})wrappedDB := &database.DB{DB: db}- settingsService, err := NewSettingsService(context.Background(), wrappedDB)+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), wrappedDB)require.NoError(t, err)eventService := NewEventService(wrappedDB, &config.Config{}, nil)diff --git a/backend/internal/services/gitops_sync_service.go b/backend/internal/services/gitops_sync_service.goindex 4f0b0433d..901e3a0e2 100644--- a/backend/internal/services/gitops_sync_service.go+++ b/backend/internal/services/gitops_sync_service.go@@ -11,10 +11,10 @@ import ("path/filepath""slices""strings"- "sync""time""emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models"@@ -27,22 +27,9 @@ import (schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/swarm""github.com/moby/moby/client"- "github.com/samber/mo""gorm.io/gorm")-// DynamicScheduler is the subset of the job scheduler used by services that-// register per-entity jobs at runtime (GitOps syncs, environment health). It is a-// consumer-side interface satisfied by *pkg/scheduler.JobScheduler; the scheduler-// is injected post-construction via SetScheduler because it is created after the-// service graph is built (and pkg/scheduler imports this package, so it cannot be-// a wire input here).-type DynamicScheduler interface {- AddJob(ctx context.Context, job schedulertypes.Job) error- RemoveJob(ctx context.Context, name string)- HasJob(name string) bool-}-type GitOpsSyncService struct {db *database.DBrepoService *GitRepositoryService@@ -52,13 +39,9 @@ type GitOpsSyncService struct {settingsService *SettingsService// scheduler and lifecycleCtx are injected post-construction via SetScheduler.- scheduler DynamicScheduler- lifecycleCtx context.Context- // runningSyncs guards against overlapping PerformSync runs for the same sync.- // The previous single global job serialized syncs implicitly; with independent- // per-sync jobs (plus manual/webhook/startup triggers) a sync could otherwise- // run concurrently with itself and race the clone/redeploy of one project.- runningSyncs sync.Map+ scheduler schedulertypes.DynamicScheduler+ lifecycleCtx context.Context+ admissionGate *actors.Gate[actors.AdmissionKey]}const defaultGitSyncTimeout = 5 * time.Minute@@ -367,6 +350,7 @@ func NewGitOpsSyncService(db *database.DB, repoService *GitRepositoryService, pr}const gitOpsSyncJobPrefix = "gitops-sync:"+const gitOpsSyncAdmissionScopeInternal = "gitops-sync"func gitOpsSyncJobNameInternal(syncID string) string { return gitOpsSyncJobPrefix + syncID }@@ -374,12 +358,17 @@ func gitOpsSyncJobNameInternal(syncID string) string { return gitOpsSyncJobPrefi// called during bootstrap (after the service graph is built) before any per-sync// jobs are registered. The lifecycle context is used for background sync kicks so// they outlive the request/bootstrap goroutine that triggered them.-func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler DynamicScheduler) { //nolint:contextcheck // background sync kicks must capture the app lifecycle context, not request contexts+func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error { //nolint:contextcheck // background sync kicks must capture the app lifecycle context, not request contexts+ if scheduler == nil || admissionGate == nil {+ return errors.New("GitOps scheduler dependencies unavailable")+ }if ctx == nil {ctx = context.Background()}s.lifecycleCtx = ctxs.scheduler = scheduler+ s.admissionGate = admissionGate+ return nil}func (s *GitOpsSyncService) schedulerCtxInternal(ctx context.Context) context.Context {@@ -495,15 +484,6 @@ func isGitOpsSyncOverdueInternal(sync *models.GitOpsSync) bool {return time.Now().After(sync.LastSyncAt.Add(time.Duration(interval) * time.Minute))}-// acquireSyncLockInternal returns a release func when no sync is currently-// running for the given id; otherwise the caller should skip.-func (s *GitOpsSyncService) acquireSyncLockInternal(syncID string) mo.Option[func()] {- if _, loaded := s.runningSyncs.LoadOrStore(syncID, struct{}{}); loaded {- return mo.None[func()]()- }- return mo.Some(func() { s.runningSyncs.Delete(syncID) })-}-func (s *GitOpsSyncService) getEnvironmentSyncLimits(ctx context.Context) (int, int64, int64) {if s.settingsService == nil {return defaultMaxSyncFiles, defaultMaxSyncTotalSize, defaultMaxSyncBinarySize@@ -922,12 +902,15 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id stfunc (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id string, actor models.User) (*gitops.SyncResult, error) {// Coalesce overlapping runs for the same sync (scheduled fire, startup/enable// kick, manual trigger, webhook) so they don't race the clone/redeploy.- release, ok := s.acquireSyncLockInternal(id).Get()- if !ok {+ lease, admitted, err := s.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: id})+ if err != nil {+ return nil, errors.WrapIf(err, "failed to admit GitOps sync")+ }+ if !admitted {slog.InfoContext(ctx, "GitOps sync already in progress; skipping", "syncId", id)return &gitops.SyncResult{Success: false, Message: "sync already in progress", SyncedAt: time.Now()}, nil}- defer release()+ defer lease.Release()syncCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)defer cancel()diff --git a/backend/internal/services/gitops_sync_service_test.go b/backend/internal/services/gitops_sync_service_test.goindex 13a7653de..af547f27d 100644--- a/backend/internal/services/gitops_sync_service_test.go+++ b/backend/internal/services/gitops_sync_service_test.go@@ -9,6 +9,7 @@ import ("time""emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -19,6 +20,7 @@ import (schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest""gorm.io/gorm")@@ -29,7 +31,7 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *datdb := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsDir := t.TempDir()@@ -41,6 +43,31 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *datreturn NewGitOpsSyncService(db, nil, projectService, nil, eventService, settingsService), db, projectsDir}+func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), actorRuntime, "gitops-test-admission", "overlap")+ require.NoError(t, err)++ key := actors.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: "sync-id"}+ lease, admitted, err := gate.TryAcquire(t.Context(), key)+ require.NoError(t, err)+ require.True(t, admitted)++ service := &GitOpsSyncService{admissionGate: gate}+ result, err := service.PerformSync(t.Context(), "0", "sync-id", models.User{})+ require.NoError(t, err)+ require.False(t, result.Success)+ require.Equal(t, "sync already in progress", result.Message)+ lease.Release()++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}+type gitOpsSyncTestSchedulerInternal struct {added []stringremoved []string@@ -178,7 +205,7 @@ func TestGitOpsSyncService_RegisterAutoSyncJobsOnStartup_SkipsOrphans(t *testing}).Error)scheduler := &gitOpsSyncTestSchedulerInternal{}- svc.SetScheduler(ctx, scheduler)+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))svc.RegisterAutoSyncJobsOnStartup(ctx)require.Equal(t, []string{gitOpsSyncJobNameInternal("sync-live")}, scheduler.added)@@ -216,7 +243,7 @@ func TestGitOpsSyncService_DeleteSync_SucceedsWhenEnvironmentMismatched(t *testictx := context.Background()svc, db, _ := setupGitOpsSyncDirectoryTestService(t)scheduler := &gitOpsSyncTestSchedulerInternal{}- svc.SetScheduler(ctx, scheduler)+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))sync := &models.GitOpsSync{BaseModel: models.BaseModel{ID: "sync-env-mismatch"},@@ -273,7 +300,7 @@ func TestGitOpsSyncService_RunScheduledSync_UnregistersMissingSync(t *testing.T)ctx := context.Background()svc, _, _ := setupGitOpsSyncDirectoryTestService(t)scheduler := &gitOpsSyncTestSchedulerInternal{}- svc.SetScheduler(ctx, scheduler)+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))svc.runScheduledSyncInternal(ctx, "0", "ghost-sync")@@ -355,7 +382,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision(func TestGitOpsSyncService_GetOrCreateProject_RefusesDuplicateOnNameCollision(t *testing.T) {ctx := context.Background()svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)- svc.SetScheduler(ctx, &gitOpsSyncTestSchedulerInternal{})+ require.NoError(t, svc.SetScheduler(ctx, &gitOpsSyncTestSchedulerInternal{}, newAdmissionGateForTestInternal(t)))require.NoError(t, os.MkdirAll(filepath.Join(projectsDir, "Dozzle"), 0o755))@@ -1117,7 +1144,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_FailsWhenBoundProjectMissing(t *ctx := context.Background()svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)testScheduler := &gitOpsSyncTestSchedulerInternal{}- svc.SetScheduler(ctx, testScheduler)+ require.NoError(t, svc.SetScheduler(ctx, testScheduler, newAdmissionGateForTestInternal(t)))missingProjectID := "missing-project"sync := &models.GitOpsSync{@@ -1172,7 +1199,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_DisablesAutoSyncWhenBoundProjectctx := context.Background()svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)testScheduler := &gitOpsSyncTestSchedulerInternal{}- svc.SetScheduler(ctx, testScheduler)+ require.NoError(t, svc.SetScheduler(ctx, testScheduler, newAdmissionGateForTestInternal(t)))for _, dirName := range []string{"Radarr-3", "Radarr-30"} {projectPath := filepath.Join(projectsDir, dirName)@@ -1227,7 +1254,7 @@ func TestGitOpsSyncService_GetOrCreateProjectInternal_FailsWhenBoundProjectMissictx := context.Background()svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)testScheduler := &gitOpsSyncTestSchedulerInternal{}- svc.SetScheduler(ctx, testScheduler)+ require.NoError(t, svc.SetScheduler(ctx, testScheduler, newAdmissionGateForTestInternal(t)))missingProjectID := "missing-project"sync := &models.GitOpsSync{@@ -1286,7 +1313,7 @@ func TestEnvContentChangedInternal(t *testing.T) {func TestGitOpsSyncService_GetEnvironmentSyncLimits(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 123))@@ -1309,7 +1336,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "")db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 200))@@ -1364,7 +1391,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {t.Setenv("GIT_SYNC_MAX_FILES", "10000")t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "1024")t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "12")- settingsSvcEnv, svcErr := NewSettingsService(ctx, db)+ settingsSvcEnv, svcErr := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, svcErr)svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}@@ -1385,7 +1412,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {t.Setenv("GIT_SYNC_MAX_FILES", "0")t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "0")t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "0")- settingsSvcEnv, svcErr := NewSettingsService(ctx, db)+ settingsSvcEnv, svcErr := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, svcErr)svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}diff --git a/backend/internal/services/job_service.go b/backend/internal/services/job_service.goindex ce4f90dcb..5a3512db7 100644--- a/backend/internal/services/job_service.go+++ b/backend/internal/services/job_service.go@@ -2,7 +2,6 @@ package servicesimport ("context"- stderrors "errors""log/slog""sort""time"@@ -10,22 +9,13 @@ import ("emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"- "github.com/getarcaneapp/arcane/backend/v2/internal/models"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/types/v2/jobschedule""github.com/getarcaneapp/arcane/types/v2/meta"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/robfig/cron/v3"- "github.com/samber/mo"- "gorm.io/gorm")-type JobRunner interface {- GetJob(jobID string) mo.Option[schedulertypes.Job]- GetJobRuntimeState(jobID string) mo.Option[schedulertypes.JobRuntimeState]- RescheduleJob(ctx context.Context, job schedulertypes.Job) error- RunBusWatcherNow(ctx context.Context, watcherID string) error-}-// JobService manages configuration for background job schedules.//// Intervals are persisted in the existing settings table as individual keys.@@ -38,7 +28,7 @@ type JobService struct {db *database.DBsettings *SettingsServicecfg *config.Config- scheduler JobRunner+ scheduler schedulertypes.JobControllerlifecycleCtx context.Contextlocation *time.Location // Timezone for cron schedule calculations@@ -59,7 +49,7 @@ func NewJobService(db *database.DB, settings *SettingsService, cfg *config.Confi}}-func (s *JobService) SetScheduler(ctx context.Context, scheduler JobRunner) { //nolint:contextcheck // scheduler jobs must capture the app lifecycle context, not request contexts+func (s *JobService) SetScheduler(ctx context.Context, scheduler schedulertypes.JobController) { //nolint:contextcheck // scheduler jobs must capture the app lifecycle context, not request contextsif ctx == nil {ctx = context.Background()}@@ -121,49 +111,33 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule}}- changed := falsechangedKeys := make([]string, 0, len(fields))previousValues := make(map[string]string, len(fields))- upsert := func(tx *gorm.DB, key string, v *string, currentVal string) error {- if v == nil {- return nil- }- if *v == currentVal {- return nil+ valuesToUpdate := make([]libarcane.SettingUpdate, 0, len(fields))+ for _, field := range fields {+ key := field.key+ value := field.update+ currentValue := field.current+ if value == nil || *value == currentValue {+ continue}- changed = truechangedKeys = append(changedKeys, key)- previousValues[key] = currentVal- return tx.Save(&models.SettingVariable{Key: key, Value: *v}).Error+ previousValues[key] = currentValue+ valuesToUpdate = append(valuesToUpdate, libarcane.SettingUpdate{Key: key, Value: *value})}- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- for _, field := range fields {- if err := upsert(tx, field.key, field.update, field.current); err != nil {- return err- }- }- return nil- })- if err != nil {+ if len(valuesToUpdate) == 0 {+ return s.GetJobSchedules(ctx), nil+ }+ if err := s.settings.UpdateSettingValues(ctx, valuesToUpdate); err != nil {return jobschedule.Config{}, errors.WrapIf(err, "failed to update job schedules")}- // Refresh settings cache so jobs reading from SettingsService see new values.- if changed {- if err := s.settings.LoadDatabaseSettings(ctx); err != nil {- restoreErr := s.restoreJobSchedulesInternal(ctx, previousValues, changedKeys)- return jobschedule.Config{}, stderrors.Join(errors.- WrapIf(err, "failed to reload settings after job schedule update"), restoreErr,- )- }-- if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {- restoreErr := s.restoreJobSchedulesInternal(ctx, previousValues, changedKeys)- return jobschedule.Config{}, stderrors.Join(errors.- WrapIf(err, "failed to apply job schedule update"), restoreErr,- )- }+ if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {+ restoreErr := s.restoreJobSchedulesInternal(ctx, previousValues, changedKeys)+ return jobschedule.Config{}, errors.Combine(errors.+ WrapIf(err, "failed to apply job schedule update"), restoreErr,+ )}return s.GetJobSchedules(ctx), nil@@ -196,7 +170,7 @@ func (s *JobService) RescheduleJobsForSettingKeys(ctx context.Context, changedKe}}- return stderrors.Join(rescheduleErrors...)+ return errors.Combine(rescheduleErrors...)}// jobRescheduleContextInternal prefers the app lifecycle context so cron jobs@@ -225,13 +199,13 @@ func (s *JobService) rescheduleAffectedJobInternal(ctx context.Context, jobID st// Continuous bus watchers have no cron entry to reschedule; notify the// watcher so it re-reads its poll schedule instead.if jobMeta.IsContinuous {- if jobID == "image-polling" && s.settings != nil && s.settings.OnImagePollingSettingsChanged != nil {- s.settings.OnImagePollingSettingsChanged(s.jobRescheduleContextInternal(ctx))+ if jobID == "image-polling" && s.settings != nil {+ return s.settings.NotifySettingsChanges(s.jobRescheduleContextInternal(ctx), "pollingInterval")}return nil}- job, ok := s.scheduler.GetJob(jobID).Get()+ job, ok := s.scheduler.GetJob(jobID)if !ok {return errors.Errorf("job %s not found in scheduler", jobID)}@@ -242,7 +216,7 @@ func (s *JobService) rescheduleAffectedJobInternal(ctx context.Context, jobID streturn errors.WrapIff(err, "reschedule job %s", jobID)}- runtimeState, ok := s.scheduler.GetJobRuntimeState(jobID).Get()+ runtimeState, ok := s.scheduler.GetJobRuntimeState(jobID)if !ok {return errors.Errorf("job %s has no runtime scheduler state", jobID)}@@ -259,19 +233,12 @@ func (s *JobService) restoreJobSchedulesInternal(ctx context.Context, previousVareturn nil}- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- for key, value := range previousValues {- if err := tx.Save(&models.SettingVariable{Key: key, Value: value}).Error; err != nil {- return err- }- }- return nil- }); err != nil {- return errors.WrapIf(err, "failed to restore previous job schedules")+ updates := make([]libarcane.SettingUpdate, 0, len(previousValues))+ for key, value := range previousValues {+ updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})}-- if err := s.settings.LoadDatabaseSettings(ctx); err != nil {- return errors.WrapIf(err, "failed to reload restored job schedules")+ if err := s.settings.UpdateSettingValues(ctx, updates); err != nil {+ return errors.WrapIf(err, "failed to restore previous job schedules")}if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {@@ -310,7 +277,7 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponseprerequisites := s.evaluatePrerequisitesInternal(ctx, jobMeta)if s.scheduler != nil && jobMeta.SettingsKey != "" && jobMeta.ID != "environment-health" {- if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID).Get(); ok && runtimeState.Schedule != "" {+ if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok && runtimeState.Schedule != "" {schedule = runtimeState.SchedulenextRun = runtimeState.NextRun}@@ -376,7 +343,7 @@ func (s *JobService) getRunnableJobInternal(jobID string) (schedulertypes.Job, ereturn nil, errors.Errorf("job %s is manager-only and cannot run in agent mode", jobID)}- job, ok := s.scheduler.GetJob(jobID).Get()+ job, ok := s.scheduler.GetJob(jobID)if !ok {return nil, errors.Errorf("job %s not found in scheduler", jobID)}diff --git a/backend/internal/services/job_service_test.go b/backend/internal/services/job_service_test.goindex ef4ba3701..589706064 100644--- a/backend/internal/services/job_service_test.go+++ b/backend/internal/services/job_service_test.go@@ -10,7 +10,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/models""github.com/getarcaneapp/arcane/types/v2/jobschedule"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"- "github.com/samber/mo""github.com/stretchr/testify/require")@@ -18,7 +17,7 @@ func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testinctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{})@@ -32,7 +31,7 @@ func TestJobService_ListJobs_AnalyticsHeartbeatIsManagedInternally(t *testing.T)ctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{})@@ -51,7 +50,7 @@ func TestJobService_ListJobs_IncludesDisabledAutoHealJob(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", false))@@ -68,7 +67,7 @@ func TestJobService_ListJobs_IncludesDockerClientRefreshJob(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{})@@ -87,7 +86,7 @@ func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)nextRun := time.Date(2026, time.July, 10, 8, 0, 0, 0, time.UTC)@@ -112,7 +111,7 @@ func TestJobService_ListJobs_ImageUpdateWatcherIsContinuousAndRespectsEnabled(tctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetBoolSetting(ctx, "pollingEnabled", false))@@ -134,7 +133,7 @@ func TestJobService_UpdateJobSchedules_ReschedulesChangedJob(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{})@@ -153,7 +152,7 @@ func TestJobService_UpdateJobSchedules_DeprecatedPollingIntervalDoesNotReschedulctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{})@@ -172,7 +171,7 @@ func TestJobService_UpdateJobSchedules_UsesLifecycleContextForReschedule(t *testctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)type lifecycleContextKey struct{}@@ -199,7 +198,7 @@ func TestJobService_UpdateJobSchedules_RejectsInvalidCronWithoutChangingSetting(ctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))@@ -220,7 +219,7 @@ func TestJobService_UpdateJobSchedules_UnchangedScheduleDoesNotReschedule(t *tesctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{})@@ -235,7 +234,7 @@ func TestJobService_UpdateJobSchedules_RestoresPreviousScheduleWhenRescheduleFaictx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))@@ -260,7 +259,7 @@ func TestJobService_UpdateJobSchedules_SkipsManagerOnlyJobsInAgentMode(t *testinctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{AgentMode: true})@@ -279,7 +278,7 @@ func TestJobService_UpdateJobSchedules_DelegatesEnvironmentHealthReschedule(t *tctx := context.Background()db := setupSettingsTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)jobSvc := NewJobService(db, settingsSvc, &config.Config{})@@ -347,14 +346,14 @@ func newFakeJobSchedulerInternal(jobIDs ...string) *fakeJobSchedulerInternal {}}-func (s *fakeJobSchedulerInternal) GetJob(jobID string) mo.Option[schedulertypes.Job] {+func (s *fakeJobSchedulerInternal) GetJob(jobID string) (schedulertypes.Job, bool) {job, ok := s.jobs[jobID]- return mo.TupleToOption(job, ok)+ return job, ok}-func (s *fakeJobSchedulerInternal) GetJobRuntimeState(jobID string) mo.Option[schedulertypes.JobRuntimeState] {+func (s *fakeJobSchedulerInternal) GetJobRuntimeState(jobID string) (schedulertypes.JobRuntimeState, bool) {state, ok := s.runtimeStates[jobID]- return mo.TupleToOption(state, ok)+ return state, ok}func (s *fakeJobSchedulerInternal) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {diff --git a/backend/internal/services/lifecycle_service_test.go b/backend/internal/services/lifecycle_service_test.goindex 704006c0f..aaa10fffe 100644--- a/backend/internal/services/lifecycle_service_test.go+++ b/backend/internal/services/lifecycle_service_test.go@@ -32,7 +32,7 @@ func setupLifecycleTestDB(t *testing.T) *database.DB {func newLifecycleTestService(t *testing.T, db *database.DB) (*LifecycleService, *SettingsService) {t.Helper()- settings, err := NewSettingsService(context.Background(), db)+ settings, err := newSettingsServiceForTestInternal(t, context.Background(), db)require.NoError(t, err)events := NewEventService(db, nil, nil)return NewLifecycleService(db, settings, events, nil), settingsdiff --git a/backend/internal/services/oidc_service_test.go b/backend/internal/services/oidc_service_test.goindex 1f2129f6d..277140c64 100644--- a/backend/internal/services/oidc_service_test.go+++ b/backend/internal/services/oidc_service_test.go@@ -61,7 +61,7 @@ func TestGetMobileRedirectAllowlistTrimsWhitespace(t *testing.T) {func TestGetMobileRedirectAllowlistUsesSettings(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)if err != nil {t.Fatalf("NewSettingsService: %v", err)}diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.goindex ff895e1ce..b179c6d4a 100644--- a/backend/internal/services/project_service_test.go+++ b/backend/internal/services/project_service_test.go@@ -70,7 +70,7 @@ func setupProjectTestDB(t *testing.T) *database.DB {func TestProjectService_RefreshProjectImageRefs_PersistsBuildMetadata(t *testing.T) {ctx := context.Background()db := setupProjectTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectPath := t.TempDir()@@ -106,7 +106,7 @@ func TestProjectService_RefreshProjectImageRefs_PersistsBuildMetadata(t *testingfunc TestProjectService_BackfillProjectImageRefs_RetriesOnlyMissingMetadata(t *testing.T) {ctx := context.Background()db := setupProjectTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)validPath := t.TempDir()@@ -167,7 +167,7 @@ func setupProjectDestroyTestServiceInternal(t *testing.T) (*ProjectService, *datctx := context.Background()db := setupProjectTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsDir := t.TempDir()@@ -291,7 +291,7 @@ func TestProjectService_GetProjectFromDatabaseByID(t *testing.T) {ctx := context.Background()// Setup dependencies- settingsService, _ := NewSettingsService(ctx, db)+ settingsService, _ := newSettingsServiceForTestInternal(t, ctx, db)svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())// Create test project@@ -613,7 +613,7 @@ func TestProjectService_PullProjectImages_UpdatesCurrentImageRecordAfterPull(t *projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -711,7 +711,7 @@ func TestProjectService_EnsureImagesPresent_UpdatesCurrentImageRecordAfterPull(tctx := context.Background()db := setupProjectTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)imageRef := "registry.example.com/team/api:2.0.0"@@ -763,7 +763,7 @@ func TestProjectService_PullImageForService_UpdatesCurrentImageRecordAfterPull(tctx := context.Background()db := setupProjectTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)imageRef := "registry.example.com/team/worker:3.1.4"@@ -812,7 +812,7 @@ func TestProjectService_ComposePullSelectedServicesInternal_ReconcilesOnlyOnSuccctx := context.Background()db := setupProjectTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)privateImageRef := "registry.example.com/team/app:9.9.9"@@ -932,7 +932,7 @@ func TestProjectService_ComposePullSelectedServicesInternal_LeavesRecordsWhenPulctx := context.Background()db := setupProjectTestDB(t)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)imageRef := "registry.example.com/team/app:9.9.9"@@ -991,7 +991,7 @@ func TestProjectService_UpdateProjectServicesHardFailsWhenPullFailsInternal(t *tprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1061,7 +1061,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.TprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1151,7 +1151,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenVolumeMigrationPreparationFprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1221,7 +1221,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_AppliesVolumeMigratiprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -1268,7 +1268,7 @@ func TestProjectService_PrepareProjectRenameVolumeMigrationForUpdate_UsesComposeprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1383,7 +1383,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_RollsBackVolumeMigraprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -1432,7 +1432,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_SucceedsCommittedRenprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -1482,7 +1482,7 @@ func TestProjectService_UpdateProject_ClearsJournalForNonRenameWhenRecoveryDockeprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1548,7 +1548,7 @@ func TestProjectService_UpdateProject_AllowsRenameAfterJournalRecoveryDockerUnavprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1611,7 +1611,7 @@ func TestProjectService_UpdateProject_RenamesDirectoryWhenNameChanges(t *testingprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -1660,7 +1660,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenTargetDirectoryExists(t *teprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -1707,7 +1707,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenProjectRunning(t *testing.TprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -1751,7 +1751,7 @@ func TestProjectService_UpdateProject_RenameRejectsStaleStoppedWhenRuntimeIsRunnprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1809,7 +1809,7 @@ func TestProjectService_UpdateProject_RenameResolvesUnknownStoppedStatusBeforeVoprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1867,7 +1867,7 @@ func TestProjectService_UpdateProject_RenameRejectsUnknownWhenRuntimeIsRunning(tprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -1931,7 +1931,7 @@ func TestProjectService_UpdateProject_ValidatesComposeUsingExistingProjectName(tprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -1974,7 +1974,7 @@ func TestProjectService_UpdateProject_AllowsMissingEnvFileDuringComposeValidatioprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2019,7 +2019,7 @@ func TestProjectService_UpdateProject_AllowsMissingLocalIncludeDuringComposeValiprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2084,7 +2084,7 @@ func TestProjectService_UpdateProject_RejectsMissingExternalIncludeDuringComposeprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2129,7 +2129,7 @@ func TestProjectService_CreateProject_RejectsExternalInclude(t *testing.T) {projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2165,7 +2165,7 @@ func TestProjectService_CreateProject_RejectsArrayPathInclude(t *testing.T) {projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2203,7 +2203,7 @@ func TestProjectService_CreateProject_WritesStagedProjectFiles(t *testing.T) {projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2257,7 +2257,7 @@ func TestProjectService_GetProjectDetails_UsesFileTreeMaxDepthForProjectFiles(tt.Setenv("PROJECT_SCAN_MAX_DEPTH", "3")t.Setenv("PROJECT_FILE_TREE_MAX_DEPTH", "8")- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2300,7 +2300,7 @@ func TestProjectService_UpdateProject_AppliesStagedProjectFileChanges(t *testingprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2343,7 +2343,7 @@ func TestProjectService_UpdateProject_RejectsStaleProjectFileRevision(t *testingprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2380,7 +2380,7 @@ func TestProjectService_UpdateProject_RejectsStaleDeepProjectFileRevision(t *test.Setenv("PROJECT_SCAN_MAX_DEPTH", "3")t.Setenv("PROJECT_FILE_TREE_MAX_DEPTH", "8")- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2419,7 +2419,7 @@ func TestProjectService_GetProjectFileContent_RejectsExternalInclude(t *testing.projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2460,7 +2460,7 @@ func TestProjectService_GetProjectFileContent_RejectsSymlinkInclude(t *testing.TprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2504,7 +2504,7 @@ func TestProjectService_GetProjectFileContent_RejectsIntermediateSymlinkInclude(projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2548,7 +2548,7 @@ func TestProjectService_GetProjectFileContent_RejectsIntermediateSymlinkProjectFprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2585,7 +2585,7 @@ func TestProjectService_UpdateProject_UsesExistingEnvFileDuringComposeValidationprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2635,7 +2635,7 @@ func newProjectServiceForOverrideTestInternal(t *testing.T, dirName, baseComposeprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2746,7 +2746,7 @@ func TestProjectService_UpdateProject_UsesProvidedEnvContentDuringComposeValidatprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2793,7 +2793,7 @@ func TestProjectService_UpdateProject_ReturnsEnvParseErrorDuringComposeValidatioprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2838,7 +2838,7 @@ func TestProjectService_UpdateProject_UsesGlobalEnvDuringComposeValidation(t *teprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2887,7 +2887,7 @@ func TestProjectService_UpdateProject_DoesNotResolveHostEnvThroughGlobalEnvDurint.Setenv("PROJECTS_DIRECTORY", projectsDir)t.Setenv("HOST_ONLY_PATH", "/host/secret")- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2931,7 +2931,7 @@ func TestProjectService_UpdateProject_DerivesProjectOverrideEnvWhenGitSourceExisprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -2978,7 +2978,7 @@ func TestProjectService_UpdateProject_UnchangedGitEnvLeavesFilesUntouched(t *tesprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3077,7 +3077,7 @@ func TestProjectService_UpdateProject_DeletingGitBackedKeyFallsBackToGit(t *testprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3127,7 +3127,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_MigratesDirectEnvIntoProjectOveprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3178,7 +3178,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_PreservesGitEnvSyntax(t *testinprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3239,7 +3239,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_NormalizesStaleCopiedGitOverridprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3287,7 +3287,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_RemovesLegacyDeletedGitMasks(tprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3336,7 +3336,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_RemovesGitEnvSource(t *testing.projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3380,7 +3380,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_WritesAndRemovesComposeOverrideprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3431,7 +3431,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_UsesGlobalEnvDuringComposeValidprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3482,7 +3482,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_TolerantOfUndefinedComposeVar(tprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3530,7 +3530,7 @@ func TestProjectService_PersistGitSyncEnvFiles_UsesPreparedState(t *testing.T) {projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3568,7 +3568,7 @@ func TestProjectService_GetProjectDetails_ReturnsEffectiveEnvContent(t *testing.projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)@@ -3694,7 +3694,7 @@ func TestProjectService_GetProjectDetails_IncludesUpdateInfo(t *testing.T) {projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -3742,7 +3742,7 @@ func TestProjectService_GetProjectDetails_RefreshesRuntimeStatusWithoutRuntimeSeprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -3815,7 +3815,7 @@ func TestProjectService_GetProjectDetails_PopulatesRuntimeServicesFromComposePs(projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -3869,7 +3869,7 @@ func TestProjectService_ListProjects_FiltersByUpdateStatus(t *testing.T) {projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -4081,7 +4081,7 @@ func TestProjectService_ListProjects_FiltersArchivedProjects(t *testing.T) {db := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))@@ -4143,7 +4143,7 @@ func TestProjectService_ArchiveProject_RequiresStoppedProject(t *testing.T) {ctx := context.Background()projectsRoot := t.TempDir()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))@@ -4173,7 +4173,7 @@ func TestProjectService_ArchiveProject_TogglesArchiveFlag(t *testing.T) {ctx := context.Background()projectsRoot := t.TempDir()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))@@ -4310,7 +4310,7 @@ func TestProjectService_ListProjects_WithDerivedStatusFilter_AllowsAllPageSizeSedb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4542,7 +4542,7 @@ func TestProjectService_PrepareServiceBuildRequest_GeneratedImageProviderGuardrafunc TestProjectService_DeployProject_StopsOnBuildPreparationError(t *testing.T) {db := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4580,7 +4580,7 @@ func TestProjectService_DeployProject_StopsOnBuildPreparationError(t *testing.T)func TestProjectService_DeployProject_BuildsGeneratedImageWithoutPull(t *testing.T) {db := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4621,7 +4621,7 @@ func TestProjectService_SyncProjectsFromFileSystem_IgnoresSymlinkedProjectDirsWhdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4647,7 +4647,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DetectsSymlinkedProjectDirsWhdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4673,7 +4673,7 @@ func TestProjectService_CountProjectFolders_RespectsFollowProjectSymlinks(t *tesdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4701,7 +4701,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversNestedProjectsAndReldb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4734,7 +4734,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RespectsConfiguredScanMaxDeptdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4758,7 +4758,7 @@ func TestProjectService_ListProjects_LoadsProjectIconFromGlobalEnvInIncludedMetadb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4809,7 +4809,7 @@ func TestProjectService_CountProjectFolders_RecursivelyCountsNestedProjects(t *tdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4830,7 +4830,7 @@ func TestProjectService_CountProjectFolders_RespectsConfiguredScanMaxDepth(t *tedb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4851,7 +4851,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(tdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4882,7 +4882,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PrunesLeakedScratchRow(t *tesdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4912,7 +4912,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectsWhenDirectordb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// An existing-but-empty projects directory simulates a mis-mapped or unmounted@@ -4947,7 +4947,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectWithAmbiguousdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -4983,7 +4983,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesProjectsBeyondReducedSdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5029,7 +5029,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesDBRecordsWhenDirectodb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5078,7 +5078,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversReadableProjectsDespdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5129,7 +5129,7 @@ func TestProjectService_SyncProjectsFromFileSystem_AllowsDuplicateLeafDirectoriedb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5157,7 +5157,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DetectsNestedSymlinkedProjectdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5188,7 +5188,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesSymlinkedProjectsWhenDdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5222,7 +5222,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RefreshesServiceCountOnComposdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5249,7 +5249,7 @@ func TestProjectService_SyncProjectsFromFileSystem_AlignsNameToEffectiveComposeNdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5278,7 +5278,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesValidCustomNameWithodb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5311,7 +5311,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesGitOpsProjectWithCusctx := context.Background()require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5361,7 +5361,7 @@ func TestProjectService_GetProjectDetails_UsesGitOpsCustomComposeFilename(t *tesctx := context.Background()require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5414,7 +5414,7 @@ func TestProjectService_UpdateProject_WritesThroughSymlinkedProjectPath(t *testidb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5463,7 +5463,7 @@ func TestProjectService_UpdateProject_WritesThroughExternalEnvSymlink(t *testingdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5521,7 +5521,7 @@ func TestProjectService_UpdateProject_RestoresExternalEnvSymlinkTargetWhenProjecdb := setupProjectTestDB(t)ctx := context.Background()- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)projectsRoot := t.TempDir()@@ -5690,7 +5690,7 @@ func TestProjectService_UpdateProject_RenameRollsBackWhenFileRevisionIsStale(t *projectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))@@ -7492,7 +7492,7 @@ func TestProjectService_UpdateProject_FileChangeFailureRollsBackScoped(t *testinprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))diff --git a/backend/internal/services/project_service_unix_test.go b/backend/internal/services/project_service_unix_test.goindex ae53da9c9..a53d883f0 100644--- a/backend/internal/services/project_service_unix_test.go+++ b/backend/internal/services/project_service_unix_test.go@@ -29,7 +29,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_TolerantOfPermissionLockedEnv(tprojectsDir := t.TempDir()t.Setenv("PROJECTS_DIRECTORY", projectsDir)- settingsService, err := NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)eventService := NewEventService(db, nil, nil)diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.goindex 7eea62b58..01c3ea87d 100644--- a/backend/internal/services/settings_service.go+++ b/backend/internal/services/settings_service.go@@ -21,6 +21,7 @@ import ("github.com/samber/mo""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models"@@ -35,15 +36,15 @@ type SettingsService struct {db *database.DBconfig atomic.Pointer[models.Settings]envOverrides []settingsEnvOverride+ writes *actors.Executor+ effects *actors.Executor+ lifecycleCtx context.Context+ changes *actors.Signal[[]libarcane.SettingUpdate]+}- OnImagePollingSettingsChanged func(ctx context.Context)- OnAutoUpdateSettingsChanged func(ctx context.Context)- OnProjectsDirectoryChanged func(ctx context.Context)- OnTemplatesDirectoryChanged func(ctx context.Context)- OnScheduledPruneSettingsChanged func(ctx context.Context)- OnVulnerabilityScanSettingsChanged func(ctx context.Context)- OnAutoHealSettingsChanged func(ctx context.Context)- OnTimeoutSettingsChanged func(ctx context.Context, timeoutSettings []libarcane.SettingUpdate)+type settingsUpdateResultInternal struct {+ settings []models.SettingVariable+ changes []libarcane.SettingUpdate}type settingsEnvOverride struct {@@ -53,9 +54,22 @@ type settingsEnvOverride struct {value string}-func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService, error) {+func NewSettingsService(ctx context.Context, db *database.DB, writes, effects *actors.Executor) (*SettingsService, error) {+ if ctx == nil {+ return nil, errors.New("settings lifecycle context unavailable")+ }+ if writes == nil {+ return nil, errors.New("settings write executor unavailable")+ }+ if effects == nil {+ return nil, errors.New("settings effects executor unavailable")+ }svc := &SettingsService{- db: db,+ db: db,+ writes: writes,+ effects: effects,+ lifecycleCtx: ctx,+ changes: actors.NewSignal[[]libarcane.SettingUpdate](),}svc.envOverrides = resolveSettingsEnvOverridesInternal()if len(svc.envOverrides) > 0 {@@ -75,6 +89,58 @@ func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService,return svc, nil}+// SubscribeSettingsChanges invokes callback once when any subscribed key is+// present in a successful update. The callback receives only matching values.+// Callbacks run serially on the settings-effects actor, outside the write actor.+func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func() {+ if callback == nil || len(keys) == 0 {+ return func() {}+ }+ subscribed := make(map[string]struct{}, len(keys))+ for _, key := range keys {+ subscribed[key] = struct{}{}+ }+ return s.changes.Subscribe(func(updates []libarcane.SettingUpdate) {+ matching := make([]libarcane.SettingUpdate, 0, len(updates))+ for _, update := range updates {+ if _, ok := subscribed[update.Key]; ok {+ matching = append(matching, update)+ }+ }+ if len(matching) > 0 {+ _, err := actors.Submit(context.WithoutCancel(s.lifecycleCtx), s.effects, "apply settings change effects", func(context.Context) (actors.NoPayload, error) {+ callback(matching)+ return actors.NoPayload{}, nil+ }, nil)+ if err != nil && s.lifecycleCtx.Err() == nil {+ slog.ErrorContext(s.lifecycleCtx, "Failed to queue settings change effects", "error", err)+ }+ }+ })+}++// NotifySettingsChanges publishes current values for keys through the settings+// actor. It is used when another settings-table workflow performs persistence.+func (s *SettingsService) NotifySettingsChanges(ctx context.Context, keys ...string) error {+ _, err := actors.Execute(ctx, s.writes, "notify settings changes", func(context.Context) ([]libarcane.SettingUpdate, error) {+ updates := make([]libarcane.SettingUpdate, 0, len(keys))+ cfg := s.GetSettingsConfig()+ for _, key := range keys {+ value, _, _, err := cfg.FieldByKey(key)+ if err != nil {+ return nil, errors.WrapIff(err, "load changed setting '%s'", key)+ }+ updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})+ }+ return updates, nil+ }, func(updates []libarcane.SettingUpdate, err error) {+ if err == nil && len(updates) > 0 {+ s.publishSettingsChangesInternal(updates)+ }+ })+ return err+}+func (s *SettingsService) GetSettingsConfig() *models.Settings {v := s.config.Load()if v == nil {@@ -387,11 +453,32 @@ func (s *SettingsService) getEffectiveSettingsConfigInternal(ctx context.Context}func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {- if err := s.updateSettingValueNoRefreshInternal(ctx, key, value); err != nil {- return err+ if err := libarcane.ValidateCronSetting(key, value); err != nil {+ return errors.WrapIff(err, "invalid cron expression for %s", key)}+ _, err := actors.Execute(ctx, s.writes, "update setting", func(writeCtx context.Context) (actors.NoPayload, error) {+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, key, value); err != nil {+ return actors.NoPayload{}, err+ }+ return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))+ }, nil)+ return err+}- return s.refreshSettingsCacheInternal(ctx)+// UpdateSettingValues persists a group of internal setting values atomically+// and publishes the refreshed snapshot before returning.+func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []libarcane.SettingUpdate) error {+ values := make([]models.SettingVariable, 0, len(updates))+ for _, update := range updates {+ values = append(values, models.SettingVariable{Key: update.Key, Value: update.Value})+ }+ _, err := actors.Execute(ctx, s.writes, "update setting values", func(writeCtx context.Context) (actors.NoPayload, error) {+ if err := s.persistSettings(writeCtx, values); err != nil {+ return actors.NoPayload{}, err+ }+ return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))+ }, nil)+ return err}func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Context, key, value string) error {@@ -404,7 +491,38 @@ func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Contex})}+// UpdateSettings publishes the refreshed snapshot before returning. Each+// subscriber's effects remain ordered and may finish after this method returns.func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Update) ([]models.SettingVariable, error) {+ result, err := actors.Execute(ctx, s.writes, "update settings", func(writeCtx context.Context) (settingsUpdateResultInternal, error) {+ return s.updateSettingsInternal(writeCtx, updates)+ }, func(result settingsUpdateResultInternal, err error) {+ if err == nil && len(result.changes) > 0 {+ s.publishSettingsChangesInternal(result.changes)+ }+ })+ return result.settings, err+}++func (s *SettingsService) publishSettingsChangesInternal(updates []libarcane.SettingUpdate) {+ safe := make([]libarcane.SettingUpdate, 0, len(updates))+ cfg := s.GetSettingsConfig()+ for _, update := range updates {+ _, _, sensitive, err := cfg.FieldByKey(update.Key)+ if err != nil {+ slog.WarnContext(s.lifecycleCtx, "Skipping settings change notification for unknown key", "key", update.Key, "error", err)+ continue+ }+ if !sensitive {+ safe = append(safe, update)+ }+ }+ if len(safe) > 0 {+ s.changes.Publish(safe)+ }+}++func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates settings.Update) (settingsUpdateResultInternal, error) {defaultCfg := s.getDefaultSettings()cfg := s.GetSettingsConfig().Clone()trivyServerTokenUpdated := updates.TrivyServerToken != nil && *updates.TrivyServerToken != "" && !s.isEnvOverrideActiveInternal("trivyServerToken")@@ -424,7 +542,7 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Umap[string]bool{"oidcClientSecret": cfg.OidcClientSecret.Value != ""},map[string]bool{"oidcClientSecret": updates.OidcClientSecret != nil && *updates.OidcClientSecret != ""},); err != nil {- return nil, err+ return settingsUpdateResultInternal{}, err}if err := validation.ValidateCredentialTargetChange(@@ -435,12 +553,12 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Umap[string]bool{"trivyServerToken": cfg.TrivyServerToken.Value != ""},map[string]bool{"trivyServerToken": trivyServerTokenUpdated},); err != nil {- return nil, err+ return settingsUpdateResultInternal{}, err}- valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)+ valuesToUpdate, err := s.prepareUpdateValues(updates, cfg, defaultCfg)if err != nil {- return nil, err+ return settingsUpdateResultInternal{}, err}if updates.OidcClientSecret != nil && *updates.OidcClientSecret != "" {valuesToUpdate = append(valuesToUpdate, models.SettingVariable{Key: "oidcClientSecret", Value: *updates.OidcClientSecret})@@ -450,59 +568,28 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U}if err := s.persistSettings(ctx, valuesToUpdate); err != nil {- return nil, err+ return settingsUpdateResultInternal{}, err}- if err := s.refreshSettingsCacheInternal(ctx); err != nil {- return nil, err+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); err != nil {+ return settingsUpdateResultInternal{}, err}settingsCfg := s.GetSettingsConfig()-- // Now call callbacks after in-memory config is updated- if changedPolling && s.OnImagePollingSettingsChanged != nil {- s.OnImagePollingSettingsChanged(ctx)- }- if changedAutoUpdate && s.OnAutoUpdateSettingsChanged != nil {- s.OnAutoUpdateSettingsChanged(ctx)- }- if changedScheduledPrune && s.OnScheduledPruneSettingsChanged != nil {- s.OnScheduledPruneSettingsChanged(ctx)- }- if changedVulnerabilityScan && s.OnVulnerabilityScanSettingsChanged != nil {- s.OnVulnerabilityScanSettingsChanged(ctx)- }- if changedAutoHeal && s.OnAutoHealSettingsChanged != nil {- s.OnAutoHealSettingsChanged(ctx)- }- if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool {- return sv.Key == "projectsDirectory" || sv.Key == "followProjectSymlinks"- }) && s.OnProjectsDirectoryChanged != nil {- s.OnProjectsDirectoryChanged(ctx)- }- if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool {- return sv.Key == "templatesDirectory"- }) && s.OnTemplatesDirectoryChanged != nil {- s.OnTemplatesDirectoryChanged(ctx)- }- if len(changedTimeouts) > 0 && s.OnTimeoutSettingsChanged != nil {- s.OnTimeoutSettingsChanged(ctx, changedTimeouts)- }-- return settingsCfg.ToSettingVariableSlice(models.SettingVisibilityNonAdmin, false), nil+ changes := make([]libarcane.SettingUpdate, 0, len(valuesToUpdate))+ for _, value := range valuesToUpdate {+ changes = append(changes, libarcane.SettingUpdate{Key: value.Key, Value: value.Value})+ }+ return settingsUpdateResultInternal{+ settings: settingsCfg.ToSettingVariableSlice(models.SettingVisibilityNonAdmin, false),+ changes: changes,+ }, nil}-func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, bool, bool, bool, bool, bool, []libarcane.SettingUpdate, error) {+func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, error) {rt := reflect.TypeFor[settings.Update]()rv := reflect.ValueOf(updates)valuesToUpdate := make([]models.SettingVariable, 0)- changedPolling := false- changedAutoUpdate := false- changedScheduledPrune := false- changedVulnerabilityScan := false- changedAutoHeal := false- changedTimeouts := make([]libarcane.SettingUpdate, 0)-for i := range rt.NumField() {field := rt.Field(i)fieldValue := rv.Field(i)@@ -520,19 +607,16 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa}if err := cfg.UpdateField(key, value, false); err != nil {- return nil, false, false, false, false, false, nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)+ return nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)}valuesToUpdate = append(valuesToUpdate, models.SettingVariable{Key: key, Value: value})- if libarcane.IsTimeoutSettingKey(key) {- changedTimeouts = append(changedTimeouts, libarcane.SettingUpdate{Key: key, Value: value})- }continue}if err := libarcane.ValidateCronSetting(key, value); err != nil {- return nil, false, false, false, false, false, nil, errors.WrapIff(err, "invalid cron expression for %s", key)+ return nil, errors.WrapIff(err, "invalid cron expression for %s", key)}var valueToSave string@@ -551,31 +635,13 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defacontinue}if err != nil {- return nil, false, false, false, false, false, nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)+ return nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)}valuesToUpdate = append(valuesToUpdate, models.SettingVariable{Key: key, Value: valueToSave})-- switch key {- case "pollingEnabled", "pollingInterval":- changedPolling = true- case "autoUpdate", "autoUpdateInterval":- changedAutoUpdate = true- case "scheduledPruneEnabled",- "scheduledPruneInterval":- changedScheduledPrune = true- case "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyNetwork", "trivySecurityOpts", "trivyPrivileged", "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers":- changedVulnerabilityScan = true- case "autoHealEnabled", "autoHealInterval", "autoHealExcludedContainers", "autoHealMaxRestarts", "autoHealRestartWindow":- changedAutoHeal = true- }-- if libarcane.IsTimeoutSettingKey(key) {- changedTimeouts = append(changedTimeouts, libarcane.SettingUpdate{Key: key, Value: valueToSave})- }}- return valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, nil+ return valuesToUpdate, nil}func extractUpdateValue(field reflect.StructField, fieldValue reflect.Value) (string, string, bool) {@@ -608,76 +674,80 @@ func (s *SettingsService) persistSettings(ctx context.Context, values []models.S}func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {- defaultSettings := s.getDefaultSettings()- defaultSettingVars := defaultSettings.ToSettingVariableSlice(models.SettingVisibilityAll, false)-- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- for _, defaultSetting := range defaultSettingVars {- var existing models.SettingVariable- err := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error-- switch {- case errors.Is(err, gorm.ErrRecordNotFound):- if err := tx.Create(&defaultSetting).Error; err != nil {- return errors.WrapIff(err, "failed to create default setting %s", defaultSetting.Key)- }- case err != nil:- return errors.WrapIff(err, "failed to check for existing setting %s", defaultSetting.Key)- case defaultSetting.Key == "trivyImage" && existing.Value != defaultSetting.Value:- if err := tx.Model(&existing).Update("value", defaultSetting.Value).Error; err != nil {- return errors.WrapIff(err, "failed to enforce default setting %s", defaultSetting.Key)+ _, err := actors.Execute(ctx, s.writes, "ensure default settings", func(writeCtx context.Context) (actors.NoPayload, error) {+ defaultSettings := s.getDefaultSettings()+ defaultSettingVars := defaultSettings.ToSettingVariableSlice(models.SettingVisibilityAll, false)++ if err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {+ for _, defaultSetting := range defaultSettingVars {+ var existing models.SettingVariable+ err := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error++ switch {+ case errors.Is(err, gorm.ErrRecordNotFound):+ if err := tx.Create(&defaultSetting).Error; err != nil {+ return errors.WrapIff(err, "failed to create default setting %s", defaultSetting.Key)+ }+ case err != nil:+ return errors.WrapIff(err, "failed to check for existing setting %s", defaultSetting.Key)+ case defaultSetting.Key == "trivyImage" && existing.Value != defaultSetting.Value:+ if err := tx.Model(&existing).Update("value", defaultSetting.Value).Error; err != nil {+ return errors.WrapIff(err, "failed to enforce default setting %s", defaultSetting.Key)+ }}}+ return nil+ }); err != nil {+ return actors.NoPayload{}, err}- return nil- }); err != nil {- return err- }-- return nil+ return actors.NoPayload{}, nil+ }, nil)+ return err}func (s *SettingsService) PruneUnknownSettings(ctx context.Context) error {- allowedKeys := allowedSettingKeys()- if len(allowedKeys) == 0 {- return nil- }-- keys := make([]string, 0, len(allowedKeys))- for key := range allowedKeys {- keys = append(keys, key)- }-- result := s.db.WithContext(ctx).Where("key NOT IN ?", keys).Delete(&models.SettingVariable{})- if result.Error != nil {- return errors.WrapIf(result.Error, "failed to prune unknown settings")- }+ _, err := actors.Execute(ctx, s.writes, "prune unknown settings", func(writeCtx context.Context) (actors.NoPayload, error) {+ allowedKeys := allowedSettingKeys()+ if len(allowedKeys) == 0 {+ return actors.NoPayload{}, nil+ }- if result.RowsAffected > 0 {- slog.InfoContext(ctx, "Pruned unknown settings", "count", result.RowsAffected)- }+ keys := make([]string, 0, len(allowedKeys))+ for key := range allowedKeys {+ keys = append(keys, key)+ }- return nil+ result := s.db.WithContext(writeCtx).Where("key NOT IN ?", keys).Delete(&models.SettingVariable{})+ if result.Error != nil {+ return actors.NoPayload{}, errors.WrapIf(result.Error, "failed to prune unknown settings")+ }+ if result.RowsAffected > 0 {+ slog.InfoContext(writeCtx, "Pruned unknown settings", "count", result.RowsAffected)+ }+ return actors.NoPayload{}, nil+ }, nil)+ return err}func (s *SettingsService) PersistEnvSettingsIfMissing(ctx context.Context) error {- rt := reflect.TypeFor[models.Settings]()- appCfg := config.Load()- isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled-- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- for field := range rt.Fields() {- if err := s.processEnvField(ctx, tx, field, isEnvOnlyMode); err != nil {- return err+ _, err := actors.Execute(ctx, s.writes, "persist environment settings", func(writeCtx context.Context) (actors.NoPayload, error) {+ rt := reflect.TypeFor[models.Settings]()+ appCfg := config.Load()+ isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled++ if err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {+ for field := range rt.Fields() {+ if err := s.processEnvField(writeCtx, tx, field, isEnvOnlyMode); err != nil {+ return err+ }}+ return nil+ }); err != nil {+ return actors.NoPayload{}, err}- return nil- }); err != nil {- return err- }-- // Reload settings after persisting env vars- return s.LoadDatabaseSettings(ctx)+ return actors.NoPayload{}, s.LoadDatabaseSettings(context.WithoutCancel(writeCtx))+ }, nil)+ return err}func allowedSettingKeys() map[string]struct{} {@@ -842,86 +912,90 @@ func (s *SettingsService) SetStringSetting(ctx context.Context, key, value strin// the autoUpdateExcludedContainers setting. When excluded is true the container// is added to the list; when false it is removed.func (s *SettingsService) SetContainerAutoUpdateExclusionInternal(ctx context.Context, containerName string, excluded bool) error {- raw := s.GetStringSetting(ctx, "autoUpdateExcludedContainers", "")- existing := make(map[string]struct{})- var ordered []string- for part := range strings.SplitSeq(raw, ",") {- name := strings.TrimSpace(part)- if name == "" {- continue- }- if _, ok := existing[name]; !ok {- existing[name] = struct{}{}- ordered = append(ordered, name)+ _, err := actors.Execute(ctx, s.writes, "update container auto-update exclusion", func(writeCtx context.Context) (actors.NoPayload, error) {+ raw := s.GetStringSetting(writeCtx, "autoUpdateExcludedContainers", "")+ existing := make(map[string]struct{})+ var ordered []string+ for part := range strings.SplitSeq(raw, ",") {+ name := strings.TrimSpace(part)+ if name == "" {+ continue+ }+ if _, ok := existing[name]; !ok {+ existing[name] = struct{}{}+ ordered = append(ordered, name)+ }}- }- if excluded {- if _, ok := existing[containerName]; !ok {- ordered = append(ordered, containerName)- }- } else {- filtered := ordered[:0]- for _, name := range ordered {- if name != containerName {- filtered = append(filtered, name)+ if excluded {+ if _, ok := existing[containerName]; !ok {+ ordered = append(ordered, containerName)}+ } else {+ filtered := ordered[:0]+ for _, name := range ordered {+ if name != containerName {+ filtered = append(filtered, name)+ }+ }+ ordered = filtered}- ordered = filtered- }- return s.SetStringSetting(ctx, "autoUpdateExcludedContainers", strings.Join(ordered, ","))+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, "autoUpdateExcludedContainers", strings.Join(ordered, ",")); err != nil {+ return actors.NoPayload{}, err+ }+ return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))+ }, nil)+ return err}func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, error) {- const keyName = "encryptionKey"- var key string+ return actors.Execute(ctx, s.writes, "ensure encryption key", func(writeCtx context.Context) (string, error) {+ const keyName = "encryptionKey"+ var key string- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- var sv models.SettingVariable- err := tx.Where("key = ?", keyName).First(&sv).Error+ err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {+ var sv models.SettingVariable+ err := tx.Where("key = ?", keyName).First(&sv).Error- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {- return errors.WrapIf(err, "failed to load encryption key")- }+ if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {+ return errors.WrapIf(err, "failed to load encryption key")+ }- // If already present and non-empty, return it- if sv.Value != "" {- key = sv.Value- return nil- }+ if sv.Value != "" {+ key = sv.Value+ return nil+ }- notFound := errors.Is(err, gorm.ErrRecordNotFound)+ notFound := errors.Is(err, gorm.ErrRecordNotFound)+ u, genErr := uuid.NewRandom()+ if genErr != nil {+ return errors.WrapIf(genErr, "failed to generate encryption key")+ }+ sum := sha256.Sum256([]byte(u.String()))+ generatedKey := base64.StdEncoding.EncodeToString(sum[:])+ key = generatedKey- // Generate uuid -> sha256 -> base64 key (32 bytes raw -> 44 chars base64)- u, genErr := uuid.NewRandom()- if genErr != nil {- return errors.WrapIf(genErr, "failed to generate encryption key")- }- sum := sha256.Sum256([]byte(u.String()))- generatedKey := base64.StdEncoding.EncodeToString(sum[:])- key = generatedKey+ if notFound {+ if createErr := tx.Create(&models.SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {+ return errors.WrapIf(createErr, "failed to persist encryption key")+ }+ return nil+ }- if notFound {- if createErr := tx.Create(&models.SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {- return errors.WrapIf(createErr, "failed to persist encryption key")+ if updErr := tx.Model(&models.SettingVariable{}).+ Where("key = ?", keyName).+ Update("value", generatedKey).Error; updErr != nil {+ return errors.WrapIf(updErr, "failed to update encryption key")}return nil+ })+ if err != nil {+ return "", err}- // Record existed but empty value; update it- if updErr := tx.Model(&models.SettingVariable{}).- Where("key = ?", keyName).- Update("value", generatedKey).Error; updErr != nil {- return errors.WrapIf(updErr, "failed to update encryption key")- }- return nil- })- if err != nil {- return "", err- }-- return key, nil+ return key, nil+ }, nil)}func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projectsDirEnv string) error {@@ -930,50 +1004,44 @@ func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projecreturn nil}- var projectsDirSetting models.SettingVariable- err := s.db.WithContext(ctx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error-- if errors.Is(err, gorm.ErrRecordNotFound) {- slog.DebugContext(ctx, "No projectsDirectory setting found, skipping normalization")- return nil- }-- if err != nil {- return errors.WrapIf(err, "failed to load projectsDirectory setting")- }+ _, err := actors.Execute(ctx, s.writes, "normalize projects directory", func(writeCtx context.Context) (string, error) {+ var projectsDirSetting models.SettingVariable+ err := s.db.WithContext(writeCtx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ slog.DebugContext(writeCtx, "No projectsDirectory setting found, skipping normalization")+ return "", nil+ }+ if err != nil {+ return "", errors.WrapIf(err, "failed to load projectsDirectory setting")+ }- value := strings.TrimSpace(projectsDirSetting.Value)- // Detect mapping format (container:host), allowing Windows or Unix container paths.- isMapping := false- if strings.Contains(value, ":") {- // Treat as mapping if the container side looks like an absolute Unix path- // or a Windows drive path (C:/ or C:\). We purposely avoid splitting on the- // first colon to not break on Windows drive letters.- if strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value) {- isMapping = true+ value := strings.TrimSpace(projectsDirSetting.Value)+ isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))+ if filepath.IsAbs(value) || isMapping {+ slog.DebugContext(writeCtx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)+ return "", nil}- }- if !filepath.IsAbs(value) && !isMapping {- // Resolve relative path using current working directory for transparency.- // Note: In containers, WORKDIR is set to /app so "data/..." becomes "/app/data/...".cwd, _ := os.Getwd()absPath, absErr := filepath.Abs(value)if absErr != nil {- return errors.WrapIf(absErr, "failed to resolve relative path to absolute")+ return "", errors.WrapIf(absErr, "failed to resolve relative path to absolute")}- slog.InfoContext(ctx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)-- if err := s.UpdateSetting(ctx, "projectsDirectory", absPath); err != nil {- return errors.WrapIf(err, "failed to update projectsDirectory")+ slog.InfoContext(writeCtx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, "projectsDirectory", absPath); err != nil {+ return "", errors.WrapIf(err, "failed to update projectsDirectory")}-- slog.InfoContext(ctx, "Successfully normalized projects directory")- } else {- slog.DebugContext(ctx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)- }-- return nil+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx)); err != nil {+ return "", err+ }+ slog.InfoContext(writeCtx, "Successfully normalized projects directory")+ return absPath, nil+ }, func(value string, err error) {+ if err == nil && value != "" {+ s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: "projectsDirectory", Value: value}})+ }+ })+ return err}func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {@@ -984,40 +1052,41 @@ func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {return nil}- var buildsDirSetting models.SettingVariable- err := s.db.WithContext(ctx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error-- if errors.Is(err, gorm.ErrRecordNotFound) {- slog.DebugContext(ctx, "No buildsDirectory setting found, skipping normalization")- return nil- }-- if err != nil {- return errors.WrapIf(err, "failed to load buildsDirectory setting")- }+ _, err := actors.Execute(ctx, s.writes, "normalize builds directory", func(writeCtx context.Context) (string, error) {+ var buildsDirSetting models.SettingVariable+ err := s.db.WithContext(writeCtx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ slog.DebugContext(writeCtx, "No buildsDirectory setting found, skipping normalization")+ return "", nil+ }+ if err != nil {+ return "", errors.WrapIf(err, "failed to load buildsDirectory setting")+ }- value := strings.TrimSpace(buildsDirSetting.Value)- if value == "" {- slog.DebugContext(ctx, "buildsDirectory is empty, skipping normalization")- return nil- }+ value := strings.TrimSpace(buildsDirSetting.Value)+ if value == "" || filepath.IsAbs(value) {+ slog.DebugContext(writeCtx, "Builds directory already normalized or empty, skipping", "value", buildsDirSetting.Value)+ return "", nil+ }- if !filepath.IsAbs(value) {cwd, _ := os.Getwd()absPath, absErr := filepath.Abs(value)if absErr != nil {- return errors.WrapIf(absErr, "failed to resolve relative path to absolute")+ return "", errors.WrapIf(absErr, "failed to resolve relative path to absolute")}- slog.InfoContext(ctx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)-- if err := s.UpdateSetting(ctx, buildsKey, absPath); err != nil {- return errors.WrapIf(err, "failed to update buildsDirectory")+ slog.InfoContext(writeCtx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, buildsKey, absPath); err != nil {+ return "", errors.WrapIf(err, "failed to update buildsDirectory")}-- slog.InfoContext(ctx, "Successfully normalized builds directory")- } else {- slog.DebugContext(ctx, "Builds directory already normalized or custom, skipping", "value", buildsDirSetting.Value)- }-- return nil+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx)); err != nil {+ return "", err+ }+ slog.InfoContext(writeCtx, "Successfully normalized builds directory")+ return absPath, nil+ }, func(value string, err error) {+ if err == nil && value != "" {+ s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: buildsKey, Value: value}})+ }+ })+ return err}diff --git a/backend/internal/services/settings_service_test.go b/backend/internal/services/settings_service_test.goindex 4e1313b3f..acd6384c0 100644--- a/backend/internal/services/settings_service_test.go+++ b/backend/internal/services/settings_service_test.go@@ -3,12 +3,17 @@ package servicesimport ("context""path/filepath"+ "strings"+ "sync/atomic""testing"+ "time""github.com/libtnb/sqlite""github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -25,10 +30,57 @@ func setupSettingsTestDB(t *testing.T) *database.DB {return &database.DB{DB: db}}+func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*SettingsService, error) {+ t.Helper()+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)+ require.NoError(t, err)+ effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, executor.Stop(stopCtx))+ require.NoError(t, effects.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+ })+ return NewSettingsService(ctx, db, executor, effects)+}++func newAdmissionGateForTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {+ t.Helper()+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "services-test-admission", t.Name())+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+ })+ return gate+}++func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {+ t.Helper()+ _, err := actors.Execute(t.Context(), svc.writes, "wait for settings notification submission", func(context.Context) (actors.NoPayload, error) {+ return actors.NoPayload{}, nil+ }, nil)+ require.NoError(t, err)+ _, err = actors.Execute(t.Context(), svc.effects, "wait for settings notifications", func(context.Context) (actors.NoPayload, error) {+ return actors.NoPayload{}, nil+ }, nil)+ require.NoError(t, err)+}+func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -91,7 +143,7 @@ func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {func TestSettingsService_EnsureDefaultSettings_OverridesExistingTrivyImage(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyImage", "ghcr.io/aquasecurity/trivy:latest"))@@ -105,7 +157,7 @@ func TestSettingsService_EnsureDefaultSettings_OverridesExistingTrivyImage(t *tefunc TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.db.WithContext(ctx).@@ -118,7 +170,7 @@ func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.SetStringSetting(ctx, "baseServerUrl", "http://cached"))@@ -134,7 +186,7 @@ func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testinfunc TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)current, err := svc.GetSettings(ctx)@@ -155,7 +207,7 @@ func TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))@@ -175,7 +227,7 @@ func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretIntfunc TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecretInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))@@ -195,7 +247,7 @@ func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecrefunc TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))@@ -214,7 +266,7 @@ func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenIntfunc TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementTokenInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))@@ -234,7 +286,7 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementTokefunc TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))@@ -253,7 +305,7 @@ func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenInfunc TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))@@ -270,7 +322,7 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenIfunc TestSettingsService_PruneUnknownSettings_RemovesStaleKeys(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "/tmp/projects"))@@ -297,7 +349,7 @@ func TestSettingsService_GetSettings_EnvOverride_OidcMergeAccounts(t *testing.T)db := setupSettingsTestDB(t)t.Setenv("OIDC_MERGE_ACCOUNTS", "true")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -311,7 +363,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyScanTimeout(t *testing.T)db := setupSettingsTestDB(t)t.Setenv("TRIVY_SCAN_TIMEOUT", "1800")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -327,7 +379,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyResourceLimits(t *testing.t.Setenv("TRIVY_CPU_LIMIT", "2.5")t.Setenv("TRIVY_MEMORY_LIMIT_MB", "2048")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -343,7 +395,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyNetwork(t *testing.T) {db := setupSettingsTestDB(t)t.Setenv("TRIVY_NETWORK", "arcane-external")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -357,7 +409,7 @@ func TestSettingsService_GetSettings_EnvOverride_FollowProjectSymlinks(t *testindb := setupSettingsTestDB(t)t.Setenv("FOLLOW_PROJECT_SYMLINKS", "true")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -372,7 +424,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyRuntimeSecurity(t *testingt.Setenv("TRIVY_SECURITY_OPTS", "label=disable,\nlabel=type:container_runtime_t")t.Setenv("TRIVY_PRIVILEGED", "true")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -387,7 +439,7 @@ func TestSettingsService_GetStringSetting_EnvOverride_SwarmStackSourcesDirectorydb := setupSettingsTestDB(t)t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", "/mnt/swarm-from-env")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "swarmStackSourcesDirectory", "/tmp/swarm-from-db"))@@ -399,17 +451,17 @@ func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.False(t, svc.isEnvOverrideActiveInternal("oidcEnabled"))t.Setenv("OIDC_ENABLED", "false")- svcWithOverride, err := NewSettingsService(ctx, db)+ svcWithOverride, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.True(t, svcWithOverride.isEnvOverrideActiveInternal("oidcEnabled"))t.Setenv("AUTH_SESSION_TIMEOUT", "120")- svcWithNonOverrideEnv, err := NewSettingsService(ctx, db)+ svcWithNonOverrideEnv, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.False(t, svcWithNonOverrideEnv.isEnvOverrideActiveInternal("authSessionTimeout"))}@@ -417,7 +469,7 @@ func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {func TestSettingsService_GetSetHelpers(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// Defaults for missing keys@@ -439,7 +491,7 @@ func TestSettingsService_GetSetHelpers(t *testing.T) {func TestSettingsService_UpdateSetting(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.UpdateSetting(ctx, "pruneImageMode", "all"))@@ -449,10 +501,24 @@ func TestSettingsService_UpdateSetting(t *testing.T) {require.Equal(t, "all", sv.Value)}+func TestSettingsService_UpdateSettingRejectsInvalidCronBeforePersistence(t *testing.T) {+ ctx := context.Background()+ db := setupSettingsTestDB(t)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)++ require.NoError(t, svc.UpdateSetting(ctx, "pollingInterval", "*/5 * * * * *"))+ require.Error(t, svc.UpdateSetting(ctx, "pollingInterval", "not a cron schedule"))++ var setting models.SettingVariable+ require.NoError(t, svc.db.WithContext(ctx).Where("key = ?", "pollingInterval").First(&setting).Error)+ require.Equal(t, "*/5 * * * * *", setting.Value)+}+func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.Equal(t, "http://localhost", svc.GetSettingsConfig().BaseServerURL.Value)@@ -468,40 +534,174 @@ func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {func TestSettingsService_UpdateSettings_PruneModesDoNotTriggerScheduledPruneCallback(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)callbackCalls := 0- svc.OnScheduledPruneSettingsChanged = func(context.Context) {+ svc.SubscribeSettingsChanges([]string{"scheduledPruneEnabled", "scheduledPruneInterval"}, func([]libarcane.SettingUpdate) {callbackCalls++- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{PruneImageMode: new("all"),PruneContainerUntil: new("24h"),})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.Equal(t, 0, callbackCalls)}func TestSettingsService_UpdateSettings_ScheduledPruneScheduleTriggersCallback(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)callbackCalls := 0- svc.OnScheduledPruneSettingsChanged = func(context.Context) {+ svc.SubscribeSettingsChanges([]string{"scheduledPruneEnabled", "scheduledPruneInterval"}, func([]libarcane.SettingUpdate) {callbackCalls++- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{ScheduledPruneEnabled: new("true"),})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.Equal(t, 1, callbackCalls)}+func TestSettingsServiceActorSerializesConcurrentUpdatesInternal(t *testing.T) {+ ctx := t.Context()+ db := setupSettingsTestDB(t)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)+ require.NoError(t, svc.EnsureDefaultSettings(ctx))++ start := make(chan struct{})+ results := make(chan error, 2)+ go func() {+ <-start+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{ProjectsDirectory: new("/data/projects-a")})+ results <- updateErr+ }()+ go func() {+ <-start+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{TemplatesDirectory: new("/data/templates-b")})+ results <- updateErr+ }()+ close(start)++ require.NoError(t, <-results)+ require.NoError(t, <-results)+ current := svc.GetSettingsConfig()+ require.Equal(t, "/data/projects-a", current.ProjectsDirectory.Value)+ require.Equal(t, "/data/templates-b", current.TemplatesDirectory.Value)+}++func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotificationInternal(t *testing.T) {+ ctx := t.Context()+ db := setupSettingsTestDB(t)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)+ require.NoError(t, svc.EnsureDefaultSettings(ctx))++ callbackStarted := make(chan string, 1)+ releaseCallback := make(chan struct{})+ svc.SubscribeSettingsChanges([]string{"baseServerUrl"}, func(_ []libarcane.SettingUpdate) {+ callbackStarted <- svc.GetSettingsConfig().BaseServerURL.Value+ <-releaseCallback+ })++ updateResult := make(chan error, 1)+ go func() {+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{BaseServerURL: new("https://actor.example")})+ updateResult <- updateErr+ }()++ require.Equal(t, "https://actor.example", <-callbackStarted)+ select {+ case err := <-updateResult:+ require.NoError(t, err)+ case <-time.After(time.Second):+ t.Fatal("settings update waited for notification callback")+ }++ secondUpdate := make(chan error, 1)+ go func() {+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{ProjectsDirectory: new("/data/unblocked")})+ secondUpdate <- updateErr+ }()+ select {+ case err := <-secondUpdate:+ require.NoError(t, err)+ case <-time.After(time.Second):+ t.Fatal("settings subscriber blocked the next settings writer")+ }+ close(releaseCallback)+}++func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeysInternal(t *testing.T) {+ ctx := t.Context()+ db := setupSettingsTestDB(t)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)+ require.NoError(t, svc.EnsureDefaultSettings(ctx))++ var calls atomic.Int32+ notified := make(chan []libarcane.SettingUpdate, 1)+ svc.SubscribeSettingsChanges([]string{"pollingEnabled", "pollingInterval"}, func(updates []libarcane.SettingUpdate) {+ calls.Add(1)+ notified <- updates+ })++ _, err = svc.UpdateSettings(ctx, settings.Update{+ PollingEnabled: new("false"),+ PollingInterval: new("0 */5 * * * *"),+ })+ require.NoError(t, err)+ require.Len(t, <-notified, 2)+ require.Equal(t, int32(1), calls.Load())+}++func TestSettingsServiceActorDoesNotPublishSensitiveValuesInternal(t *testing.T) {+ ctx := t.Context()+ db := setupSettingsTestDB(t)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)+ require.NoError(t, svc.EnsureDefaultSettings(ctx))++ var calls atomic.Int32+ svc.SubscribeSettingsChanges([]string{"oidcClientSecret"}, func([]libarcane.SettingUpdate) {+ calls.Add(1)+ })+ _, err = svc.UpdateSettings(ctx, settings.Update{OidcClientSecret: new("should-not-leave-settings-service")})+ require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)+ require.Zero(t, calls.Load())+}++func TestSettingsServiceActorSerializesContainerExclusionUpdatesInternal(t *testing.T) {+ ctx := t.Context()+ db := setupSettingsTestDB(t)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)+ require.NoError(t, svc.EnsureDefaultSettings(ctx))++ start := make(chan struct{})+ results := make(chan error, 2)+ for _, name := range []string{"api", "worker"} {+ go func() {+ <-start+ results <- svc.SetContainerAutoUpdateExclusionInternal(ctx, name, true)+ }()+ }+ close(start)++ require.NoError(t, <-results)+ require.NoError(t, <-results)+ require.ElementsMatch(t, []string{"api", "worker"}, strings.Split(svc.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""), ","))+}+func BenchmarkSettingsService_GetSettings(b *testing.B) {ctx := context.Background()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})@@ -512,7 +712,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {b.Fatal(err)}settingsDB := &database.DB{DB: db}- svc, err := NewSettingsService(ctx, settingsDB)+ svc, err := newSettingsServiceForTestInternal(b, ctx, settingsDB)if err != nil {b.Fatal(err)}@@ -534,7 +734,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {func TestSettingsService_EnsureEncryptionKey(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)k1, err := svc.EnsureEncryptionKey(ctx)@@ -553,7 +753,7 @@ func TestSettingsService_EnsureEncryptionKey(t *testing.T) {func TestSettingsService_LoadDatabaseSettings_ReloadsChanges(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// Initially empty DB -> defaults (not persisted yet)@@ -580,7 +780,7 @@ func TestSettingsService_LoadDatabaseSettings_UIConfigurationDisabled_Env(t *tesctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// Reload explicitly (NewSettingsService already did, but explicit for clarity)@@ -596,7 +796,7 @@ func TestSettingsService_PersistEnvSettingsIfMissing_DoesNotOverrideForcedTrivyIdb := setupSettingsTestDB(t)t.Setenv("TRIVY_IMAGE", "ghcr.io/aquasecurity/trivy:latest")- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -611,7 +811,7 @@ func TestSettingsService_PersistEnvSettingsIfMissing_DoesNotOverrideForcedTrivyIfunc TestSettingsService_UpdateSettings_RefreshesCache(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -640,7 +840,7 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -662,17 +862,18 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))var callbackPayload []libarcane.SettingUpdate- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {callbackPayload = timeoutSettings- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivyScanTimeout: new("1200")})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.NotNil(t, callbackPayload)require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyScanTimeout", Value: "1200"})@@ -681,14 +882,14 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimits(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))var callbackPayload []libarcane.SettingUpdate- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {callbackPayload = timeoutSettings- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivyResourceLimitsEnabled: new("false"),@@ -696,6 +897,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimiTrivyMemoryLimitMb: new("3072"),})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.NotNil(t, callbackPayload)require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyResourceLimitsEnabled", Value: "false"})@@ -706,17 +908,18 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimifunc TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentScanContainers(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))var callbackPayload []libarcane.SettingUpdate- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {callbackPayload = timeoutSettings- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivyConcurrentScanContainers: new("4")})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.NotNil(t, callbackPayload)require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyConcurrentScanContainers", Value: "4"})@@ -725,81 +928,85 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentScfunc TestSettingsService_UpdateSettings_TrivyNetworkTriggersVulnerabilityCallback(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))callbackCalled := false- svc.OnVulnerabilityScanSettingsChanged = func(_ context.Context) {+ svc.SubscribeSettingsChanges([]string{"trivyNetwork"}, func(_ []libarcane.SettingUpdate) {callbackCalled = true- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.True(t, callbackCalled)}func TestSettingsService_UpdateSettings_TrivyNetworkDoesNotTriggerTimeoutCallback(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))var callbackPayload []libarcane.SettingUpdate- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {callbackPayload = timeoutSettings- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.Nil(t, callbackPayload)}func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityTriggersVulnerabilityCallback(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))callbackCalled := false- svc.OnVulnerabilityScanSettingsChanged = func(_ context.Context) {+ svc.SubscribeSettingsChanges([]string{"trivySecurityOpts", "trivyPrivileged"}, func(_ []libarcane.SettingUpdate) {callbackCalled = true- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivySecurityOpts: new("label=disable"),TrivyPrivileged: new("true"),})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.True(t, callbackCalled)}func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityDoesNotTriggerTimeoutCallback(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))var callbackPayload []libarcane.SettingUpdate- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {callbackPayload = timeoutSettings- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivySecurityOpts: new("label=disable"),TrivyPrivileged: new("true"),})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.Nil(t, callbackPayload)}func TestSettingsService_UpdateSettings_TrivyPreserveCacheOnVolumePrunePersists(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -819,17 +1026,18 @@ func TestSettingsService_UpdateSettings_TrivyPreserveCacheOnVolumePrunePersists(func TestSettingsService_UpdateSettings_TrivyPreserveCacheOnVolumePruneDoesNotTriggerTimeoutCallback(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))var callbackPayload []libarcane.SettingUpdate- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {callbackPayload = timeoutSettings- }+ })_, err = svc.UpdateSettings(ctx, settings.Update{TrivyPreserveCacheOnVolumePrune: new("false")})require.NoError(t, err)+ waitForSettingsNotificationsInternal(t, svc)require.Nil(t, callbackPayload)}@@ -845,7 +1053,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)internalVal := "test-instance-id"require.NoError(t, db.DB.Create(&models.SettingVariable{Key: internalKey, Value: internalVal}).Error)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// Reload explicitly to trigger the env loading path@@ -859,7 +1067,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// Seed with relative path@@ -882,7 +1090,7 @@ func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(tfunc TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// Seed with relative path@@ -901,7 +1109,7 @@ func TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.Tfunc TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)customPath := "/custom/projects/path"@@ -920,7 +1128,7 @@ func TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(tfunc TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)// Don't create the setting at all@@ -933,7 +1141,7 @@ func TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.Tfunc TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalization(t *testing.T) {ctx := context.Background()db := setupSettingsTestDB(t)- svc, err := NewSettingsService(ctx, db)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, svc.EnsureDefaultSettings(ctx))@@ -955,3 +1163,25 @@ func TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalizatirequire.Equal(t, expectedPath, cfg2.ProjectsDirectory.Value)require.True(t, filepath.IsAbs(cfg2.ProjectsDirectory.Value), "path should be absolute")}++func TestSettingsService_NormalizeProjectsDirectoryPublishesChangeInternal(t *testing.T) {+ ctx := t.Context()+ db := setupSettingsTestDB(t)+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)+ require.NoError(t, svc.EnsureDefaultSettings(ctx))+ require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "data/projects"))++ notified := make(chan []libarcane.SettingUpdate, 1)+ unsubscribe := svc.SubscribeSettingsChanges([]string{"projectsDirectory"}, func(updates []libarcane.SettingUpdate) {+ notified <- updates+ })+ defer unsubscribe()++ require.NoError(t, svc.NormalizeProjectsDirectory(ctx, ""))+ waitForSettingsNotificationsInternal(t, svc)+ updates := <-notified+ require.Len(t, updates, 1)+ require.Equal(t, "projectsDirectory", updates[0].Key)+ require.True(t, filepath.IsAbs(updates[0].Value))+}diff --git a/backend/internal/services/swarm_service_test.go b/backend/internal/services/swarm_service_test.goindex 8b593b4a1..8efc7b53c 100644--- a/backend/internal/services/swarm_service_test.go+++ b/backend/internal/services/swarm_service_test.go@@ -38,7 +38,7 @@ func TestDefaultSwarmListenAddrInternal(t *testing.T) {func TestSwarmService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToken(t *testing.T) {ctx := context.Background()db := setupEnvironmentServiceTestDB(t)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)envSvc := NewEnvironmentService(db, nil, nil, nil, settingsSvc, nil)@@ -82,7 +82,7 @@ func TestSwarmService_UpdateAndGetStackSource_UsesStoredFilesWithoutSwarmManagerrootDir := t.TempDir()t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)svc := NewSwarmService(nil, settingsSvc, nil, nil, nil)@@ -140,7 +140,7 @@ func TestSwarmService_UpdateAndGetStackSource_RoundTripsOverride(t *testing.T) {rootDir := t.TempDir()t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)- settingsSvc, err := NewSettingsService(ctx, db)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)svc := NewSwarmService(nil, settingsSvc, nil, nil, nil)diff --git a/backend/internal/services/template_service_test.go b/backend/internal/services/template_service_test.goindex 8ae24d68a..32757bb91 100644--- a/backend/internal/services/template_service_test.go+++ b/backend/internal/services/template_service_test.go@@ -483,7 +483,7 @@ func minimalSettingsServiceForTest(t *testing.T) *SettingsService {db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))- svc, err := NewSettingsService(context.Background(), &database.DB{DB: db})+ svc, err := newSettingsServiceForTestInternal(t, context.Background(), &database.DB{DB: db})require.NoError(t, err)return svc}diff --git a/backend/internal/services/variable_service_test.go b/backend/internal/services/variable_service_test.goindex 77e07175e..7500345bb 100644--- a/backend/internal/services/variable_service_test.go+++ b/backend/internal/services/variable_service_test.go@@ -50,7 +50,7 @@ func setupVariableServiceTest(t *testing.T) (*VariableService, *database.DB, strprojectsDir := t.TempDir()dbWrap := &database.DB{DB: db}- settingsSvc, err := NewSettingsService(context.Background(), dbWrap)+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), dbWrap)require.NoError(t, err)require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "projectsDirectory", projectsDir))diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.goindex 924e767e0..aa1a35249 100644--- a/backend/pkg/libarcane/edge/client.go+++ b/backend/pkg/libarcane/edge/client.go@@ -9,13 +9,16 @@ import ("net""net/http""strings"+ "sync""time""emperror.dev/errors""github.com/cenkalti/backoff/v5""github.com/coder/websocket"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/google/uuid""github.com/samber/mo")@@ -29,7 +32,8 @@ const (healthyTunnelSessionDuration = 60 * time.Second// maxPollRetryInterval caps the exponential backoff applied to failed poll// control requests. Successful polls return to the manager-advertised interval.- maxPollRetryInterval = 60 * time.Second+ maxPollRetryInterval = 60 * time.Second+ tunnelSessionWorkerDrainTimeout = 10 * time.Second// DefaultHeartbeatInterval is how often the client sends heartbeatsDefaultHeartbeatInterval = 30 * time.Second@@ -66,14 +70,6 @@ func (t *commandRequestTransfer) stopInternal() {}}-// getConn returns the active tunnel connection, or nil if none is established.-func (c *TunnelClient) getConn() TunnelConnection {- if box := c.conn.Load(); box != nil {- return box.conn- }- return nil-}-// NewTunnelClient creates a new tunnel clientfunc NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {reconnectInterval := time.Duration(cfg.EdgeReconnectInterval) * time.Second@@ -104,8 +100,6 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {managerURL: managerURL,managerGRPCAddr: managerGRPCAddr,localPort: localPort,- httpClient: &http.Client{},- stopCh: make(chan struct{}),requestTimeout: DefaultRequestTimeout,agentInstanceID: uuid.NewString(),}@@ -113,11 +107,10 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {// StartWithErrorChan runs the tunnel client and optionally emits connection errors.func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error) {- slog.InfoContext(ctx, "Starting edge agent session client", StartupLogAttrs(c.cfg)...)if errCh != nil {defer close(errCh)}-+ slog.InfoContext(ctx, "Starting edge agent session client", StartupLogAttrs(c.cfg)...)// A fixed reconnect interval meant a permanently broken or unauthorized agent// hammered the manager's auth and DB path forever. Back off exponentially// (with jitter) up to maxReconnectInterval, and reset only once a session has@@ -131,19 +124,15 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)case <-ctx.Done():slog.InfoContext(ctx, "Edge tunnel client shutting down")return- case <-c.stopCh:- slog.InfoContext(ctx, "Edge tunnel client stopped")- returndefault:sessionStart := time.Now()if err := c.connectAndServe(ctx); err != nil {+ slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)if errCh != nil {select {case errCh <- err:default:}- } else {- slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)}}@@ -158,9 +147,6 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)case <-ctx.Done():reconnectTimer.Stop()return- case <-c.stopCh:- reconnectTimer.Stop()- returncase <-reconnectTimer.C:reconnectTimer.Stop()slog.InfoContext(ctx, "Attempting to reconnect edge tunnel", "delay", delay)@@ -393,20 +379,20 @@ func (c *TunnelClient) grpcFailureStreakInternal() int {func (c *TunnelClient) registerMessageInternal() *TunnelMessage {capabilities := AdvertisedEdgeCommands()capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity)+ registration, _ := c.registration.Load()return &TunnelMessage{Type: MessageTypeRegister,AgentToken: c.cfg.AgentToken,AgentInstance: c.agentInstanceID,Capabilities: capabilities,- ResumeSession: c.sessionID,+ ResumeSession: registration.sessionID,}}-func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMessage, error) {+func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn TunnelConnection) (*TunnelMessage, error) {if c == nil {return nil, errors.New("edge tunnel connection is not initialized")}- conn := c.getConn()if conn == nil {return nil, errors.New("edge tunnel connection is not initialized")}@@ -446,8 +432,7 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMeif !result.msg.Accepted {return nil, errors.Errorf("manager rejected tunnel registration: %s", result.msg.Error)}- c.sessionID = result.msg.SessionID- c.managerCapabilities = append([]string(nil), result.msg.Capabilities...)+ c.registration.Store(clientRegistrationInternal{sessionID: result.msg.SessionID})return result.msg, nil}}@@ -455,19 +440,32 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMe// serveTunnelSessionInternal runs the shared register/heartbeat/message// lifecycle on an established tunnel connection, for every transport.func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn TunnelConnection, managerAddr string) error {- c.conn.Store(&connBox{conn: conn})+ box := &connBox{conn: conn}+ c.conn.Store(box)setActiveAgentTunnelConn(conn)- defer clearActiveAgentTunnelConn(conn)- // Always tear down through the wrapper so the peer sees a clean close- // (websocket close frame / gRPC half-close) and later Sends hit the- // closed guard instead of the dead socket.- defer func() { _ = conn.Close() }()+ connCtx, connCancel := context.WithCancel(ctx)+ var workers sync.WaitGroup+ stopCloseWatch := context.AfterFunc(connCtx, func() { _ = conn.Close() })+ defer func() {+ connCancel()+ c.closeAllStreams()+ c.clearCommandRequestTransfersInternal(conn)+ _ = conn.Close()+ stopCloseWatch()+ drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(ctx), tunnelSessionWorkerDrainTimeout)+ if err := utils.WaitGroup(drainCtx, &workers); err != nil {+ slog.WarnContext(context.WithoutCancel(ctx), "Timed out waiting for edge tunnel session workers", "error", err)+ }+ cancelDrain()+ c.conn.CompareAndSwap(box, nil)+ clearActiveAgentTunnelConn(conn)+ }()if err := conn.Send(c.registerMessageInternal()); err != nil {return errors.WrapIff(err, "failed to send %s tunnel register message", conn.Transport())}- registerMsg, err := c.awaitRegistrationInternal(ctx)+ registerMsg, err := c.awaitRegistrationInternal(ctx, conn)if err != nil {return err}@@ -480,21 +478,13 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunn)c.markTransportConnectedInternal(conn.Transport())- connCtx, connCancel := context.WithCancel(ctx)- defer connCancel()- // A blocked websocket Receive is not ctx-aware; closing the connection is- // what unblocks messageLoop when the session context is cancelled.- go func() {- <-connCtx.Done()- _ = conn.Close()- }()- go c.heartbeatLoop(connCtx)+ workers.Go(func() { c.heartbeatLoop(connCtx, conn) })- return c.messageLoop(connCtx)+ return c.messageLoop(connCtx, conn, &workers)}// heartbeatLoop sends periodic heartbeats-func (c *TunnelClient) heartbeatLoop(ctx context.Context) {+func (c *TunnelClient) heartbeatLoop(ctx context.Context, conn TunnelConnection) {ticker := time.NewTicker(c.heartbeatInterval)defer ticker.Stop()@@ -503,7 +493,6 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context) {case <-ctx.Done():returncase <-ticker.C:- conn := c.getConn()if conn == nil || conn.IsClosed() {return}@@ -527,13 +516,7 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context) {}// messageLoop processes incoming messages from the manager-func (c *TunnelClient) messageLoop(ctx context.Context) error {- // Tear down any streams opened on this connection when the loop exits so a- // reconnect does not leak goroutines, local sockets, or activeStreams entries.- defer c.closeAllStreams()-- conn := c.getConn()- defer c.clearCommandRequestTransfersInternal(conn)+func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, workers *sync.WaitGroup) error {for {select {case <-ctx.Done():@@ -546,19 +529,19 @@ func (c *TunnelClient) messageLoop(ctx context.Context) error {switch msg.Type {case MessageTypeRequest:- go c.handleRequest(ctx, conn, msg)+ workers.Go(func() { c.handleRequest(ctx, conn, msg) })case MessageTypeCommandRequest:if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {c.beginCommandRequestTransferInternal(ctx, conn, transferID, msg)} else {- go c.handleCommandRequest(ctx, conn, msg)+ workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })}case MessageTypeFileChunk:- c.handleCommandRequestChunkInternal(ctx, msg)+ c.handleCommandRequestChunkInternal(ctx, msg, workers)case MessageTypeWebSocketStart:- c.handleWebSocketStart(ctx, conn, msg)+ c.handleWebSocketStart(ctx, conn, msg, workers)case MessageTypeStreamOpen:- c.handleStreamOpen(ctx, conn, msg)+ c.handleStreamOpen(ctx, conn, msg, workers)case MessageTypeWebSocketData:c.handleWebSocketData(ctx, msg)case MessageTypeStreamData:@@ -614,7 +597,7 @@ func (c *TunnelClient) beginCommandRequestTransferInternal(ctx context.Context,}}-func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, msg *TunnelMessage) {+func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, msg *TunnelMessage, workers *sync.WaitGroup) {if c == nil || msg == nil {return}@@ -647,7 +630,7 @@ func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, msif c.requestTransfers.CompareAndDelete(msg.ID, transfer) {transfer.stopInternal()transfer.request.Body = append([]byte(nil), transfer.body.Bytes()...)- go c.handleCommandRequest(ctx, transfer.conn, transfer.request)+ workers.Go(func() { c.handleCommandRequest(ctx, transfer.conn, transfer.request) })}}@@ -828,7 +811,7 @@ func (c *TunnelClient) handleRequestStreaming(ctx context.Context, conn TunnelCo}// handleWebSocketStart handles a WebSocket stream start request from the manager.-func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {+func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {if msg.Command == "" {if commandName, ok := ResolveEdgeCommandName(http.MethodGet, msg.Path, true).Get(); ok {msg.Command = commandName@@ -865,11 +848,11 @@ func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnstreamCtx, cancel := context.WithCancel(ctx)stream := c.registerStream(conn, streamID, ws, cancel)- go c.startLocalWebSocketReadLoop(ctx, streamCtx, streamID, ws, stream)- go c.startLocalWebSocketWriteLoop(ctx, streamCtx, ws, stream, cancel)+ workers.Go(func() { c.startLocalWebSocketReadLoop(ctx, streamCtx, streamID, ws, stream) })+ workers.Go(func() { c.startLocalWebSocketWriteLoop(ctx, streamCtx, ws, stream, cancel) })}-func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {+func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {if !ValidateEdgeCommand(msg.Command, http.MethodGet, msg.Path, true) {c.sendStreamCloseMessage(conn, msg.ID, "unsupported edge stream")return@@ -882,7 +865,7 @@ func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnectiPath: msg.Path,Query: msg.Query,Headers: msg.Headers,- })+ }, workers)}func (c *TunnelClient) buildLocalWebSocketURLInternal(msg *TunnelMessage) string {@@ -1439,8 +1422,8 @@ func (r *streamingResponseRecorder) writeHeaderLocked(statusCode int) error {return nil}-// StartTunnelClientWithErrors starts the tunnel client and returns a channel for connection errors.-func StartTunnelClientWithErrors(ctx context.Context, cfg *Config, handler http.Handler) (<-chan error, error) {+// StartTunnelClient starts the tunnel client on the shared actor runtime.+func StartTunnelClient(ctx context.Context, runtime *actors.Runtime, cfg *Config, handler http.Handler) (func(context.Context) error, error) {if !cfg.EdgeAgent {return nil, errors.New("edge tunnel disabled")}@@ -1471,7 +1454,12 @@ func StartTunnelClientWithErrors(ctx context.Context, cfg *Config, handler http.}client := NewTunnelClient(cfg, handler)- errCh := make(chan error, 1)- go client.StartWithErrorChan(ctx, errCh)- return errCh, nil+ runner, err := actors.NewRunner(ctx, runtime, "edge", "agent-client", "Edge tunnel client", 3, func(runCtx context.Context) error {+ client.StartWithErrorChan(runCtx, nil)+ return nil+ })+ if err != nil {+ return nil, err+ }+ return runner.Stop, nil}diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.goindex beca1a404..199c20264 100644--- a/backend/pkg/libarcane/edge/client_test.go+++ b/backend/pkg/libarcane/edge/client_test.go@@ -689,12 +689,10 @@ func TestTunnelClient_HeartbeatLoop_ClosesConnectionOnSendFailure(t *testing.T)client := &TunnelClient{heartbeatInterval: 5 * time.Millisecond,}- client.conn.Store(&connBox{conn: conn})-ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)defer cancel()- client.heartbeatLoop(ctx)+ client.heartbeatLoop(ctx, conn)assert.True(t, conn.closeCalled)}@@ -989,17 +987,17 @@ func TestTunnelClient_useTLSForManagerGRPC(t *testing.T) {}}-func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {+func TestStartTunnelClient_GRPCValidation(t *testing.T) {ctx := t.Context()t.Run("edge mode required", func(t *testing.T) {- _, err := StartTunnelClientWithErrors(ctx, &Config{}, http.NotFoundHandler())+ _, err := StartTunnelClient(ctx, nil, &Config{}, http.NotFoundHandler())require.Error(t, err)assert.Contains(t, err.Error(), "edge tunnel disabled")})t.Run("manager url required for grpc transport", func(t *testing.T) {- _, err := StartTunnelClientWithErrors(ctx, &Config{+ _, err := StartTunnelClient(ctx, nil, &Config{EdgeAgent: true,EdgeTransport: EdgeTransportGRPC,AgentToken: "token",@@ -1009,7 +1007,7 @@ func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {})t.Run("agent token required", func(t *testing.T) {- _, err := StartTunnelClientWithErrors(ctx, &Config{+ _, err := StartTunnelClient(ctx, nil, &Config{EdgeAgent: true,EdgeTransport: EdgeTransportGRPC,ManagerApiUrl: "https://manager.example.com/arcane/api",@@ -1019,7 +1017,7 @@ func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {})t.Run("mtls requires https manager url", func(t *testing.T) {- _, err := StartTunnelClientWithErrors(ctx, &Config{+ _, err := StartTunnelClient(ctx, nil, &Config{EdgeAgent: true,EdgeTransport: EdgeTransportGRPC,ManagerApiUrl: "http://manager.example.com/api",@@ -1031,7 +1029,7 @@ func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {})t.Run("required mtls auto-enrollment failure surfaces", func(t *testing.T) {- _, err := StartTunnelClientWithErrors(ctx, &Config{+ _, err := StartTunnelClient(ctx, nil, &Config{EdgeAgent: true,EdgeTransport: EdgeTransportGRPC,ManagerApiUrl: "https://127.0.0.1:1/api",@@ -1113,7 +1111,8 @@ func TestTunnelClient_connectAndServeGRPC_TimesOutWithoutRegisterResponse(t *tesassert.Contains(t, err.Error(), "timed out waiting for tunnel registration response")assert.Contains(t, err.Error(), "not forwarding gRPC")assert.EqualValues(t, 1, service.connectCount.Load())- assert.True(t, client.getConn() == nil || client.getConn().IsClosed())+ active := client.conn.Load()+ assert.True(t, active == nil || active.conn.IsClosed())}func TestTunnelClient_awaitRegistrationInternal_ClosesConnOnContextDone(t *testing.T) {@@ -1126,7 +1125,7 @@ func TestTunnelClient_awaitRegistrationInternal_ClosesConnOnContextDone(t *testi}client.conn.Store(&connBox{conn: conn})- msg, err := client.awaitRegistrationInternal(ctx)+ msg, err := client.awaitRegistrationInternal(ctx, conn)require.Nil(t, msg)require.ErrorIs(t, err, context.DeadlineExceeded)assert.True(t, conn.IsClosed())@@ -1155,7 +1154,7 @@ func TestTunnelClient_awaitRegistrationInternal_ClosesAttemptConnWhenClientConnC}resultCh := make(chan registrationResult, 1)go func() {- msg, err := client.awaitRegistrationInternal(ctx)+ msg, err := client.awaitRegistrationInternal(ctx, firstConn)resultCh <- registrationResult{msg: msg, err: err}}()@@ -1842,14 +1841,13 @@ func TestTunnelClient_pollTunnelControlInternal_UsesConfiguredHTTPClient(t *test})client := NewTunnelClient(&Config{AgentToken: "valid-token"}, http.NotFoundHandler())- client.httpClient = &http.Client{Transport: rewriteTransport}+ httpClient := &http.Client{Transport: rewriteTransport}- resp, err := client.pollTunnelControlInternal(context.Background(), "http://127.0.0.1:1/api/tunnel/poll", false)+ resp, err := client.pollTunnelControlInternal(context.Background(), httpClient, "http://127.0.0.1:1/api/tunnel/poll", false)require.NoError(t, err)assert.Equal(t, TunnelStatusIdle, resp.Status)assert.Equal(t, 1, resp.PollIntervalSeconds)- assert.NotNil(t, client.httpClient)- assert.NotSame(t, http.DefaultClient, client.httpClient)+ assert.NotSame(t, http.DefaultClient, httpClient)defaultReq, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://127.0.0.1:1/api/tunnel/poll", nil)require.NoError(t, err)@@ -2006,9 +2004,19 @@ func TestTunnelClient_syncPollManagedSessionInternal_IdleUsesBoundedStopTimeout(}nextSession, err := (&TunnelClient{}).syncPollManagedSessionInternal(context.Background(), session, TunnelStatusIdle)- require.Nil(t, nextSession)+ require.Same(t, session, nextSession)require.Error(t, err)assert.EqualError(t, err, "timed out waiting for poll-managed websocket session to stop")++ fencedSession, err := (&TunnelClient{}).syncPollManagedSessionInternal(context.Background(), nextSession, TunnelStatusRequired)+ require.NoError(t, err)+ require.Same(t, session, fencedSession)++ started := time.Now()+ stillDraining, err := (&TunnelClient{}).syncPollManagedSessionInternal(context.Background(), nextSession, TunnelStatusIdle)+ require.NoError(t, err)+ require.Same(t, session, stillDraining)+ require.Less(t, time.Since(started), defaultPollManagedSessionStopTimeout)}func startTestPollAndGRPCManagerInternal(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer, pollResp TunnelPollResponse) (string, func()) {diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.goindex 2be6b6d04..e79ebcb20 100644--- a/backend/pkg/libarcane/edge/client_transport_poll.go+++ b/backend/pkg/libarcane/edge/client_transport_poll.go@@ -19,6 +19,8 @@ const defaultTunnelPollRequestTimeout = 15 * time.Secondvar defaultPollManagedSessionStopTimeout = 5 * time.Second+var errPollManagedSessionStopTimeout = errors.Sentinel("timed out waiting for poll-managed websocket session to stop")+func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {managerBaseURL := strings.TrimRight(strings.TrimSpace(c.cfg.GetManagerBaseURL()), "/")if managerBaseURL == "" {@@ -28,8 +30,6 @@ func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {if err != nil {return errors.WrapIf(err, "failed to configure edge poll client")}- c.httpClient = httpClient-pollURL := managerBaseURL + "/api/tunnel/poll"interval := DefaultTunnelPollInterval// Failed polls back off exponentially instead of retrying every 2s forever;@@ -64,7 +64,7 @@ func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {return ctx.Err()}- status, err := c.pollTunnelControlInternal(ctx, pollURL, session != nil)+ status, err := c.pollTunnelControlInternal(ctx, httpClient, pollURL, session != nil)if err != nil {if ctx.Err() != nil {return ctx.Err()@@ -93,7 +93,10 @@ func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {session, err = c.syncPollManagedSessionInternal(ctx, session, status.Status)if err != nil {- return err+ if !errors.Is(err, errPollManagedSessionStopTimeout) {+ return err+ }+ slog.WarnContext(ctx, "Poll-managed tunnel is still draining; replacement remains fenced", "error", err)}session, err = waitForNextPollCycleInternal(ctx, session, interval)@@ -136,13 +139,17 @@ func (c *TunnelClient) syncPollManagedSessionInternal(ctx context.Context, sessiif session == nil {return nil, nil}+ if session.stopping {+ return session, nil+ }slog.InfoContext(ctx, "Poll control plane marked edge tunnel idle; closing session")+ session.stopping = trueidleCtx, idleCancel := context.WithTimeout(ctx, defaultPollManagedSessionStopTimeout)err := c.stopPollManagedSessionInternal(idleCtx, session)idleCancel()if err != nil {- return nil, err+ return session, err}return nil, nildefault:@@ -173,7 +180,7 @@ func waitForNextPollCycleInternal(ctx context.Context, session *pollManagedTunne}}-func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, pollURL string, connected bool) (*TunnelPollResponse, error) {+func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, httpClient *http.Client, pollURL string, connected bool) (*TunnelPollResponse, error) {pollReq := TunnelPollRequest{Transport: EdgeTransportPoll,Connected: connected,@@ -196,7 +203,7 @@ func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, pollURL streq.Header.Set(header, value)}- resp, err := c.httpClient.Do(req)+ resp, err := httpClient.Do(req)if err != nil {return nil, errors.WrapIf(err, "poll request failed")}@@ -234,6 +241,7 @@ func (c *TunnelClient) stopPollManagedSessionInternal(ctx context.Context, sessiif session == nil {return nil}+ session.stopping = truesession.cancel()select {@@ -241,7 +249,7 @@ func (c *TunnelClient) stopPollManagedSessionInternal(ctx context.Context, sessireturn errcase <-ctx.Done():if errors.Is(ctx.Err(), context.DeadlineExceeded) {- return errors.New("timed out waiting for poll-managed websocket session to stop")+ return errPollManagedSessionStopTimeout}return ctx.Err()}diff --git a/backend/pkg/libarcane/edge/client_transport_websocket.go b/backend/pkg/libarcane/edge/client_transport_websocket.goindex 740ebd58b..c12ffcef6 100644--- a/backend/pkg/libarcane/edge/client_transport_websocket.go+++ b/backend/pkg/libarcane/edge/client_transport_websocket.go@@ -19,8 +19,6 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {if managerWSURL == "" {return errors.New("manager WebSocket URL is empty")}- c.managerURL = managerWSURL-transport := &http.Transport{Proxy: http.ProxyFromEnvironment}if strings.HasPrefix(strings.ToLower(managerWSURL), "wss://") {tlsConfig, err := buildManagerClientTLSConfigInternal(c.cfg)diff --git a/backend/pkg/libarcane/edge/registry.go b/backend/pkg/libarcane/edge/registry.goindex 30e212d87..861d0bf96 100644--- a/backend/pkg/libarcane/edge/registry.go+++ b/backend/pkg/libarcane/edge/registry.go@@ -1,11 +1,14 @@package edgeimport (- "github.com/samber/mo"-+ "context""log/slog""sync""time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/samber/mo")// NewAgentTunnelWithConn creates a new agent tunnel from a transport-agnostic connection.@@ -94,78 +97,105 @@ func (t *AgentTunnel) CloseWithReason(reason string) error {// NewTunnelRegistry creates a new tunnel registryfunc NewTunnelRegistry() *TunnelRegistry {return &TunnelRegistry{- tunnels: make(map[string]*AgentTunnel),+ tunnels: actors.NewStateMap[string, *AgentTunnel](),}}+// NewActorTunnelRegistry creates a registry owned by the shared actor runtime.+func NewActorTunnelRegistry(ctx context.Context, runtime *actors.Runtime) (*TunnelRegistry, error) {+ tunnels, err := actors.NewActorStateMap[string, *AgentTunnel](ctx, runtime, "edge", "tunnel-registry", 3)+ if err != nil {+ return nil, err+ }+ return &TunnelRegistry{tunnels: tunnels}, nil+}+// Get retrieves a tunnel by environment IDfunc (r *TunnelRegistry) Get(envID string) mo.Option[*AgentTunnel] {- r.mu.RLock()- defer r.mu.RUnlock()- tunnel, ok := r.tunnels[envID]+ tunnel, ok := r.tunnels.Get(envID)return mo.TupleToOption(tunnel, ok)}// Register adds a tunnel to the registry, closing any existing tunnel for the same envfunc (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {- r.mu.Lock()- defer r.mu.Unlock()-- // Close existing tunnel if any- if existing, ok := r.tunnels[envID]; ok {+ if r == nil || tunnel == nil {+ slog.Error("Failed to register edge agent tunnel", "environment_id", envID, "error", "tunnel is required")+ return+ }+ previous, replaced, err := r.tunnels.Store(context.Background(), "register edge tunnel", envID, tunnel)+ if err != nil {+ slog.Error("Failed to register edge agent tunnel", "environment_id", envID, "error", err)+ return+ }+ if replaced && previous != tunnel {slog.Info("Replacing existing edge tunnel")- _ = existing.CloseWithReason("")+ _ = previous.CloseWithReason("")}-- r.tunnels[envID] = tunnelslog.Info("Edge agent tunnel registered")}+type registerSessionResultInternal struct {+ accepted bool+ drainPrevious bool+ reason string+ previous *AgentTunnel+}+// RegisterSession adds a tunnel with duplicate-session handling.-func (r *TunnelRegistry) RegisterSession(tunnel *AgentTunnel, staleAfter time.Duration) (accepted bool, drainPrevious bool, reason string) {+func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunnel, staleAfter time.Duration) (bool, bool, string, error) {if r == nil || tunnel == nil {- return false, false, "tunnel is required"+ return false, false, "tunnel is required", nil}- r.mu.Lock()- defer r.mu.Unlock()-envID := tunnel.EnvironmentIDif envID == "" {- return false, false, "environment ID is required"+ return false, false, "environment ID is required", nil}- if existing, ok := r.tunnels[envID]; ok && existing != nil {- if existing == tunnel {- return true, false, ""- }-- existingStale := staleAfter > 0 && time.Since(existing.GetLastHeartbeat()) > staleAfter- sameAgentInstance := existing.AgentInstance != "" && tunnel.AgentInstance != "" && existing.AgentInstance == tunnel.AgentInstance+ result, err := actors.ApplyStateMap(ctx, r.tunnels, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResultInternal, bool, error) {+ var result registerSessionResultInternal+ if existing := tunnels[envID]; existing != nil {+ if existing == tunnel {+ result.accepted = true+ return result, false, nil+ }- if existing.Conn == nil || existing.Conn.IsClosed() || existingStale || sameAgentInstance {- if sameAgentInstance {- drainPrevious = true+ existingMetadata := existing.MetadataSnapshot()+ tunnelMetadata := tunnel.MetadataSnapshot()+ existingStale := staleAfter > 0 && time.Since(existingMetadata.LastHeartbeat) > staleAfter+ sameAgentInstance := existingMetadata.AgentInstance != "" && tunnelMetadata.AgentInstance != "" && existingMetadata.AgentInstance == tunnelMetadata.AgentInstance+ if existing.Conn != nil && !existing.Conn.IsClosed() && !existingStale && !sameAgentInstance {+ result.reason = "another edge agent session is already active"+ return result, false, nil}- _ = existing.CloseWithReason("replaced by newer edge tunnel session")- } else {- return false, false, "another edge agent session is already active"+ result.previous = existing+ result.drainPrevious = sameAgentInstance}- }- r.tunnels[envID] = tunnel- slog.Info("Edge agent tunnel registered", "environment_id", envID, "session_id", tunnel.SessionID, "security_mode", tunnel.SecurityMode)- return true, drainPrevious, ""+ tunnels[envID] = tunnel+ result.accepted = true+ return result, true, nil+ })+ if err != nil {+ return false, false, "", err+ }+ if result.previous != nil {+ _ = result.previous.CloseWithReason("replaced by newer edge tunnel session")+ }+ metadata := tunnel.MetadataSnapshot()+ slog.Info("Edge agent tunnel registered", "environment_id", envID, "session_id", metadata.SessionID, "security_mode", metadata.SecurityMode)+ return result.accepted, result.drainPrevious, result.reason, nil}// Unregister removes a tunnel from the registryfunc (r *TunnelRegistry) Unregister(envID string) {- r.mu.Lock()- defer r.mu.Unlock()-- if tunnel, ok := r.tunnels[envID]; ok {+ tunnel, removed, err := r.tunnels.Remove(context.Background(), "unregister edge tunnel", envID)+ if err != nil {+ slog.Error("Failed to unregister edge agent tunnel", "environment_id", envID, "error", err)+ return+ }+ if removed {_ = tunnel.CloseWithReason("")- delete(r.tunnels, envID)slog.Info("Edge agent tunnel unregistered")}}@@ -173,48 +203,76 @@ func (r *TunnelRegistry) Unregister(envID string) {// UnregisterCurrent removes the active tunnel only when it still matches the// provided tunnel reference. It reports whether the tunnel was removed and// whether another active session for the environment is already present.-func (r *TunnelRegistry) UnregisterCurrent(envID string, current *AgentTunnel) (removed bool, activeReplacement bool) {+func (r *TunnelRegistry) UnregisterCurrent(ctx context.Context, envID string, current *AgentTunnel) (bool, bool) {if r == nil || current == nil {return false, false}- r.mu.Lock()- defer r.mu.Unlock()-- existing, ok := r.tunnels[envID]- if !ok || existing != current {- if ok && existing != nil && existing.Conn != nil && !existing.Conn.IsClosed() {- return false, true+ type resultInternal struct {+ removed bool+ activeReplacement bool+ }+ result, err := actors.ApplyStateMap(ctx, r.tunnels, "unregister current edge tunnel", func(tunnels map[string]*AgentTunnel) (resultInternal, bool, error) {+ var result resultInternal+ existing := tunnels[envID]+ if existing != current {+ result.activeReplacement = existing != nil && existing.Conn != nil && !existing.Conn.IsClosed()+ return result, false, nil}+ delete(tunnels, envID)+ result.removed = true+ return result, true, nil+ })+ if err != nil {+ slog.ErrorContext(ctx, "Failed to unregister current edge agent tunnel", "environment_id", envID, "error", err)return false, false}+ if !result.removed {+ return false, result.activeReplacement+ }- _ = existing.CloseWithReason("")- delete(r.tunnels, envID)- slog.Info("Edge agent tunnel unregistered", "environment_id", envID, "session_id", current.SessionID)+ _ = current.CloseWithReason("")+ metadata := current.MetadataSnapshot()+ slog.Info("Edge agent tunnel unregistered", "environment_id", envID, "session_id", metadata.SessionID)return true, false}// CleanupStale removes tunnels that haven't had a heartbeat within the given duration.-func (r *TunnelRegistry) CleanupStale(maxAge time.Duration) []*AgentTunnel {- r.mu.Lock()- defer r.mu.Unlock()-+func (r *TunnelRegistry) CleanupStale(ctx context.Context, maxAge time.Duration) []*AgentTunnel {now := time.Now()- removed := make([]*AgentTunnel, 0)-- for envID, tunnel := range r.tunnels {- if now.Sub(tunnel.GetLastHeartbeat()) > maxAge {- slog.Warn("Removing stale edge tunnel", "last_heartbeat", tunnel.GetLastHeartbeat())- _ = tunnel.CloseWithReason("edge tunnel heartbeat expired")- delete(r.tunnels, envID)- removed = append(removed, tunnel)+ removed, err := r.tunnels.RemoveWhere(ctx, "clean up stale edge tunnels", func(_ string, tunnel *AgentTunnel) bool {+ return tunnel == nil || now.Sub(tunnel.GetLastHeartbeat()) > maxAge+ })+ if err != nil {+ slog.ErrorContext(ctx, "Failed to clean up stale edge tunnels", "error", err)+ return nil+ }+ for _, tunnel := range removed {+ if tunnel == nil {+ continue}+ slog.Warn("Removing stale edge tunnel", "last_heartbeat", tunnel.GetLastHeartbeat())+ _ = tunnel.CloseWithReason("edge tunnel heartbeat expired")}return removed}+// Stop drains registry mutations, closes active tunnels, and joins its actor.+func (r *TunnelRegistry) Stop(ctx context.Context) error {+ fallback := r.tunnels.Values()+ removed, err := r.tunnels.Drain(ctx, "stop edge tunnel registry")+ if err != nil {+ removed = fallback+ }+ for _, tunnel := range removed {+ if tunnel != nil {+ err = errors.Combine(err, tunnel.CloseWithReason("edge tunnel registry stopped"))+ }+ }+ return errors.Combine(err, r.tunnels.Stop(ctx))+}+var (defaultRegistryMu sync.RWMutexdefaultRegistry = NewTunnelRegistry()@@ -238,3 +296,12 @@ func SetDefaultRegistry(registry *TunnelRegistry) {defaultRegistry = registrydefaultRegistryMu.Unlock()}++// ClearDefaultRegistry replaces registry only when it is still the active default.+func ClearDefaultRegistry(registry *TunnelRegistry) {+ defaultRegistryMu.Lock()+ if defaultRegistry == registry {+ defaultRegistry = NewTunnelRegistry()+ }+ defaultRegistryMu.Unlock()+}diff --git a/backend/pkg/libarcane/edge/registry_test.go b/backend/pkg/libarcane/edge/registry_test.goindex 1bb96949f..de442d8da 100644--- a/backend/pkg/libarcane/edge/registry_test.go+++ b/backend/pkg/libarcane/edge/registry_test.go@@ -1,6 +1,7 @@package edgeimport (+ "context""net/http""net/http/httptest""strings"@@ -8,10 +9,45 @@ import ("time""github.com/coder/websocket"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest")+type controlledCloseTunnelConn struct {+ closeStarted chan struct{}+ releaseClose chan struct{}+ closed chan struct{}+}++func (c *controlledCloseTunnelConn) Send(*TunnelMessage) error { return nil }+func (c *controlledCloseTunnelConn) Receive() (*TunnelMessage, error) { return nil, nil }+func (c *controlledCloseTunnelConn) IsExpectedReceiveError(error) bool { return false }+func (c *controlledCloseTunnelConn) Transport() string { return EdgeTransportWebSocket }+func (c *controlledCloseTunnelConn) IsClosed() bool {+ select {+ case <-c.closed:+ return true+ default:+ return false+ }+}+func (c *controlledCloseTunnelConn) Close() error {+ select {+ case <-c.closeStarted:+ default:+ close(c.closeStarted)+ }+ <-c.releaseClose+ select {+ case <-c.closed:+ default:+ close(c.closed)+ }+ return nil+}+func createTestConn(t *testing.T) *websocket.Conn {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {conn, err := websocket.Accept(w, r, nil)@@ -91,7 +127,8 @@ func TestTunnelRegistry_RegisterSessionRejectsCompetingAgent(t *testing.T) {tunnel1.AgentInstance = "agent-a"tunnel1.SessionID = "session-a"- accepted, drainPrevious, reason := r.RegisterSession(tunnel1, TunnelStaleTimeout)+ accepted, drainPrevious, reason, err := r.RegisterSession(t.Context(), tunnel1, TunnelStaleTimeout)+ require.NoError(t, err)assert.True(t, accepted)assert.False(t, drainPrevious)assert.Equal(t, "", reason)@@ -102,7 +139,8 @@ func TestTunnelRegistry_RegisterSessionRejectsCompetingAgent(t *testing.T) {tunnel2.AgentInstance = "agent-b"tunnel2.SessionID = "session-b"- accepted, drainPrevious, reason = r.RegisterSession(tunnel2, TunnelStaleTimeout)+ accepted, drainPrevious, reason, err = r.RegisterSession(t.Context(), tunnel2, TunnelStaleTimeout)+ require.NoError(t, err)assert.False(t, accepted)assert.False(t, drainPrevious)assert.Equal(t, "another edge agent session is already active", reason)@@ -122,7 +160,8 @@ func TestTunnelRegistry_RegisterSessionReplacesSameAgentInstance(t *testing.T) {tunnel1 := newWebSocketAgentTunnel(envID, conn1)tunnel1.AgentInstance = "agent-a"tunnel1.SessionID = "session-a"- accepted, drainPrevious, reason := r.RegisterSession(tunnel1, TunnelStaleTimeout)+ accepted, drainPrevious, reason, err := r.RegisterSession(t.Context(), tunnel1, TunnelStaleTimeout)+ require.NoError(t, err)assert.True(t, accepted)assert.False(t, drainPrevious)assert.Equal(t, "", reason)@@ -132,7 +171,8 @@ func TestTunnelRegistry_RegisterSessionReplacesSameAgentInstance(t *testing.T) {tunnel2 := newWebSocketAgentTunnel(envID, conn2)tunnel2.AgentInstance = "agent-a"tunnel2.SessionID = "session-b"- accepted, drainPrevious, reason = r.RegisterSession(tunnel2, TunnelStaleTimeout)+ accepted, drainPrevious, reason, err = r.RegisterSession(t.Context(), tunnel2, TunnelStaleTimeout)+ require.NoError(t, err)assert.True(t, accepted)assert.True(t, drainPrevious)assert.Equal(t, "", reason)@@ -143,6 +183,80 @@ func TestTunnelRegistry_RegisterSessionReplacesSameAgentInstance(t *testing.T) {assert.Equal(t, tunnel2, got)}+func TestActorTunnelRegistryPublishesReplacementBeforeClosingPreviousInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ registry, err := NewActorTunnelRegistry(t.Context(), runtime)+ require.NoError(t, err)++ previousConn := &controlledCloseTunnelConn{+ closeStarted: make(chan struct{}),+ releaseClose: make(chan struct{}),+ closed: make(chan struct{}),+ }+ previous := NewAgentTunnelWithConn("env-actor-registry", previousConn)+ previous.AgentInstance = "agent-a"+ accepted, _, reason, err := registry.RegisterSession(t.Context(), previous, TunnelStaleTimeout)+ require.NoError(t, err)+ require.True(t, accepted, reason)++ replacementConn := &controlledCloseTunnelConn{+ closeStarted: make(chan struct{}),+ releaseClose: make(chan struct{}),+ closed: make(chan struct{}),+ }+ close(replacementConn.releaseClose)+ replacement := NewAgentTunnelWithConn("env-actor-registry", replacementConn)+ replacement.AgentInstance = "agent-a"+ registered := make(chan bool, 1)+ go func() {+ accepted, _, _, registerErr := registry.RegisterSession(t.Context(), replacement, TunnelStaleTimeout)+ require.NoError(t, registerErr)+ registered <- accepted+ }()++ select {+ case <-previousConn.closeStarted:+ case <-time.After(time.Second):+ t.Fatal("replacement did not begin closing the previous tunnel")+ }+ active, ok := registry.Get("env-actor-registry").Get()+ require.True(t, ok)+ require.Same(t, replacement, active)+ close(previousConn.releaseClose)+ require.True(t, <-registered)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, registry.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}++func TestActorTunnelRegistryReportsStoppedExecutorAsInfrastructureFailureInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ registry, err := NewActorTunnelRegistry(t.Context(), runtime)+ require.NoError(t, err)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, registry.Stop(stopCtx))++ tunnel := NewAgentTunnelWithConn("env-stopped-registry", &controlledCloseTunnelConn{+ closeStarted: make(chan struct{}),+ releaseClose: make(chan struct{}),+ closed: make(chan struct{}),+ })+ accepted, drainPrevious, reason, err := registry.RegisterSession(t.Context(), tunnel, TunnelStaleTimeout)+ require.Error(t, err)+ require.False(t, accepted)+ require.False(t, drainPrevious)+ require.Empty(t, reason)+ require.NoError(t, lifecycle.Stop(stopCtx))+}+func TestTunnelRegistry_CleanupStale(t *testing.T) {r := NewTunnelRegistry()envID := "env-1"@@ -159,7 +273,7 @@ func TestTunnelRegistry_CleanupStale(t *testing.T) {r.Register(envID, tunnel)// Cleanup- removed := r.CleanupStale(5 * time.Minute)+ removed := r.CleanupStale(t.Context(), 5*time.Minute)require.Len(t, removed, 1)assert.Same(t, tunnel, removed[0])diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.goindex 61cada4f6..a473b690a 100644--- a/backend/pkg/libarcane/edge/server.go+++ b/backend/pkg/libarcane/edge/server.go@@ -378,7 +378,16 @@ func tokenFromHeadersWithSourceInternal(req *http.Request) (string, string) {}func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx context.Context, tunnel *AgentTunnel) {- accepted, drainPrevious, rejectReason := s.registry.RegisterSession(tunnel, TunnelStaleTimeout)+ accepted, drainPrevious, rejectReason, err := s.registry.RegisterSession(callbackCtx, tunnel, TunnelStaleTimeout)+ if err != nil {+ slog.ErrorContext(ctx, "Failed to register edge agent session",+ "environment_id", tunnel.EnvironmentID,+ "agent_instance_id", tunnel.AgentInstance,+ "error", err,+ )+ _ = tunnel.CloseWithReason("edge agent session registration unavailable")+ return+ }if !accepted {slog.WarnContext(ctx, "Rejected duplicate edge agent session","environment_id", tunnel.EnvironmentID,@@ -421,7 +430,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx co}); err != nil {slog.WarnContext(ctx, "Failed to send register response", "environment_id", tunnel.EnvironmentID, "error", err)_ = tunnel.CloseWithReason("")- removed, active := s.registry.UnregisterCurrent(tunnel.EnvironmentID, tunnel)+ removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)if removed && !active {s.updateConnectionStatusInternal(callbackCtx, tunnel, false)}@@ -431,7 +440,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx cos.updateConnectionStatusInternal(callbackCtx, tunnel, true)defer func() {- removed, active := s.registry.UnregisterCurrent(tunnel.EnvironmentID, tunnel)+ removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)if !removed {return}@@ -587,7 +596,7 @@ func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {case <-ctx.Done():returncase <-ticker.C:- removed := s.registry.CleanupStale(TunnelStaleTimeout)+ removed := s.registry.CleanupStale(ctx, TunnelStaleTimeout)for _, tunnel := range removed {s.updateConnectionStatusInternal(context.WithoutCancel(ctx), tunnel, false)}diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.goindex 0c0f279c9..93d629539 100644--- a/backend/pkg/libarcane/edge/server_test.go+++ b/backend/pkg/libarcane/edge/server_test.go@@ -16,9 +16,11 @@ import ("github.com/coder/websocket""github.com/coder/websocket/wsjson"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/labstack/echo/v5""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest""google.golang.org/grpc/codes""google.golang.org/grpc/metadata""google.golang.org/grpc/status"@@ -644,3 +646,25 @@ func TestTunnelServer_ManageConnectedTunnel_RegistersBeforeSendingGRPCRegisterRe_, ok := server.registry.Get(envID).Get()assert.False(t, ok)}++func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCancellationInternal(t *testing.T) {+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ registry, err := NewActorTunnelRegistry(t.Context(), runtime)+ require.NoError(t, err)+ server := NewTunnelServerWithRegistry(registry, nil, nil)+ tunnel := NewAgentTunnelWithConn("env-cancelled-disconnect", ®isterResponseOrderConn{recvErr: context.Canceled})++ connectionCtx, cancelConnection := context.WithCancel(context.Background())+ cancelConnection()+ server.manageConnectedTunnel(connectionCtx, context.WithoutCancel(connectionCtx), tunnel)++ _, ok := registry.Get(tunnel.EnvironmentID).Get()+ require.False(t, ok)++ stopCtx, cancelStop := context.WithTimeout(context.Background(), time.Second)+ defer cancelStop()+ require.NoError(t, registry.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/pkg/libarcane/edge/tunnel.go b/backend/pkg/libarcane/edge/tunnel.goindex f425eb7cd..2986e597a 100644--- a/backend/pkg/libarcane/edge/tunnel.go+++ b/backend/pkg/libarcane/edge/tunnel.go@@ -368,11 +368,21 @@ func (t *GRPCAgentTunnelConn) Close() error {return nil}+ // Preserve a clean half-close unless an in-flight send must be cancelled first.+ locked := t.mu.TryLock()+ if !locked {+ if t.cancel != nil {+ t.cancel()+ }+ t.mu.Lock()+ }+ defer t.mu.Unlock()+var err errorif t.stream != nil {err = t.stream.CloseSend()}- if t.cancel != nil {+ if locked && t.cancel != nil {t.cancel()}return errdiff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.goindex be8aa4f1e..c5a9b5ccc 100644--- a/backend/pkg/libarcane/edge/types.go+++ b/backend/pkg/libarcane/edge/types.go@@ -9,6 +9,7 @@ import ("time""github.com/coder/websocket"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""google.golang.org/grpc")@@ -47,9 +48,7 @@ type TunnelClient struct {managerURL stringmanagerGRPCAddr stringlocalPort string // Port the agent is running on locally- httpClient *http.Clientconn atomic.Pointer[connBox]- stopCh chan struct{}requestTimeout time.DurationactiveStreams sync.Map // map[string]*activeWSStreamrequestTransfers sync.Map // map[string]*commandRequestTransfer@@ -57,8 +56,11 @@ type TunnelClient struct {preferWebSocketUntil time.TimegrpcFailureStreak intagentInstanceID string- sessionID string- managerCapabilities []string+ registration actors.Snapshot[clientRegistrationInternal]+}++type clientRegistrationInternal struct {+ sessionID string}// connBox wraps the active TunnelConnection so it can be swapped atomically on@@ -106,8 +108,9 @@ type streamingResponseRecorder struct {}type pollManagedTunnelSession struct {- cancel context.CancelFunc- done chan error+ cancel context.CancelFunc+ done chan error+ stopping bool}type CommandRequest struct {@@ -213,8 +216,7 @@ type AgentTunnel struct {// TunnelRegistry manages active edge agent tunnel connectionstype TunnelRegistry struct {- tunnels map[string]*AgentTunnel // environmentID -> tunnel- mu sync.RWMutex+ tunnels *actors.StateMap[string, *AgentTunnel]}type internalTunnelRequestContextKey struct{}diff --git a/backend/pkg/libarcane/settings.go b/backend/pkg/libarcane/settings.goindex 0644a74b4..7eff3522a 100644--- a/backend/pkg/libarcane/settings.go+++ b/backend/pkg/libarcane/settings.go@@ -44,8 +44,9 @@ var cronSettingKeys = []string{var cronParser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)-func IsTimeoutSettingKey(key string) bool {- return slices.Contains(timeoutSettingKeys, key)+// TimeoutSettingKeys returns the settings propagated to remote environments.+func TimeoutSettingKeys() []string {+ return slices.Clone(timeoutSettingKeys)}func IsCronSettingKey(key string) bool {diff --git a/backend/pkg/scheduler/actor_test.go b/backend/pkg/scheduler/actor_test.gonew file mode 100644index 000000000..bec3b5573--- /dev/null+++ b/backend/pkg/scheduler/actor_test.go@@ -0,0 +1,65 @@+package scheduler++import (+ "context"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest"+)++func newTestAdmissionGateInternal(t *testing.T) *actors.Gate[actors.AdmissionKey] {+ t.Helper()+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "scheduler-test-admission", t.Name())+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+ })+ return gate+}++func newJobSchedulerForTestInternal(t testing.TB, ctx context.Context, location *time.Location) *jobSchedulerInternal {+ t.Helper()+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ created, err := NewJobScheduler(ctx, runtime, location)+ require.NoError(t, err)+ scheduler := created.(*jobSchedulerInternal)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, scheduler.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+ })+ return scheduler+}++func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*services.SettingsService, error) {+ t.Helper()+ lifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)+ require.NoError(t, err)+ effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, executor.Stop(stopCtx))+ require.NoError(t, effects.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+ })+ return services.NewSettingsService(ctx, db, executor, effects)+}diff --git a/backend/pkg/scheduler/analytics_job_test.go b/backend/pkg/scheduler/analytics_job_test.goindex 3fb806089..576f26ec4 100644--- a/backend/pkg/scheduler/analytics_job_test.go+++ b/backend/pkg/scheduler/analytics_job_test.go@@ -30,7 +30,7 @@ func setupAnalyticsStateServicesInternal(t *testing.T) (*database.DB, *services.require.NoError(t, db.AutoMigrate(&models.KVEntry{}))wrappedDB := &database.DB{DB: db}- settingsService, err := services.NewSettingsService(ctx, wrappedDB)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, wrappedDB)require.NoError(t, err)require.NoError(t, settingsService.SetStringSetting(ctx, "instanceId", "test-instance"))@@ -157,7 +157,7 @@ func TestAnalyticsJob_Run_SkipsWithinHeartbeatWindowAfterRestart(t *testing.T) {job.now = func() time.Time { return firstAttemptAt }job.Run(ctx)- reloadedSettingsService, err := services.NewSettingsService(ctx, wrappedDB)+ reloadedSettingsService, err := newSettingsServiceForTestInternal(t, ctx, wrappedDB)require.NoError(t, err)restartedJob := NewAnalyticsJob(reloadedSettingsService, services.NewKVService(wrappedDB), server.Client(), cfg)restartedJob.heartbeatURL = server.URLdiff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.goindex 116c3d191..6dbeda741 100644--- a/backend/pkg/scheduler/auto_heal_job.go+++ b/backend/pkg/scheduler/auto_heal_job.go@@ -6,14 +6,15 @@ import ("slices""strings""sync"- "sync/atomic""time"+ "emperror.dev/errors""github.com/moby/moby/api/types/container""github.com/moby/moby/client""github.com/robfig/cron/v3""golang.org/x/sync/errgroup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/models""github.com/getarcaneapp/arcane/backend/v2/internal/services"dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"@@ -24,6 +25,7 @@ import (const AutoHealJobName = "auto-heal"const autoHealInspectConcurrency = 4+const autoHealAdmissionScopeInternal = "auto-heal"// restartRecord tracks restart timestamps for a single container.type restartRecord struct {@@ -35,11 +37,7 @@ type AutoHealJob struct {settingsService *services.SettingsServiceeventService *services.EventServicenotificationService *services.NotificationService-- // running guards against overlapping cron ticks: inspecting every candidate- // can outlast the (as low as 30s) interval, and stacked runs re-evaluate the- // same unhealthy containers concurrently.- running atomic.Bool+ admissionGate *actors.Gate[actors.AdmissionKey]mu sync.Mutexrestarts map[string]*restartRecord@@ -59,14 +57,19 @@ func NewAutoHealJob(settingsService *services.SettingsService,eventService *services.EventService,notificationService *services.NotificationService,-) *AutoHealJob {+ admissionGate *actors.Gate[actors.AdmissionKey],+) (*AutoHealJob, error) {+ if admissionGate == nil {+ return nil, errors.New("auto-heal admission gate unavailable")+ }return &AutoHealJob{dockerClientService: dockerClientService,settingsService: settingsService,eventService: eventService,notificationService: notificationService,+ admissionGate: admissionGate,restarts: make(map[string]*restartRecord),- }+ }, nil}func (j *AutoHealJob) Name() string {@@ -99,11 +102,16 @@ func (j *AutoHealJob) Run(ctx context.Context) {return}- if !j.running.CompareAndSwap(false, true) {+ lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoHealAdmissionScopeInternal})+ if err != nil {+ slog.ErrorContext(ctx, "auto-heal admission failed", "error", err)+ return+ }+ if !admitted {slog.WarnContext(ctx, "auto-heal run still in progress; skipping overlapping run")return}- defer j.running.Store(false)+ defer lease.Release()dockerClient, err := j.getDockerClientInternal(ctx)if err != nil {diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.goindex 85345de3f..dcd6e9766 100644--- a/backend/pkg/scheduler/auto_heal_job_test.go+++ b/backend/pkg/scheduler/auto_heal_job_test.go@@ -8,7 +8,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models"- "github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/libtnb/sqlite""github.com/moby/moby/api/types/container""github.com/moby/moby/client"@@ -133,7 +132,8 @@ func TestAutoHeal_Schedule_Default(t *testing.T) {func TestAutoHeal_ShouldSchedule(t *testing.T) {ctx := context.Background()_, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)- job := NewAutoHealJob(nil, settingsSvc, nil, nil)+ job, err := NewAutoHealJob(nil, settingsSvc, nil, nil, newTestAdmissionGateInternal(t))+ require.NoError(t, err)require.False(t, job.ShouldSchedule(ctx))@@ -164,12 +164,13 @@ func TestAutoHeal_Run_UsesBoundedConcurrency(t *testing.T) {require.NoError(t, err)require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))- settingsSvc, err := services.NewSettingsService(ctx, &database.DB{DB: db})+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, &database.DB{DB: db})require.NoError(t, err)require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", true))require.NoError(t, settingsSvc.SetStringSetting(ctx, "autoHealExcludedContainers", "skip-me"))- job := NewAutoHealJob(nil, settingsSvc, nil, nil)+ job, err := NewAutoHealJob(nil, settingsSvc, nil, nil, newTestAdmissionGateInternal(t))+ require.NoError(t, err)job.getDockerClient = func() (*client.Client, error) { return nil, nil }job.listContainers = func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {return []container.Summary{@@ -219,3 +220,42 @@ func TestAutoHeal_Run_UsesBoundedConcurrency(t *testing.T) {require.LessOrEqual(t, atomic.LoadInt32(&maxConcurrent), int32(autoHealInspectConcurrency))require.Equal(t, int32(4), atomic.LoadInt32(&restarts))}++func TestAutoHealJob_OverlappingRunIsSkippedInternal(t *testing.T) {+ ctx := context.Background()+ _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)+ require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", true))++ job, err := NewAutoHealJob(nil, settingsSvc, nil, nil, newTestAdmissionGateInternal(t))+ require.NoError(t, err)+ job.getDockerClient = func() (*client.Client, error) { return nil, nil }+ started := make(chan struct{}, 1)+ release := make(chan struct{})+ var calls atomic.Int32+ job.listContainers = func(context.Context, *client.Client) ([]container.Summary, error) {+ calls.Add(1)+ select {+ case started <- struct{}{}:+ default:+ }+ <-release+ return nil, nil+ }++ firstDone := make(chan struct{})+ go func() {+ job.Run(ctx)+ close(firstDone)+ }()+ <-started++ job.Run(ctx)+ require.Equal(t, int32(1), calls.Load())++ close(release)+ select {+ case <-firstDone:+ case <-time.After(time.Second):+ t.Fatal("first auto-heal run did not finish")+ }+}diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.goindex c1d5c498c..50cdcf331 100644--- a/backend/pkg/scheduler/auto_update_job.go+++ b/backend/pkg/scheduler/auto_update_job.go@@ -3,12 +3,15 @@ package schedulerimport ("context""log/slog"- "sync/atomic"+ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/types/v2/updater")+const autoUpdateAdmissionScopeInternal = "auto-update"+// pendingUpdateApplierInternal is the slice of UpdaterService the job needs,// kept as an interface so the overlap guard is testable with a fake.type pendingUpdateApplierInternal interface {@@ -18,17 +21,18 @@ type pendingUpdateApplierInternal interface {type AutoUpdateJob struct {updaterService pendingUpdateApplierInternalsettingsService *services.SettingsService- // running guards against overlapping cron ticks: an update-all run can- // outlast its schedule interval, and stacked runs exhaust the shared- // activity slots.- running atomic.Bool+ admissionGate *actors.Gate[actors.AdmissionKey]}-func NewAutoUpdateJob(updaterService *services.UpdaterService, settingsService *services.SettingsService) *AutoUpdateJob {+func NewAutoUpdateJob(updaterService *services.UpdaterService, settingsService *services.SettingsService, admissionGate *actors.Gate[actors.AdmissionKey]) (*AutoUpdateJob, error) {+ if admissionGate == nil {+ return nil, errors.New("auto-update admission gate unavailable")+ }return &AutoUpdateJob{updaterService: updaterService,settingsService: settingsService,- }+ admissionGate: admissionGate,+ }, nil}func (j *AutoUpdateJob) Name() string {@@ -58,11 +62,16 @@ func (j *AutoUpdateJob) Run(ctx context.Context) {return}- if !j.running.CompareAndSwap(false, true) {+ lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})+ if err != nil {+ slog.ErrorContext(ctx, "auto-update admission failed", "error", err)+ return+ }+ if !admitted {slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")return}- defer j.running.Store(false)+ defer lease.Release()slog.InfoContext(ctx, "auto-update run started")diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.goindex f370aa85e..11dc38a5c 100644--- a/backend/pkg/scheduler/auto_update_job_test.go+++ b/backend/pkg/scheduler/auto_update_job_test.go@@ -13,7 +13,8 @@ import (func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T) {ctx := context.Background()_, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)- job := NewAutoUpdateJob(nil, settingsSvc)+ job, err := NewAutoUpdateJob(nil, settingsSvc, newTestAdmissionGateInternal(t))+ require.NoError(t, err)require.False(t, job.ShouldSchedule(ctx))@@ -52,7 +53,7 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {started: make(chan struct{}, 1),release: make(chan struct{}),}- job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc}+ job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc, admissionGate: newTestAdmissionGateInternal(t)}firstDone := make(chan struct{})go func() {diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.goindex d17341d5e..40ffb3841 100644--- a/backend/pkg/scheduler/filesystem_watcher_job.go+++ b/backend/pkg/scheduler/filesystem_watcher_job.go@@ -4,9 +4,10 @@ import ("context""log/slog""runtime"- "sync""time"+ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/services""github.com/getarcaneapp/arcane/backend/v2/pkg/fswatch""github.com/getarcaneapp/arcane/backend/v2/pkg/projects"@@ -17,160 +18,53 @@ type FilesystemWatcherJob struct {templateService *services.TemplateServicesettingsService *services.SettingsServiceprojectScanDepth int- projectsWatcher *fswatch.Watcher- templatesWatcher *fswatch.Watcher- mu sync.Mutex+ lifecycleCtx context.Context+ projectsWatcher *actors.Resource[*fswatch.Watcher]+ templatesWatcher *actors.Resource[*fswatch.Watcher]}func NewFilesystemWatcherJob(+ ctx context.Context,+ actorRuntime *actors.Runtime,projectService *services.ProjectService,templateService *services.TemplateService,settingsService *services.SettingsService,projectScanDepth int,-) *FilesystemWatcherJob {+) (*FilesystemWatcherJob, error) {+ projectsWatcher, err := actors.NewResource(ctx, actorRuntime, "filesystem-watcher", "projects", 3, (*fswatch.Watcher).Stop)+ if err != nil {+ return nil, err+ }+ templatesWatcher, err := actors.NewResource(ctx, actorRuntime, "filesystem-watcher", "templates", 3, (*fswatch.Watcher).Stop)+ if err != nil {+ return nil, errors.Combine(err, projectsWatcher.Stop(ctx))+ }return &FilesystemWatcherJob{projectService: projectService,templateService: templateService,settingsService: settingsService,projectScanDepth: projectScanDepth,- }-}--func RegisterFilesystemWatcherJob(ctx context.Context, projectService *services.ProjectService, templateService *services.TemplateService, settingsService *services.SettingsService, projectScanDepth int) (*FilesystemWatcherJob, error) {- job := NewFilesystemWatcherJob(projectService, templateService, settingsService, projectScanDepth)-- go func() {- if err := job.Start(ctx); err != nil {- slog.ErrorContext(ctx, "Filesystem watcher failed", "error", err)- }- }()-- slog.InfoContext(ctx, "Filesystem watcher job registered")- return job, nil+ lifecycleCtx: ctx,+ projectsWatcher: projectsWatcher,+ templatesWatcher: templatesWatcher,+ }, nil}func (j *FilesystemWatcherJob) Start(ctx context.Context) error {- settings, err := j.settingsService.GetSettings(ctx)- if err != nil {- return err- }- projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)- if err != nil {- return err- }- followProjectSymlinks := settings.FollowProjectSymlinks.IsTrue()- j.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)-- // Watchers are held locally until both are running, then published under the- // lock: assigning to j.projectsWatcher/j.templatesWatcher as they were built- // raced with Stop and RestartProjectsWatcher, and left every watcher created- // on a path that later failed holding its inotify/kqueue descriptors.- var projectsWatcher, templatesWatcher *fswatch.Watcher- published := false- defer func() {- if published {- return- }- // Startup failed somewhere below; release whatever was built. The job- // never took ownership of these, so nothing else will free them.- for _, watcher := range []*fswatch.Watcher{projectsWatcher, templatesWatcher} {- if watcher == nil {- continue- }- if stopErr := watcher.Stop(); stopErr != nil {- slog.ErrorContext(ctx, "Failed to stop watcher after startup error", "error", stopErr)- }- }- }()-- projectsWatcher, err = fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))- if err != nil {- return err- }-- templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)- if err != nil {+ if err := j.projectsWatcher.Restart(ctx, "start projects filesystem watcher", j.startProjectsWatcherInternal); err != nil {return err}-- if j.templateService != nil {- if directoriesOverlapInternal(projectsDirectory, templatesDir) {- slog.ErrorContext(ctx,- "Templates and projects directories overlap; templates watcher disabled to prevent compose files being treated as projects",- "projectsDirectory", projectsDirectory,- "templatesDirectory", templatesDir)- } else {- templatesWatcher, err = fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{- Debounce: 3 * time.Second,- OnChange: j.handleTemplatesChange,- MaxDepth: 1,- })- if err != nil {- return err- }- }- }-- if err := projectsWatcher.Start(ctx); err != nil {- return err- }- if templatesWatcher != nil {- if err := templatesWatcher.Start(ctx); err != nil {- return err- }- }-- j.mu.Lock()- j.projectsWatcher = projectsWatcher- j.templatesWatcher = templatesWatcher- j.mu.Unlock()- published = true-- slog.InfoContext(ctx, "Filesystem watcher started for projects directory",- "path", projectsDirectory)- if templatesWatcher != nil {- slog.InfoContext(ctx, "Filesystem watcher started for templates directory",- "path", templatesDir)- }-- // Initial sync to surface pre-existing resources- if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {- slog.ErrorContext(ctx, "Initial project sync failed", "error", err)+ if err := j.templatesWatcher.Restart(ctx, "start templates filesystem watcher", j.startTemplatesWatcherInternal); err != nil {+ return errors.Combine(err, j.projectsWatcher.Clear(ctx, "clear projects watcher after template startup failure"))}- if j.templateService != nil {- if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {- slog.ErrorContext(ctx, "Initial template sync failed", "error", err)- }- }-- <-ctx.Done()-- return j.Stop()+ return nil}-func (j *FilesystemWatcherJob) Stop() error {- j.mu.Lock()- projectsWatcher := j.projectsWatcher- templatesWatcher := j.templatesWatcher- j.projectsWatcher = nil- j.templatesWatcher = nil- j.mu.Unlock()-- var firstErr error- if projectsWatcher != nil {- if err := projectsWatcher.Stop(); err != nil {- firstErr = err- }- }- if templatesWatcher != nil {- if err := templatesWatcher.Stop(); err != nil && firstErr == nil {- firstErr = err- }- }- return firstErr+func (j *FilesystemWatcherJob) Stop(ctx context.Context) error {+ return errors.Combine(j.projectsWatcher.Stop(ctx), j.templatesWatcher.Stop(ctx))}-func (j *FilesystemWatcherJob) handleFilesystemChange(ctx context.Context) {+func (j *FilesystemWatcherJob) handleFilesystemChangeInternal(ctx context.Context) {slog.InfoContext(ctx, "Filesystem change detected, syncing projects")if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {@@ -185,74 +79,65 @@ func (j *FilesystemWatcherJob) handleProjectFilePathsChangedInternal(ctx contextif len(paths) == 0 || j.projectService == nil {return}- j.handleFilesystemChange(ctx)- j.projectService.HandleProjectFilesChanged(ctx, paths)+ err := j.projectsWatcher.Do(ctx, "sync changed project files", func(workCtx context.Context, _ *fswatch.Watcher) error {+ j.handleFilesystemChangeInternal(workCtx)+ j.projectService.HandleProjectFilesChanged(workCtx, paths)+ return nil+ })+ if err != nil && ctx.Err() == nil && !errors.Is(err, actors.ErrResourceStopped) {+ slog.ErrorContext(ctx, "Failed to dispatch changed project files", "error", err)+ }}-func (j *FilesystemWatcherJob) handleTemplatesChange(ctx context.Context) {- slog.InfoContext(ctx, "Template directory change detected, syncing templates")- if j.templateService == nil {- return- }- if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {- slog.ErrorContext(ctx, "Failed to sync templates after filesystem change", "error", err)- } else {- slog.InfoContext(ctx, "Template sync completed after filesystem change")+func (j *FilesystemWatcherJob) handleTemplatesChangeInternal(ctx context.Context) {+ err := j.templatesWatcher.Do(ctx, "sync changed templates", func(workCtx context.Context, _ *fswatch.Watcher) error {+ slog.InfoContext(workCtx, "Template directory change detected, syncing templates")+ if j.templateService == nil {+ return nil+ }+ if syncErr := j.templateService.SyncLocalTemplatesFromFilesystem(workCtx); syncErr != nil {+ slog.ErrorContext(workCtx, "Failed to sync templates after filesystem change", "error", syncErr)+ } else {+ slog.InfoContext(workCtx, "Template sync completed after filesystem change")+ }+ return nil+ })+ if err != nil && ctx.Err() == nil && !errors.Is(err, actors.ErrResourceStopped) {+ slog.ErrorContext(ctx, "Failed to dispatch template filesystem change", "error", err)}}func (j *FilesystemWatcherJob) RestartProjectsWatcher(ctx context.Context) error {slog.InfoContext(ctx, "Restarting projects filesystem watcher")+ return j.projectsWatcher.Restart(ctx, "restart projects filesystem watcher", j.startProjectsWatcherInternal)+}- j.mu.Lock()- oldProjectsWatcher := j.projectsWatcher- j.projectsWatcher = nil- j.mu.Unlock()-- if oldProjectsWatcher != nil {- if err := oldProjectsWatcher.Stop(); err != nil {- slog.WarnContext(ctx, "Failed to stop projects watcher during restart", "error", err)- }- }-- // Get fresh settings to get the new projects directory+func (j *FilesystemWatcherJob) startProjectsWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {settings, err := j.settingsService.GetSettings(ctx)if err != nil {- return err+ return nil, err}projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)if err != nil {- return err+ return nil, err}- followProjectSymlinks := settings.FollowProjectSymlinks.IsTrue()j.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)- // Create a new watcher with the updated path- sw, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))+ watcher, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(settings.FollowProjectSymlinks.IsTrue()))if err != nil {- return err+ return nil, err}-- // Start the new watcher- if err := sw.Start(ctx); err != nil {- if stopErr := sw.Stop(); stopErr != nil {- slog.ErrorContext(ctx, "Failed to stop projects watcher after restart error", "error", stopErr)- }- return err+ if err := watcher.Start(j.lifecycleCtx); err != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.+ return watcher, err}- j.mu.Lock()- j.projectsWatcher = sw- j.mu.Unlock()-- slog.InfoContext(ctx, "Projects filesystem watcher restarted", "path", projectsDirectory)-- // Perform a sync to ensure we have the latest state from the new directory- if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {- slog.ErrorContext(ctx, "Initial project sync after watcher restart failed", "error", err)+ slog.InfoContext(ctx, "Projects filesystem watcher started", "path", projectsDirectory)+ if j.projectService != nil {+ if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {+ slog.ErrorContext(ctx, "Initial project sync after watcher start failed", "error", err)+ }}-- return nil+ return watcher, nil}func (j *FilesystemWatcherJob) logRecursiveProjectsWatchLimitWarningInternal(ctx context.Context, projectsDirectory string) {@@ -276,66 +161,55 @@ func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlin}func (j *FilesystemWatcherJob) RestartTemplatesWatcher(ctx context.Context) error {- if j.templateService == nil {- return nil- }slog.InfoContext(ctx, "Restarting templates filesystem watcher")+ return j.templatesWatcher.Restart(ctx, "restart templates filesystem watcher", j.startTemplatesWatcherInternal)+}- j.mu.Lock()- oldTemplatesWatcher := j.templatesWatcher- j.templatesWatcher = nil- j.mu.Unlock()-- if oldTemplatesWatcher != nil {- if err := oldTemplatesWatcher.Stop(); err != nil {- slog.WarnContext(ctx, "Failed to stop templates watcher during restart", "error", err)- }+func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {+ if j.templateService == nil {+ return nil, nil}settings, err := j.settingsService.GetSettings(ctx)if err != nil {- return err+ return nil, err}projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)if err != nil {- return err+ return nil, err}templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)if err != nil {- return err+ return nil, err+ }++ if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {+ slog.ErrorContext(ctx, "Initial template sync failed", "error", err)}if directoriesOverlapInternal(projectsDirectory, templatesDir) {slog.ErrorContext(ctx,- "Templates and projects directories overlap; templates watcher not restarted",+ "Templates and projects directories overlap; templates watcher disabled","projectsDirectory", projectsDirectory,"templatesDirectory", templatesDir)- return nil+ return nil, nil}- tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{+ watcher, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{Debounce: 3 * time.Second,- OnChange: j.handleTemplatesChange,+ OnChange: j.handleTemplatesChangeInternal,MaxDepth: 1,})if err != nil {- return err+ return nil, err}- if err := tw.Start(ctx); err != nil {- return err+ if err := watcher.Start(j.lifecycleCtx); err != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.+ return watcher, err}- j.mu.Lock()- j.templatesWatcher = tw- j.mu.Unlock()-- slog.InfoContext(ctx, "Templates filesystem watcher restarted", "path", templatesDir)-- if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {- slog.ErrorContext(ctx, "Initial template sync after watcher restart failed", "error", err)- }+ slog.InfoContext(ctx, "Templates filesystem watcher started", "path", templatesDir)- return nil+ return watcher, nil}// directoriesOverlapInternal returns true when a or b is the same as or contained in thediff --git a/backend/pkg/scheduler/filesystem_watcher_job_test.go b/backend/pkg/scheduler/filesystem_watcher_job_test.goindex e6a8a37cc..408c63d89 100644--- a/backend/pkg/scheduler/filesystem_watcher_job_test.go+++ b/backend/pkg/scheduler/filesystem_watcher_job_test.go@@ -1,9 +1,16 @@package schedulerimport (+ "context"+ "sync""testing"+ "time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/fswatch""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "go.uber.org/fx/fxtest")func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *testing.T) {@@ -16,3 +23,48 @@ func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *teassert.Equal(t, 1, opts.MaxDepth)assert.True(t, opts.FollowSymlinkDirs)}++func TestFilesystemWatcherJob_ConcurrentProjectRestartsAreSerializedInternal(t *testing.T) {+ _, settingsService, _ := setupAnalyticsStateServicesInternal(t)+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "projectsDirectory", t.TempDir()))+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "templatesDirectory", t.TempDir()))++ lifecycle := fxtest.NewLifecycle(t)+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ job, err := NewFilesystemWatcherJob(t.Context(), actorRuntime, nil, nil, settingsService, 2)+ require.NoError(t, err)++ restartErrors := make(chan error, 2)+ var callers sync.WaitGroup+ callers.Add(2)+ for range 2 {+ go func() {+ defer callers.Done()+ restartErrors <- job.RestartProjectsWatcher(t.Context())+ }()+ }+ callers.Wait()+ close(restartErrors)+ for restartErr := range restartErrors {+ require.NoError(t, restartErr)+ }+ var earlyWatcher *fswatch.Watcher+ require.NoError(t, job.projectsWatcher.Do(t.Context(), "capture early watcher", func(_ context.Context, watcher *fswatch.Watcher) error {+ earlyWatcher = watcher+ return nil+ }))+ require.NotNil(t, earlyWatcher)+ require.NoError(t, job.Start(t.Context()))+ var startedWatcher *fswatch.Watcher+ require.NoError(t, job.projectsWatcher.Do(t.Context(), "capture started watcher", func(_ context.Context, watcher *fswatch.Watcher) error {+ startedWatcher = watcher+ return nil+ }))+ require.NotSame(t, earlyWatcher, startedWatcher)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, job.Stop(stopCtx))+ require.NoError(t, lifecycle.Stop(stopCtx))+}diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.goindex d677af00b..fdd564ade 100644--- a/backend/pkg/scheduler/image_update_watcher.go+++ b/backend/pkg/scheduler/image_update_watcher.go@@ -9,6 +9,7 @@ import ("emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/services"@@ -22,6 +23,8 @@ const (imageUpdateWatcherDebounce = 2 * time.SecondimageUpdateWatcherBackfillRetry = 5 * time.SecondimageUpdateWatcherDefaultSchedule = "0 0 * * * *"+ imageUpdateWatcherStopTimeout = 30 * time.Second+ imageUpdateWatcherActorID = "image-update-watcher")type imageUpdateScannerInternal interface {@@ -45,8 +48,23 @@ type projectImageRefsBackfillerInternal interface {BackfillProjectImageRefs(ctx context.Context) (int, error)}-type imageUpdateScanRunInternal struct {- done chan struct{}+type imageUpdateMessageKindInternal uint8++const (+ imageUpdateTriggerMessageInternal imageUpdateMessageKindInternal = iota+ imageUpdateScheduleRefreshMessageInternal+ imageUpdateBackfillCompletedMessageInternal+ imageUpdateDebounceElapsedMessageInternal+ imageUpdateScheduledPollMessageInternal+ imageUpdateScanCompletedMessageInternal+ imageUpdateManualScanRequestMessageInternal+ imageUpdateManualScanAdmissionMessageInternal+)++type imageUpdateScanCompletionInternal struct {+ automatic bool+ triggerGeneration uint64+ done *actors.Promise[error]}// ImageUpdateWatcher continuously reconciles image update state after Docker image changes.@@ -56,18 +74,27 @@ type ImageUpdateWatcher struct {environmentService registryCredentialLoaderInternaldockerService dockerEventBusProviderInternalprojectService projectImageRefsBackfillerInternal- triggerCh chan struct{}- scheduleRefreshCh chan struct{}+ actorRuntime *actors.Runtime+ actorProcess atomic.Pointer[actors.Actor]+ stopping atomic.Bool+ triggerIngress *actors.Ingress[imageUpdateMessageKindInternal, time.Time]+ scheduleIngress *actors.Ingress[imageUpdateMessageKindInternal, actors.NoPayload]location *time.Locationdebounce time.DurationbackfillRetry time.DurationmetadataReady chan struct{}metadataReadyOnce sync.Once- activeRun atomic.Pointer[imageUpdateScanRunInternal]+ started chan struct{}+ startedOnce sync.Once+ stopped chan struct{}+ stoppedOnce sync.Once}// NewImageUpdateWatcher constructs the image update watcher from the existing services.-func NewImageUpdateWatcher(cfg *config.Config, imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService, projectService *services.ProjectService) *ImageUpdateWatcher {+func NewImageUpdateWatcher(runtime *actors.Runtime, cfg *config.Config, imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService, projectService *services.ProjectService) (*ImageUpdateWatcher, error) {+ if runtime == nil || imageUpdateService == nil || settingsService == nil || environmentService == nil || dockerService == nil || projectService == nil {+ return nil, errors.New("image update watcher dependencies unavailable")+ }location := time.UTCif cfg != nil {location = cfg.GetLocation()@@ -78,13 +105,16 @@ func NewImageUpdateWatcher(cfg *config.Config, imageUpdateService *services.ImagenvironmentService: environmentService,dockerService: dockerService,projectService: projectService,- triggerCh: make(chan struct{}, 1),- scheduleRefreshCh: make(chan struct{}, 1),+ actorRuntime: runtime,+ triggerIngress: actors.NewIngress[imageUpdateMessageKindInternal, time.Time](imageUpdateTriggerMessageInternal),+ scheduleIngress: actors.NewIngress[imageUpdateMessageKindInternal, actors.NoPayload](imageUpdateScheduleRefreshMessageInternal),location: location,debounce: imageUpdateWatcherDebounce,backfillRetry: imageUpdateWatcherBackfillRetry,metadataReady: make(chan struct{}),- }+ started: make(chan struct{}),+ stopped: make(chan struct{}),+ }, nil}// Name identifies the watcher in scheduler lifecycle logs.@@ -92,109 +122,153 @@ func (w *ImageUpdateWatcher) Name() string {return "image-polling"}-// Start subscribes to Docker image events, prepares project metadata, and runs scans until ctx is canceled.+// Start subscribes to Docker image events and owns the watcher actor until ctx is canceled.func (w *ImageUpdateWatcher) Start(ctx context.Context) error {- if w == nil || w.dockerService == nil || w.dockerService.EventBus() == nil {- return errors.New("docker event bus unavailable")+ if w.stopping.Load() {+ return errors.New("image update watcher cannot be restarted after stop")+ }+ if w.actorProcess.Load() != nil {+ return errors.New("image update watcher already started")}- if w.projectService == nil || w.metadataReady == nil {- return errors.New("project image metadata backfiller unavailable")+ if w.debounce <= 0 {+ w.debounce = imageUpdateWatcherDebounce+ }+ if w.backfillRetry <= 0 {+ w.backfillRetry = imageUpdateWatcherBackfillRetry}- eventCh, unsubscribe := w.dockerService.EventBus().Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))+ eventBus := w.dockerService.EventBus()+ if eventBus == nil {+ return errors.New("docker event bus unavailable")+ }+ eventCh, unsubscribe := eventBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))defer unsubscribe()- var listener sync.WaitGroup- listenerStarted := make(chan struct{})- listener.Go(func() {- close(listenerStarted)- for {- select {- case <-ctx.Done():- return- case _, ok := <-eventCh:- if !ok {- return- }- w.Trigger()+ process, err := actors.NewActor(+ ctx,+ w.actorRuntime,+ "scheduler",+ imageUpdateWatcherActorID,+ 3,+ func() actors.Behavior {+ state := &imageUpdateWatcherActorInternal{watcher: w}+ return actors.Behavior{+ Initialize: state.initializeInternal,+ Handle: state.receiveInternal,+ Cleanup: state.cleanupInternal,}- }- })- listener.Go(func() {- w.runScheduledPollsInternal(ctx)- })-- select {- case <-ctx.Done():- listener.Wait()- return nil- case <-listenerStarted:+ },+ w.triggerIngress,+ w.scheduleIngress,+ )+ if err != nil {+ return errors.WrapIf(err, "start image update watcher actor")}+ w.actorProcess.Store(process)+ w.startedOnce.Do(func() { close(w.started) })slog.InfoContext(ctx, "image update watcher started")defer func() {- listener.Wait()- slog.InfoContext(ctx, "image update watcher stopped")+ w.actorProcess.CompareAndSwap(process, nil)+ stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), imageUpdateWatcherStopTimeout)+ defer cancel()+ if err := process.Stop(stopCtx); err != nil {+ slog.ErrorContext(context.WithoutCancel(ctx), "failed to stop image update watcher actor", "error", err)+ }+ slog.InfoContext(context.WithoutCancel(ctx), "image update watcher stopped")}()- if !w.backfillProjectImageRefsInternal(ctx) {- return nil+ for {+ select {+ case <-ctx.Done():+ return nil+ case <-process.Done():+ if ctx.Err() != nil || w.stopping.Load() {+ return nil+ }+ return errors.New("image update watcher actor stopped unexpectedly")+ case _, ok := <-eventCh:+ if !ok {+ slog.WarnContext(ctx, "docker image event subscription closed; scheduled image polling remains active")+ eventCh = nil+ continue+ }+ w.Trigger()+ }}-- w.Trigger()- w.runTriggeredScansInternal(ctx)- return nil}-// Trigger queues an image scan without blocking the Docker event publisher.-func (w *ImageUpdateWatcher) Trigger() {- if w == nil {- return+// Stop terminates the watcher actor within the caller's lifecycle deadline.+func (w *ImageUpdateWatcher) Stop(ctx context.Context) error {+ if ctx == nil {+ return errors.New("image update watcher stop context unavailable")}- select {- case w.triggerCh <- struct{}{}:- default:+ w.stopping.Store(true)+ w.stoppedOnce.Do(func() { close(w.stopped) })+ process := w.actorProcess.Load()+ if process == nil {+ return nil}+ return process.Stop(ctx)}-// RefreshSchedule wakes the scheduled-poll loop so it re-reads pollingInterval.+// Trigger records a trailing-edge image scan without blocking the event publisher.+func (w *ImageUpdateWatcher) Trigger() {+ w.triggerIngress.Send(time.Now())+}++// RefreshSchedule wakes the actor so it re-reads pollingInterval.func (w *ImageUpdateWatcher) RefreshSchedule() {- if w == nil || w.scheduleRefreshCh == nil {- return- }- select {- case w.scheduleRefreshCh <- struct{}{}:- default:- }+ w.scheduleIngress.Send(actors.NoPayload{})}-// RunNow performs the same full-host image scan used by automatic watcher-// triggers. It does not wait behind an active scan: a request during one-// returns ImageScanInProgressError immediately instead of parking-// (potentially forever, given the detached run-now context) on the-// single-flight gate. The triggered-scan loop waits and retries itself.+// RunNow requests immediate admission from the watcher actor and waits for that scan.+// An active scan is refused immediately instead of being queued.func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {- if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil || w.metadataReady == nil {- return errors.New("image update watcher is not initialized")+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-w.stopped:+ return errors.New("image update watcher is not running")+ case <-w.started:}select {case <-ctx.Done():return ctx.Err()+ case <-w.stopped:+ return errors.New("image update watcher is not running")case <-w.metadataReady:}- run := &imageUpdateScanRunInternal{done: make(chan struct{})}- for !w.activeRun.CompareAndSwap(nil, run) {- if w.activeRun.Load() != nil {- return common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress"))- }- // Lost the CAS to a run that has since been released; retry.+ process := w.actorProcess.Load()+ if process == nil {+ return errors.New("image update watcher is not running")}- defer func() {- w.activeRun.Store(nil)- close(run.done)- }()+ admission, err := actors.Request[imageUpdateMessageKindInternal, context.Context, *actors.Promise[error]](+ ctx,+ process,+ actors.Message[imageUpdateMessageKindInternal, context.Context]{Kind: imageUpdateManualScanRequestMessageInternal, Value: ctx},+ )+ if err != nil {+ return errors.WrapIf(err, "request image scan admission")+ }+ if admission.Kind != imageUpdateManualScanAdmissionMessageInternal {+ return errors.New("invalid image scan admission response")+ }+ if admission.Err != nil {+ return admission.Err+ }++ select {+ case <-ctx.Done():+ return ctx.Err()+ case err := <-admission.Value.Done():+ return err+ }+}++func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {if err := ctx.Err(); err != nil {return err}@@ -204,7 +278,6 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {}slog.InfoContext(ctx, "image scan run started")-creds, err := w.environmentService.GetEnabledRegistryCredentials(ctx)if err != nil {slog.WarnContext(ctx, "failed to load registry credentials for image scan", "error", err.Error())@@ -216,7 +289,6 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {return errors.WrapIf(err, "image scan failed")}- total := len(results)updates := 0scanErrors := 0for _, result := range results {@@ -231,146 +303,185 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {updates++}}-- slog.InfoContext(ctx, "image scan run completed", "checked", total, "updates", updates, "errors", scanErrors)+ slog.InfoContext(ctx, "image scan run completed", "checked", len(results), "updates", updates, "errors", scanErrors)return nil}-func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context) bool {- retryDelay := w.backfillRetry- if retryDelay <= 0 {- retryDelay = imageUpdateWatcherBackfillRetry+func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context, attempt int) (actors.NoPayload, error) {+ startedAt := time.Now()+ count, err := w.projectService.BackfillProjectImageRefs(ctx)+ duration := time.Since(startedAt)+ if err == nil {+ slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", duration, "attempt", attempt)+ return actors.NoPayload{}, nil}+ if ctx.Err() == nil {+ slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", duration, "attempt", attempt, "retryIn", w.backfillRetry, "error", err)+ }+ return actors.NoPayload{}, err+}- for attempt := 1; ; attempt++ {- if ctx.Err() != nil {- return false- }-- startedAt := time.Now()- count, err := w.projectService.BackfillProjectImageRefs(ctx)- duration := time.Since(startedAt)- if err == nil {- slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", duration, "attempt", attempt)- w.metadataReadyOnce.Do(func() { close(w.metadataReady) })- return true- }- if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {- return false- }+type imageUpdateWatcherActorInternal struct {+ watcher *ImageUpdateWatcher+ scanRunning bool+ automaticPending bool+ triggerGeneration uint64+ metadataReady bool+ debounceTimer actors.Timer[imageUpdateMessageKindInternal]+ scheduleTimer actors.Timer[imageUpdateMessageKindInternal]+}- slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", duration, "attempt", attempt, "retryIn", retryDelay, "error", err)- timer := time.NewTimer(retryDelay)- select {- case <-ctx.Done():- timer.Stop()- return false- case <-timer.C:- }+func (a *imageUpdateWatcherActorInternal) initializeInternal(ctx *actors.Context) {+ select {+ case <-a.watcher.metadataReady:+ a.metadataReady = true+ default:+ a.startBackfillInternal(ctx)}+ a.resetScheduleInternal(ctx)}-func (w *ImageUpdateWatcher) runTriggeredScansInternal(ctx context.Context) {- for {- select {- case <-ctx.Done():+func (a *imageUpdateWatcherActorInternal) cleanupInternal(*actors.Context) {+ a.debounceTimer.Stop()+ a.scheduleTimer.Stop()+}++func (a *imageUpdateWatcherActorInternal) receiveInternal(ctx *actors.Context, rawMessage any) {+ switch message := rawMessage.(type) {+ case actors.Message[imageUpdateMessageKindInternal, actors.NoPayload]:+ a.receiveSignalInternal(ctx, message)+ case actors.Message[imageUpdateMessageKindInternal, uint64]:+ a.receiveTimerInternal(ctx, message)+ case actors.Message[imageUpdateMessageKindInternal, context.Context]:+ if message.Kind != imageUpdateManualScanRequestMessageInternal {return- case <-w.triggerCh:}-- if !w.waitForDebounceInternal(ctx) {+ if a.scanRunning {+ ctx.Respond(actors.Message[imageUpdateMessageKindInternal, *actors.Promise[error]]{+ Kind: imageUpdateManualScanAdmissionMessageInternal,+ Err: common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress")),+ })return}- err := w.RunNow(ctx)- if errors.Is(err, common.ErrImageScanInProgress) {- // A manual scan holds the gate; wait it out and re-queue the- // trigger so this scan attempt is not lost.- if !w.waitForActiveScanInternal(ctx) {- return- }- w.Trigger()- continue+ done := actors.NewPromise[error]()+ a.startScanInternal(message.Value, ctx, false, 0, done)+ ctx.Respond(actors.Message[imageUpdateMessageKindInternal, *actors.Promise[error]]{Kind: imageUpdateManualScanAdmissionMessageInternal, Value: done})+ case actors.Message[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]:+ defer message.Acknowledge()+ if message.Kind != imageUpdateScanCompletedMessageInternal {+ return+ }+ a.scanRunning = false+ if message.Value.automatic {+ a.watcher.triggerIngress.Acknowledge(message.Value.triggerGeneration)+ a.automaticPending = a.watcher.triggerIngress.Pending()+ }+ if message.Value.automatic && message.Err != nil && !errors.Is(message.Err, context.Canceled) {+ slog.ErrorContext(ctx.Context(), "image update watcher scan failed", "error", message.Err)+ }+ if a.automaticPending {+ a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, a.watcher.debounce)}- if err != nil && !errors.Is(err, context.Canceled) {- slog.ErrorContext(ctx, "image update watcher scan failed", "error", err)+ if message.Value.done != nil {+ message.Value.done.Resolve(message.Err)}}}-// runScheduledPollsInternal periodically queues scans based on the-// pollingInterval cron setting so registry-side updates are detected even when-// no local Docker image events occur. Fires funnel through Trigger, sharing-// the same debounce and single-flight path as event-driven scans.-func (w *ImageUpdateWatcher) runScheduledPollsInternal(ctx context.Context) {- if w.scheduleRefreshCh == nil {+func (a *imageUpdateWatcherActorInternal) receiveSignalInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, actors.NoPayload]) {+ if message.Kind == imageUpdateTriggerMessageInternal {+ triggeredAt, generation := a.watcher.triggerIngress.Begin()+ a.automaticPending = true+ a.triggerGeneration = generation+ if a.metadataReady && !a.scanRunning {+ a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, max(time.Until(triggeredAt.Add(a.watcher.debounce)), 0))+ }+ return+ }+ if message.Kind == imageUpdateScheduleRefreshMessageInternal {+ a.watcher.scheduleIngress.Take()+ a.resetScheduleInternal(ctx)return}- location := w.location- if location == nil {- location = time.UTC+ if message.Kind != imageUpdateBackfillCompletedMessageInternal {+ return+ }+ defer message.Acknowledge()+ if message.Err != nil {+ slog.ErrorContext(ctx.Context(), "image update metadata backfill worker failed", "error", message.Err)+ return}+ a.metadataReady = true+ a.watcher.metadataReadyOnce.Do(func() { close(a.watcher.metadataReady) })+ a.watcher.Trigger()+}- for {- spec := imageUpdateWatcherDefaultSchedule- if w.settingsService != nil {- spec = w.settingsService.GetStringSetting(ctx, "pollingInterval", imageUpdateWatcherDefaultSchedule)- }- schedule, err := cronScheduleParser.Parse(spec)- if err != nil {- slog.WarnContext(ctx, "invalid pollingInterval cron expression; using default schedule", "pollingInterval", spec, "default", imageUpdateWatcherDefaultSchedule, "error", err)- schedule, err = cronScheduleParser.Parse(imageUpdateWatcherDefaultSchedule)- if err != nil {- slog.ErrorContext(ctx, "failed to parse default image polling schedule", "error", err)- return- }+func (a *imageUpdateWatcherActorInternal) receiveTimerInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, uint64]) {+ if message.Kind == imageUpdateDebounceElapsedMessageInternal {+ if !a.debounceTimer.Current(message.Value) || !a.automaticPending || a.scanRunning {+ return}-- timer := time.NewTimer(time.Until(schedule.Next(time.Now().In(location))))- select {- case <-ctx.Done():- timer.Stop()+ if remaining := time.Until(a.watcher.triggerIngress.Latest().Add(a.watcher.debounce)); remaining > 0 {+ a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, remaining)return- case <-w.scheduleRefreshCh:- timer.Stop()- case <-timer.C:- w.Trigger()}+ a.startScanInternal(ctx.Context(), ctx, true, a.triggerGeneration, nil)+ return+ }+ if message.Kind != imageUpdateScheduledPollMessageInternal || !a.scheduleTimer.Current(message.Value) {+ return}+ a.watcher.Trigger()+ a.resetScheduleInternal(ctx)}-// waitForActiveScanInternal blocks until any in-flight scan releases the-// single-flight gate. It returns false when ctx is cancelled first.-func (w *ImageUpdateWatcher) waitForActiveScanInternal(ctx context.Context) bool {- activeRun := w.activeRun.Load()- if activeRun == nil {- return true+func (a *imageUpdateWatcherActorInternal) startBackfillInternal(ctx *actors.Context) {+ if a.watcher.backfillRetry <= 0 {+ a.watcher.backfillRetry = imageUpdateWatcherBackfillRetry}- select {- case <-ctx.Done():- return false- case <-activeRun.done:- return true+ attempt := 0+ worker := actors.Worker[imageUpdateMessageKindInternal, actors.NoPayload]{+ Actor: ctx,+ WorkContext: ctx.Context(),+ Label: "image update metadata backfill worker",+ CompletionKind: imageUpdateBackfillCompletedMessageInternal,+ RetryDelay: a.watcher.backfillRetry,}+ worker.Start(func(workerCtx context.Context) (actors.NoPayload, error) {+ attempt+++ return a.watcher.backfillProjectImageRefsInternal(workerCtx, attempt)+ })}-func (w *ImageUpdateWatcher) waitForDebounceInternal(ctx context.Context) bool {- timer := time.NewTimer(w.debounce)- defer timer.Stop()+func (a *imageUpdateWatcherActorInternal) startScanInternal(scanCtx context.Context, ctx *actors.Context, automatic bool, triggerGeneration uint64, done *actors.Promise[error]) {+ a.scanRunning = true+ worker := actors.Worker[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]{+ Actor: ctx,+ WorkContext: scanCtx,+ Label: "image update scan worker",+ CompletionKind: imageUpdateScanCompletedMessageInternal,+ PanicValue: imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done},+ ActorStopped: func(_ imageUpdateScanCompletionInternal, err error) {+ done.Resolve(err)+ },+ }+ worker.Start(func(workerCtx context.Context) (imageUpdateScanCompletionInternal, error) {+ return imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done}, a.watcher.executeScanInternal(workerCtx)+ })+}- for {- select {- case <-ctx.Done():- return false- case <-w.triggerCh:- if !timer.Stop() {- select {- case <-timer.C:- default:- }- }- timer.Reset(w.debounce)- case <-timer.C:- return true+func (a *imageUpdateWatcherActorInternal) resetScheduleInternal(ctx *actors.Context) {+ spec := a.watcher.settingsService.GetStringSetting(ctx.Context(), "pollingInterval", imageUpdateWatcherDefaultSchedule)+ schedule, err := cronScheduleParser.Parse(spec)+ if err != nil {+ slog.WarnContext(ctx.Context(), "invalid pollingInterval cron expression; using default schedule", "pollingInterval", spec, "default", imageUpdateWatcherDefaultSchedule, "error", err)+ schedule, err = cronScheduleParser.Parse(imageUpdateWatcherDefaultSchedule)+ if err != nil {+ slog.ErrorContext(ctx.Context(), "failed to parse default image polling schedule", "error", err)+ return}}++ delay := time.Until(schedule.Next(time.Now().In(a.watcher.location)))+ a.scheduleTimer.Reset(ctx, imageUpdateScheduledPollMessageInternal, delay)}diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex b6cba9172..368639271 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -3,19 +3,22 @@ package schedulerimport ("bytes""context"- "errors"+ "fmt""log/slog""strings""sync""testing""time"+ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/imageupdate""github.com/moby/moby/api/types/events""github.com/stretchr/testify/require""go.getarcane.app/streams/bus"+ "go.uber.org/fx/fxtest")type imageUpdateScannerFakeInternal struct {@@ -24,6 +27,7 @@ type imageUpdateScannerFakeInternal struct {active intmaxActive interrors []error+ panics []boolstartedCh chan intreleaseCh <-chan struct{}}@@ -38,9 +42,15 @@ func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ iif call <= len(s.errors) {err = s.errors[call-1]}+ shouldPanic := call <= len(s.panics) && s.panics[call-1]startedCh := s.startedChreleaseCh := s.releaseChs.mu.Unlock()+ defer func() {+ s.mu.Lock()+ s.active--+ s.mu.Unlock()+ }()if startedCh != nil {select {@@ -55,10 +65,9 @@ func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ icase <-releaseCh:}}-- s.mu.Lock()- s.active--- s.mu.Unlock()+ if shouldPanic {+ panic("deliberate image scan panic")+ }return map[string]*imageupdate.Response{}, err}@@ -166,29 +175,37 @@ func (b *lockedBufferInternal) stringInternal() string {return b.String()}-func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {+func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {+ t.Helper()if backfiller == nil {backfiller = &projectImageRefsBackfillerFakeInternal{}}+ lifecycle := fxtest.NewLifecycle(t)+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, lifecycle.Stop(stopCtx))+ })return &ImageUpdateWatcher{imageUpdateService: scanner,settingsService: settings,environmentService: registryCredentialLoaderFakeInternal{},dockerService: dockerEventBusProviderFakeInternal{eventBus: eventBus},projectService: backfiller,- triggerCh: make(chan struct{}, 1),- scheduleRefreshCh: make(chan struct{}, 1),+ actorRuntime: actorRuntime,+ triggerIngress: actors.NewIngress[imageUpdateMessageKindInternal, time.Time](imageUpdateTriggerMessageInternal),+ scheduleIngress: actors.NewIngress[imageUpdateMessageKindInternal, actors.NoPayload](imageUpdateScheduleRefreshMessageInternal),location: time.UTC,debounce: 10 * time.Millisecond,backfillRetry: 10 * time.Millisecond,metadataReady: make(chan struct{}),+ started: make(chan struct{}),+ stopped: make(chan struct{}),}}-func markImageUpdateWatcherMetadataReadyForTestInternal(watcher *ImageUpdateWatcher) {- watcher.metadataReadyOnce.Do(func() { close(watcher.metadataReady) })-}-func startImageUpdateWatcherForTestInternal(t *testing.T, watcher *ImageUpdateWatcher) (context.CancelFunc, <-chan error) {t.Helper()ctx, cancel := context.WithCancel(context.Background())@@ -207,7 +224,7 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tesscanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -235,6 +252,22 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tesrequire.Equal(t, 2, scanner.countInternal())}+func TestImageUpdateWatcher_TrailingEdgeDebounceExtendsWithNewTriggers(t *testing.T) {+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ watcher.debounce = 50 * time.Millisecond+ startImageUpdateWatcherForTestInternal(t, watcher)++ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)+ watcher.Trigger()+ time.Sleep(30 * time.Millisecond)+ watcher.Trigger()+ time.Sleep(30 * time.Millisecond)+ require.Equal(t, 1, scanner.countInternal())+ require.Eventually(t, func() bool { return scanner.countInternal() == 2 }, time.Second, time.Millisecond)+}+func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testing.T) {releaseCh := make(chan struct{})scanner := &imageUpdateScannerFakeInternal{@@ -243,7 +276,7 @@ func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testin}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Equal(t, 1, <-scanner.startedCh)@@ -262,7 +295,7 @@ func TestImageUpdateWatcher_DisabledTriggersAreSkippedUntilEnabled(t *testing.T)scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: false}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)time.Sleep(30 * time.Millisecond)@@ -281,7 +314,7 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{errors: []error{errors.New("registry unavailable")}}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)@@ -296,22 +329,46 @@ func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *testreleaseCh: releaseCh,}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)- markImageUpdateWatcherMetadataReadyForTestInternal(watcher)-- firstErrCh := make(chan error, 1)- go func() { firstErrCh <- watcher.RunNow(context.Background()) }()+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ startImageUpdateWatcherForTestInternal(t, watcher)require.Equal(t, 1, <-scanner.startedCh)require.ErrorIs(t, watcher.RunNow(context.Background()), common.ErrImageScanInProgress)require.Equal(t, 1, scanner.countInternal())close(releaseCh)- require.NoError(t, <-firstErrCh)+ require.Eventually(t, func() bool {+ return watcher.RunNow(context.Background()) == nil+ }, time.Second, time.Millisecond)+ require.Equal(t, 2, scanner.countInternal())require.Equal(t, 1, scanner.maxActiveInternal())+}++func TestImageUpdateWatcher_RunNowConsumesHollywoodResponseInternal(t *testing.T) {+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ startImageUpdateWatcherForTestInternal(t, watcher)++ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)+ require.Eventually(t, func() bool {+ return watcher.RunNow(context.Background()) == nil+ }, time.Second, time.Millisecond)+ for range 10 {+ require.NoError(t, watcher.RunNow(context.Background()))+ }+}++func TestImageUpdateWatcher_RunNowReturnsContainedScanPanicInternal(t *testing.T) {+ scanner := &imageUpdateScannerFakeInternal{panics: []bool{false, true}}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ startImageUpdateWatcherForTestInternal(t, watcher)+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)- // The gate is free again after the first scan finishes.- require.NoError(t, watcher.RunNow(context.Background()))+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.ErrorContains(t, watcher.RunNow(ctx), "image update scan worker panicked")require.Equal(t, 2, scanner.countInternal())}@@ -323,18 +380,26 @@ func TestImageUpdateWatcher_TriggeredScanRetriesAfterManualScanFinishes(t *testi}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)startImageUpdateWatcherForTestInternal(t, watcher)// Let the startup scan finish so the gate is free.require.Equal(t, 1, <-scanner.startedCh)releaseCh <- struct{}{}- require.Eventually(t, func() bool { return watcher.activeRun.Load() == nil }, time.Second, time.Millisecond)-// A manual scan takes the gate, then a Docker event fires: the triggered// loop must wait out the manual scan and still run its own scan after.manualErrCh := make(chan error, 1)- go func() { manualErrCh <- watcher.RunNow(context.Background()) }()+ go func() {+ for {+ err := watcher.RunNow(context.Background())+ if errors.Is(err, common.ErrImageScanInProgress) {+ time.Sleep(time.Millisecond)+ continue+ }+ manualErrCh <- err+ return+ }+ }()require.Equal(t, 2, <-scanner.startedCh)eventBus.Publish(events.Message{Type: events.ImageEventType, Action: events.ActionPull})@@ -375,7 +440,7 @@ func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *testscanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}eventBus := bus.NewDockerEventBus()- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, backfiller)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, backfiller)startImageUpdateWatcherForTestInternal(t, watcher)<-backfillStarted@@ -397,13 +462,31 @@ func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *testt.Logf("coalesced %d image events into one scan after backfilling %d projects in %s", eventCount, projectCount, time.Since(burstStartedAt))}+func TestImageUpdateWatcher_BackfillRetriesContainedPanicInternal(t *testing.T) {+ backfiller := &projectImageRefsBackfillerFakeInternal{+ run: func(_ context.Context, call int) (int, error) {+ if call == 1 {+ panic("deliberate backfill panic")+ }+ return 1, nil+ },+ }+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)+ startImageUpdateWatcherForTestInternal(t, watcher)++ require.Eventually(t, func() bool { return backfiller.countInternal() == 2 }, time.Second, time.Millisecond)+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)+}+func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {secondAttemptStarted := make(chan struct{})releaseSecondAttempt := make(chan struct{})backfiller := &projectImageRefsBackfillerFakeInternal{run: func(ctx context.Context, call int) (int, error) {if call == 1 {- return 0, errors.New("database unavailable")+ return 0, fmt.Errorf("database statement timeout: %w", context.DeadlineExceeded)}close(secondAttemptStarted)select {@@ -416,7 +499,7 @@ func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {}scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)startImageUpdateWatcherForTestInternal(t, watcher)select {@@ -442,7 +525,7 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.}scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)ctx, cancel := context.WithCancel(context.Background())errCh := make(chan error, 1)go func() { errCh <- watcher.Start(ctx) }()@@ -461,17 +544,61 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true, schedule: "* * * * * *"}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)startImageUpdateWatcherForTestInternal(t, watcher)require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, 3*time.Second, 10*time.Millisecond)}+func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsActorRunning(t *testing.T) {+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ eventBus := bus.NewDockerEventBus()+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)+ startImageUpdateWatcherForTestInternal(t, watcher)++ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)+ eventBus.Close()+ watcher.Trigger()++ require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, time.Second, 5*time.Millisecond)+ require.NotNil(t, watcher.actorProcess.Load())+}++func TestImageUpdateWatcher_RunnerRestartsAfterUnexpectedActorExit(t *testing.T) {+ scanner := &imageUpdateScannerFakeInternal{}+ settings := &pollingSettingReaderFakeInternal{enabled: true}+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)+ runner, err := actors.NewRunner(+ t.Context(),+ watcher.actorRuntime,+ "image-update-watcher-test",+ "supervision",+ "supervised image update watcher",+ 3,+ watcher.Start,+ )+ require.NoError(t, err)++ require.Eventually(t, func() bool { return watcher.actorProcess.Load() != nil }, time.Second, time.Millisecond)+ firstProcess := watcher.actorProcess.Load()+ require.NoError(t, firstProcess.Stop(t.Context()))+ require.Eventually(t, func() bool {+ current := watcher.actorProcess.Load()+ return current != nil && current != firstProcess+ }, time.Second, time.Millisecond)++ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, runner.Stop(stopCtx))+ require.NoError(t, watcher.Stop(stopCtx))+}+func TestImageUpdateWatcher_RunNowWaitsForMetadataReadiness(t *testing.T) {scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true}- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)defer cancel()diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.goindex dcea6ab5b..b4ccd5d1e 100644--- a/backend/pkg/scheduler/job_schedule_integration_test.go+++ b/backend/pkg/scheduler/job_schedule_integration_test.go@@ -23,13 +23,13 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {appConfig := &config.Config{Timezone: "UTC"}lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())- jobScheduler := NewJobScheduler(lifecycleCtx, appConfig.GetLocation())+ jobScheduler := newJobSchedulerForTestInternal(t, lifecycleCtx, appConfig.GetLocation())jobService := services.NewJobService(db, settingsService, appConfig)jobService.SetScheduler(lifecycleCtx, jobScheduler)- jobScheduler.StartScheduler()+ require.NoError(t, jobScheduler.StartScheduler())- _, ok := jobScheduler.GetJobRuntimeState("image-polling").Get()+ _, ok := jobScheduler.GetJobRuntimeState("image-polling")require.False(t, ok)require.Empty(t, jobScheduler.cron.Entries())@@ -45,7 +45,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {require.Equal(t, requestedSchedule, persisted.Value)require.Equal(t, requestedSchedule, settingsService.GetStringSetting(ctx, "pollingInterval", ""))- _, ok = jobScheduler.GetJobRuntimeState("image-polling").Get()+ _, ok = jobScheduler.GetJobRuntimeState("image-polling")require.False(t, ok)require.Empty(t, jobScheduler.cron.Entries())@@ -63,17 +63,17 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {restartDB, restartSettingsService := openJobScheduleTestDatabaseInternal(t, ctx, databasePath)restartLifecycleCtx, cancelRestartLifecycle := context.WithCancel(context.Background())- restartScheduler := NewJobScheduler(restartLifecycleCtx, appConfig.GetLocation())+ restartScheduler := newJobSchedulerForTestInternal(t, restartLifecycleCtx, appConfig.GetLocation())restartJobService := services.NewJobService(restartDB, restartSettingsService, appConfig)restartJobService.SetScheduler(restartLifecycleCtx, restartScheduler)- restartScheduler.StartScheduler()+ require.NoError(t, restartScheduler.StartScheduler())t.Cleanup(func() {cancelRestartLifecycle()waitForSchedulerStopInternal(restartScheduler)closeJobScheduleTestDatabaseInternal(t, restartDB)})- _, ok = restartScheduler.GetJobRuntimeState("image-polling").Get()+ _, ok = restartScheduler.GetJobRuntimeState("image-polling")require.False(t, ok)require.Empty(t, restartScheduler.cron.Entries())require.Equal(t, requestedSchedule, restartSettingsService.GetStringSetting(ctx, "pollingInterval", ""))@@ -87,7 +87,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {_, err = restartJobService.UpdateJobSchedules(ctx, jobschedule.Update{PollingInterval: &disabledSchedule})require.NoError(t, err)- _, ok = restartScheduler.GetJobRuntimeState("image-polling").Get()+ _, ok = restartScheduler.GetJobRuntimeState("image-polling")require.False(t, ok)require.Empty(t, restartScheduler.cron.Entries())@@ -106,7 +106,7 @@ func openJobScheduleTestDatabaseInternal(t *testing.T, ctx context.Context, datarequire.NoError(t, gormDB.AutoMigrate(&models.SettingVariable{}))db := &database.DB{DB: gormDB}- settingsService, err := services.NewSettingsService(ctx, db)+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsService.EnsureDefaultSettings(ctx))require.NoError(t, settingsService.LoadDatabaseSettings(ctx))@@ -122,7 +122,7 @@ func closeJobScheduleTestDatabaseInternal(t *testing.T, db *database.DB) {require.NoError(t, sqlDB.Close())}-func waitForSchedulerStopInternal(jobScheduler *JobScheduler) {+func waitForSchedulerStopInternal(jobScheduler *jobSchedulerInternal) {<-jobScheduler.cron.Stop().Done()}diff --git a/backend/pkg/scheduler/scheduled_prune_job_test.go b/backend/pkg/scheduler/scheduled_prune_job_test.goindex a703c82e1..a7cc2c63d 100644--- a/backend/pkg/scheduler/scheduled_prune_job_test.go+++ b/backend/pkg/scheduler/scheduled_prune_job_test.go@@ -21,7 +21,7 @@ func setupScheduledPruneSettingsServiceInternal(t *testing.T) *services.Settingsrequire.NoError(t, err)require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))- svc, err := services.NewSettingsService(context.Background(), &database.DB{DB: db})+ svc, err := newSettingsServiceForTestInternal(t, context.Background(), &database.DB{DB: db})require.NoError(t, err)return svcdiff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.goindex 2c7ee3e64..647574960 100644--- a/backend/pkg/scheduler/scheduler.go+++ b/backend/pkg/scheduler/scheduler.go@@ -3,14 +3,15 @@ package schedulerimport ("context""log/slog"- "sync"+ "maps"+ "slices""time""emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/robfig/cron/v3"- "github.com/samber/mo")// cronScheduleParser is the shared parser for all cron settings: six fields@@ -18,115 +19,164 @@ import (// schedule with the same spec so Jobs-UI cron values behave identically.var cronScheduleParser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)-type JobScheduler struct {- // mu guards jobs, jobsByID, entryIDs and schedules. It is held across the cron- // add/remove calls (which are themselves quick and never block on job- // execution) but never across job.Run — Run executes on cron's own- // goroutine, so a running job must not call back into a locking method here.- mu sync.Mutex- cron *cron.Cron- jobs []schedulertypes.Job- jobsByID map[string]schedulertypes.Job- watchers map[string]schedulertypes.BusWatcher- entryIDs map[string]cron.EntryID- schedules map[string]string- parser cron.Parser- context context.Context- location *time.Location- watcherWG sync.WaitGroup+var errJobSchedulerStoppedInternal = errors.Sentinel("job scheduler stopped")++type schedulerStateInternal struct {+ jobs []schedulertypes.Job+ jobsByID map[string]schedulertypes.Job+ watchers map[string]schedulertypes.BusWatcher+ allWatchers map[string]schedulertypes.BusWatcher+ runners map[string]*actors.Runner+ entryIDs map[string]cron.EntryID+ schedules map[string]string+ stopping bool}-// NewJobScheduler creates a new job scheduler with the specified timezone location.-// The location is used for interpreting cron expressions.-// If location is nil, UTC is used.-func NewJobScheduler(ctx context.Context, location *time.Location) *JobScheduler {+type watcherRegistrationInternal struct {+ watcher schedulertypes.BusWatcher+ runner *actors.Runner+}++type schedulerShutdownInternal struct {+ cronDone <-chan struct{}+ watchers []watcherRegistrationInternal+}++// jobSchedulerInternal serializes its control plane through one shared actor executor.+// robfig/cron remains the timing engine and invokes job bodies outside the actor.+type jobSchedulerInternal struct {+ cron *cron.Cron+ parser cron.Parser+ context context.Context+ location *time.Location+ runtime *actors.Runtime+ state *actors.State[schedulerStateInternal]+}++// NewJobScheduler creates an actor-owned scheduler control plane.+func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *time.Location) (schedulertypes.JobScheduler, error) {+ if ctx == nil {+ return nil, errors.New("job scheduler context unavailable")+ }+ if runtime == nil {+ return nil, errors.New("actor runtime unavailable")+ }if location == nil {location = time.UTC}+ initial := schedulerStateInternal{+ jobs: []schedulertypes.Job{},+ jobsByID: make(map[string]schedulertypes.Job),+ watchers: make(map[string]schedulertypes.BusWatcher),+ allWatchers: make(map[string]schedulertypes.BusWatcher),+ runners: make(map[string]*actors.Runner),+ entryIDs: make(map[string]cron.EntryID),+ schedules: make(map[string]string),+ }+ state, err := actors.NewState(ctx, runtime, "scheduler", "control-plane", 3, initial, func(value schedulerStateInternal) schedulerStateInternal {+ value.jobs = slices.Clone(value.jobs)+ value.jobsByID = maps.Clone(value.jobsByID)+ value.watchers = maps.Clone(value.watchers)+ value.allWatchers = maps.Clone(value.allWatchers)+ value.runners = maps.Clone(value.runners)+ value.entryIDs = maps.Clone(value.entryIDs)+ value.schedules = maps.Clone(value.schedules)+ return value+ })+ if err != nil {+ return nil, err+ }parser := cronScheduleParser- slog.InfoContext(ctx, "Initializing job scheduler", "timezone", location.String())- return &JobScheduler{- cron: cron.New(cron.WithParser(parser), cron.WithLocation(location)),- jobs: []schedulertypes.Job{},- jobsByID: make(map[string]schedulertypes.Job),- watchers: make(map[string]schedulertypes.BusWatcher),- entryIDs: make(map[string]cron.EntryID),- schedules: make(map[string]string),- parser: parser,- context: ctx,- location: location,+ scheduler := &jobSchedulerInternal{+ cron: cron.New(cron.WithParser(parser), cron.WithLocation(location)),+ parser: parser,+ context: ctx,+ location: location,+ runtime: runtime,+ state: state,}+ slog.InfoContext(ctx, "Initializing job scheduler", "timezone", location.String())+ return scheduler, nil}-// RegisterJob records a static job to be scheduled when StartScheduler runs. Use-// AddJob for jobs added dynamically at runtime.-func (js *JobScheduler) RegisterJob(job schedulertypes.Job) {- js.mu.Lock()- defer js.mu.Unlock()- js.jobs = append(js.jobs, job)- js.jobsByID[job.Name()] = job+// RegisterJob records a static job to be scheduled when StartScheduler runs.+func (js *jobSchedulerInternal) RegisterJob(job schedulertypes.Job) error {+ if job == nil {+ return errors.New("scheduler job unavailable")+ }+ return js.state.Apply(js.context, "register scheduler job", func(_ context.Context, state *schedulerStateInternal) error {+ if state.stopping {+ return errJobSchedulerStoppedInternal+ }+ state.jobs = append(state.jobs, job)+ state.jobsByID[job.Name()] = job+ return nil+ })}-// RegisterBusWatcher starts a continuous event watcher on the scheduler lifecycle.-func (js *JobScheduler) RegisterBusWatcher(watcher schedulertypes.BusWatcher, canRunManually bool) {+// RegisterBusWatcher starts a continuous watcher through a shared actor runner.+func (js *jobSchedulerInternal) RegisterBusWatcher(watcher schedulertypes.BusWatcher, canRunManually bool) error {if watcher == nil {- return+ return errors.New("scheduler bus watcher unavailable")}- if canRunManually {- js.mu.Lock()- js.watchers[watcher.Name()] = watcher- js.mu.Unlock()- }-- js.watcherWG.Go(func() {- // A panic in a watcher goroutine would crash the process; contain it the- // same way scheduled job runs are contained.- defer utils.RecoverToError(nil, "bus watcher", "name", watcher.Name())- if err := watcher.Start(js.context); err != nil {- slog.ErrorContext(js.context, "Bus watcher failed", "name", watcher.Name(), "error", err)+ return js.state.Apply(js.context, "register scheduler bus watcher", func(_ context.Context, state *schedulerStateInternal) error {+ if state.stopping {+ return errJobSchedulerStoppedInternal+ }+ runner, err := actors.NewRunner(js.context, js.runtime, "scheduler-watchers", watcher.Name(), "bus watcher "+watcher.Name(), 3, watcher.Start) //nolint:contextcheck // watcher lifetime belongs to the scheduler, not one state-actor receiver incarnation.+ if err != nil {+ return err}+ state.runners[watcher.Name()] = runner+ state.allWatchers[watcher.Name()] = watcher+ if canRunManually {+ state.watchers[watcher.Name()] = watcher+ }+ return nil})}// RunBusWatcherNow runs a watcher through its serialized manual path.-func (js *JobScheduler) RunBusWatcherNow(ctx context.Context, watcherID string) error {- js.mu.Lock()- watcher, ok := js.watchers[watcherID]- js.mu.Unlock()+func (js *jobSchedulerInternal) RunBusWatcherNow(ctx context.Context, watcherID string) error {+ snapshot, ok := js.state.Load()+ if !ok {+ return errors.New("job scheduler state unavailable")+ }+ watcher, ok := snapshot.watchers[watcherID]if !ok {return errors.Errorf("bus watcher %s is not manually runnable", watcherID)}-return watcher.RunNow(ctx)}-func (js *JobScheduler) GetJob(jobID string) mo.Option[schedulertypes.Job] {- js.mu.Lock()- defer js.mu.Unlock()- job, ok := js.jobsByID[jobID]- return mo.TupleToOption(job, ok)+func (js *jobSchedulerInternal) GetJob(jobID string) (schedulertypes.Job, bool) {+ snapshot, ok := js.state.Load()+ if !ok {+ return nil, false+ }+ job, ok := snapshot.jobsByID[jobID]+ return job, ok}// GetJobRuntimeState returns the schedule currently installed for a registered job.-func (js *JobScheduler) GetJobRuntimeState(jobID string) mo.Option[schedulertypes.JobRuntimeState] {- js.mu.Lock()- defer js.mu.Unlock()-- if _, ok := js.jobsByID[jobID]; !ok {- return mo.None[schedulertypes.JobRuntimeState]()+func (js *jobSchedulerInternal) GetJobRuntimeState(jobID string) (schedulertypes.JobRuntimeState, bool) {+ snapshot, ok := js.state.Load()+ if !ok {+ return schedulertypes.JobRuntimeState{}, false+ }+ if _, ok := snapshot.jobsByID[jobID]; !ok {+ return schedulertypes.JobRuntimeState{}, false}- state := schedulertypes.JobRuntimeState{Schedule: js.schedules[jobID]}- entryID, ok := js.entryIDs[jobID]+ state := schedulertypes.JobRuntimeState{Schedule: snapshot.schedules[jobID]}+ entryID, ok := snapshot.entryIDs[jobID]if !ok {- return mo.Some(state)+ return state, true}-entry := js.cron.Entry(entryID)if entry.ID == 0 {- return mo.Some(state)+ return state, true}-state.Scheduled = truenextRun := entry.Nextif nextRun.IsZero() && entry.Schedule != nil {@@ -135,102 +185,139 @@ func (js *JobScheduler) GetJobRuntimeState(jobID string) mo.Option[schedulertypeif !nextRun.IsZero() {state.NextRun = new(nextRun)}-- return mo.Some(state)+ return state, true}-// HasJob reports whether a job with the given name is currently registered.-func (js *JobScheduler) HasJob(jobID string) bool {- js.mu.Lock()- defer js.mu.Unlock()- _, ok := js.jobsByID[jobID]+// HasJob reports whether a job with the given name is registered.+func (js *jobSchedulerInternal) HasJob(jobID string) bool {+ snapshot, ok := js.state.Load()+ if !ok {+ return false+ }+ _, ok = snapshot.jobsByID[jobID]return ok}-func (js *JobScheduler) StartScheduler() {- js.mu.Lock()- for _, job := range js.jobs {- if err := js.upsertJobInternal(js.context, job); err != nil {- slog.ErrorContext(js.context, "Failed to schedule job", "name", job.Name(), "error", err)+// StartScheduler installs static schedules and starts cron.+func (js *jobSchedulerInternal) StartScheduler() error {+ return js.state.Apply(js.context, "start job scheduler", func(actorCtx context.Context, state *schedulerStateInternal) error {+ if state.stopping {+ return errJobSchedulerStoppedInternal}- }- js.mu.Unlock()- js.cron.Start()+ for _, job := range state.jobs {+ if err := js.upsertJobInternal(actorCtx, state, job); err != nil {+ slog.ErrorContext(actorCtx, "Failed to schedule job; continuing scheduler startup", "name", job.Name(), "error", err)+ }+ }+ js.cron.Start()+ return nil+ })}-// AddJob registers and schedules a job at runtime. It is an idempotent upsert: a-// replacement expression is validated before the existing entry is removed, then-// the replacement is installed without leaking a second live entry.-// Safe to call before or after StartScheduler.-func (js *JobScheduler) AddJob(ctx context.Context, job schedulertypes.Job) error {- js.mu.Lock()- defer js.mu.Unlock()- return js.upsertJobInternal(ctx, job)+// AddJob registers and schedules a job at runtime.+func (js *jobSchedulerInternal) AddJob(ctx context.Context, job schedulertypes.Job) error {+ return js.state.Apply(ctx, "add scheduler job", func(_ context.Context, state *schedulerStateInternal) error {+ if state.stopping {+ return errJobSchedulerStoppedInternal+ }+ return js.upsertJobInternal(ctx, state, job)+ })}-// RemoveJob unschedules and forgets a job by name. It is a no-op (not an error)-// when no job with that name is registered.-func (js *JobScheduler) RemoveJob(ctx context.Context, jobName string) {- js.mu.Lock()- defer js.mu.Unlock()-- if entryID, ok := js.entryIDs[jobName]; ok {- js.cron.Remove(entryID)- delete(js.entryIDs, jobName)- }- delete(js.jobsByID, jobName)- delete(js.schedules, jobName)- for i, j := range js.jobs {- if j.Name() == jobName {- js.jobs = append(js.jobs[:i], js.jobs[i+1:]...)- break+// RemoveJob unschedules and forgets a job. Actor failures are logged because+// DynamicScheduler preserves the historical no-result removal contract.+func (js *jobSchedulerInternal) RemoveJob(ctx context.Context, jobName string) {+ err := js.state.Apply(ctx, "remove scheduler job", func(_ context.Context, state *schedulerStateInternal) error {+ if state.stopping {+ return errJobSchedulerStoppedInternal+ }+ if entryID, ok := state.entryIDs[jobName]; ok {+ js.cron.Remove(entryID)+ delete(state.entryIDs, jobName)+ }+ delete(state.jobsByID, jobName)+ delete(state.schedules, jobName)+ for index, job := range state.jobs {+ if job.Name() == jobName {+ state.jobs = append(state.jobs[:index], state.jobs[index+1:]...)+ break+ }}+ slog.DebugContext(ctx, "Job removed", "name", jobName)+ return nil+ })+ if err != nil && !errors.Is(err, errJobSchedulerStoppedInternal) {+ slog.ErrorContext(ctx, "Failed to remove scheduler job", "name", jobName, "error", err)}- slog.DebugContext(ctx, "Job removed", "name", jobName)}-func (js *JobScheduler) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {- js.mu.Lock()- defer js.mu.Unlock()- return js.upsertJobInternal(ctx, job)+func (js *jobSchedulerInternal) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {+ return js.state.Apply(ctx, "reschedule job", func(_ context.Context, state *schedulerStateInternal) error {+ if state.stopping {+ return errJobSchedulerStoppedInternal+ }+ return js.upsertJobInternal(ctx, state, job)+ })}-// GetLocation returns the timezone location used by the scheduler for cron expressions.-func (js *JobScheduler) GetLocation() *time.Location {+func (js *jobSchedulerInternal) GetLocation() *time.Location {return js.location}-// Stop stops the scheduler and waits for running jobs and bus watchers to finish.-// If ctx is canceled or reaches its deadline first, Stop returns ctx.Err().-func (js *JobScheduler) Stop(ctx context.Context) error {- cronDone := js.cron.Stop().Done()- watchersDone := make(chan struct{})- go func() {- js.watcherWG.Wait()- close(watchersDone)- }()-- select {- case <-cronDone:- case <-ctx.Done():- return ctx.Err()+// Stop fences the control plane, stops cron, and joins every actor-owned watcher.+func (js *jobSchedulerInternal) Stop(ctx context.Context) error {+ var shutdown schedulerShutdownInternal+ err := js.state.Apply(ctx, "stop job scheduler", func(_ context.Context, state *schedulerStateInternal) error {+ if state.stopping {+ return errJobSchedulerStoppedInternal+ }+ state.stopping = true+ registrations := make([]watcherRegistrationInternal, 0, len(state.runners))+ for name, runner := range state.runners {+ registrations = append(registrations, watcherRegistrationInternal{watcher: state.allWatchers[name], runner: runner})+ }+ shutdown = schedulerShutdownInternal{cronDone: js.cron.Stop().Done(), watchers: registrations}+ return nil+ })+ if errors.Is(err, errJobSchedulerStoppedInternal) {+ return js.state.Stop(ctx)+ }+ var stopErr error+ if err != nil {+ if snapshot, ok := js.state.Load(); ok && snapshot.stopping {+ return nil+ }+ stopErr = err+ shutdown.cronDone = js.cron.Stop().Done()+ if snapshot, ok := js.state.Load(); ok {+ shutdown.watchers = make([]watcherRegistrationInternal, 0, len(snapshot.runners))+ for name, runner := range snapshot.runners {+ shutdown.watchers = append(shutdown.watchers, watcherRegistrationInternal{watcher: snapshot.allWatchers[name], runner: runner})+ }+ }}+ for _, registration := range shutdown.watchers {+ stopErr = errors.Combine(stopErr, registration.runner.Stop(ctx))+ if stopper, ok := registration.watcher.(schedulertypes.StoppableBusWatcher); ok {+ stopErr = errors.Combine(stopErr, stopper.Stop(ctx))+ }+ }select {- case <-watchersDone:- return nil+ case <-shutdown.cronDone:case <-ctx.Done():- return ctx.Err()+ stopErr = errors.Combine(stopErr, ctx.Err())}+ return errors.Combine(stopErr, js.state.Stop(ctx))}-// upsertJobInternal records the job and (re)schedules it. A replacement expression-// is parsed before the previous entry is removed so invalid input leaves the last-// valid entry untouched. Callers must hold js.mu.-func (js *JobScheduler) upsertJobInternal(ctx context.Context, job schedulertypes.Job) error {+func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *schedulerStateInternal, job schedulertypes.Job) error {+ if job == nil {+ return errors.New("scheduler job unavailable")+ }jobName := job.Name()- previousSchedule := js.schedules[jobName]- previousEntryID, hadPreviousEntry := js.entryIDs[jobName]+ previousSchedule := state.schedules[jobName]+ previousEntryID, hadPreviousEntry := state.entryIDs[jobName]schedule := job.Schedule(ctx)shouldSchedule := true@@ -255,33 +342,26 @@ func (js *JobScheduler) upsertJobInternal(ctx context.Context, job schedulertypeif hadPreviousEntry {js.cron.Remove(previousEntryID)- delete(js.entryIDs, jobName)+ delete(state.entryIDs, jobName)}if shouldSchedule {entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule)- js.entryIDs[jobName] = entryID+ state.entryIDs[jobName] = entryID}-- js.jobsByID[jobName] = job- js.schedules[jobName] = schedule+ state.jobsByID[jobName] = job+ state.schedules[jobName] = scheduleif previousSchedule == "" && shouldSchedule {slog.InfoContext(ctx, "Starting Job", "name", jobName, "schedule", schedule)} else if previousSchedule != schedule || hadPreviousEntry != shouldSchedule {slog.InfoContext(ctx, "Job rescheduled", "name", jobName, "previousSchedule", previousSchedule, "newSchedule", schedule, "nextRun", nextRun)}-slog.DebugContext(ctx, "Job scheduled", "name", jobName, "scheduled", shouldSchedule, "contextCanceled", ctx.Err() != nil)return nil}-// addCronEntryInternal adds a job closure that always runs with the scheduler's-// lifecycle context. Callers must hold js.mu.-func (js *JobScheduler) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {+func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {entryID := js.cron.Schedule(parsedSchedule, cron.FuncJob(func() {- // robfig/cron has no default panic recovery, so a panic in any job would- // otherwise take down the whole process. Recovering here covers every- // scheduled job in one place.defer utils.RecoverToError(nil, "scheduled job", "name", job.Name(), "schedule", schedule)slog.InfoContext(js.context, "Job starting", "name", job.Name(), "schedule", schedule)job.Run(js.context)@@ -296,6 +376,5 @@ func (js *JobScheduler) addCronEntryInternal(job schedulertypes.Job, schedule stif nextRun.IsZero() {return entryID, nil}-return entryID, new(nextRun)}diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.goindex 51dff1105..e5716e65c 100644--- a/backend/pkg/scheduler/scheduler_test.go+++ b/backend/pkg/scheduler/scheduler_test.go@@ -6,6 +6,7 @@ import ("testing""time"+ "emperror.dev/errors"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/stretchr/testify/require")@@ -38,6 +39,36 @@ type testBusWatcherInternal struct {ranNow chan context.Context}+type orderedStopBusWatcherInternal struct {+ name string+ started chan struct{}+ runnerExited chan struct{}+}++func (w *orderedStopBusWatcherInternal) Name() string {+ return w.name+}++func (w *orderedStopBusWatcherInternal) Start(ctx context.Context) error {+ close(w.started)+ <-ctx.Done()+ close(w.runnerExited)+ return nil+}++func (*orderedStopBusWatcherInternal) RunNow(context.Context) error {+ return nil+}++func (w *orderedStopBusWatcherInternal) Stop(context.Context) error {+ select {+ case <-w.runnerExited:+ return nil+ default:+ return errors.New("watcher stopped before its runner exited")+ }+}+func (w *testBusWatcherInternal) RunNow(ctx context.Context) error {if w.ranNow != nil {w.ranNow <- ctx@@ -63,7 +94,7 @@ func (j *conditionalTestSchedulerJob) ShouldSchedule(ctx context.Context) bool {}func TestJobScheduler_StartScheduler_SkipsDisabledConditionalJobs(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)job := &conditionalTestSchedulerJob{testSchedulerJob: &testSchedulerJob{@@ -73,17 +104,37 @@ func TestJobScheduler_StartScheduler_SkipsDisabledConditionalJobs(t *testing.T)shouldSchedule: func(context.Context) bool { return false },}- js.RegisterJob(job)- js.StartScheduler()+ require.NoError(t, js.RegisterJob(job))+ require.NoError(t, js.StartScheduler())defer js.cron.Stop()- require.NotContains(t, js.entryIDs, job.Name())+ state, ok := js.GetJobRuntimeState(job.Name())+ require.True(t, ok)+ require.False(t, state.Scheduled)require.Empty(t, js.cron.Entries())}+func TestJobScheduler_StartScheduler_ContinuesAfterInvalidJobSchedule(t *testing.T) {+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)+ invalid := &testSchedulerJob{name: "invalid-startup-job", schedule: "not a cron schedule"}+ valid := &testSchedulerJob{name: "valid-startup-job", schedule: "*/1 * * * * *"}++ require.NoError(t, js.RegisterJob(invalid))+ require.NoError(t, js.RegisterJob(valid))+ require.NoError(t, js.StartScheduler())++ invalidState, ok := js.GetJobRuntimeState(invalid.Name())+ require.True(t, ok)+ require.False(t, invalidState.Scheduled)+ validState, ok := js.GetJobRuntimeState(valid.Name())+ require.True(t, ok)+ require.True(t, validState.Scheduled)+ require.Len(t, js.cron.Entries(), 1)+}+func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {ctx, cancel := context.WithCancel(context.Background())- js := NewJobScheduler(ctx, nil)+ js := newJobSchedulerForTestInternal(t, ctx, nil)watcher := &testBusWatcherInternal{name: "test-bus-watcher",started: make(chan struct{}),@@ -91,8 +142,8 @@ func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {ranNow: make(chan context.Context, 1),}- js.RegisterBusWatcher(watcher, true)- js.StartScheduler()+ require.NoError(t, js.RegisterBusWatcher(watcher, true))+ require.NoError(t, js.StartScheduler())select {case <-watcher.started:@@ -117,10 +168,27 @@ func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {}}+func TestJobScheduler_StopJoinsRunnerBeforeStoppingWatcher(t *testing.T) {+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)+ watcher := &orderedStopBusWatcherInternal{+ name: "ordered-stop-watcher",+ started: make(chan struct{}),+ runnerExited: make(chan struct{}),+ }++ require.NoError(t, js.RegisterBusWatcher(watcher, false))+ select {+ case <-watcher.started:+ case <-time.After(time.Second):+ t.Fatal("timed out waiting for bus watcher to start")+ }+ require.NoError(t, js.Stop(t.Context()))+}+func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {ctx := t.Context()- js := NewJobScheduler(ctx, nil)+ js := newJobSchedulerForTestInternal(t, ctx, nil)manualWatcher := &testBusWatcherInternal{name: "manual-bus-watcher",started: make(chan struct{}),@@ -133,8 +201,8 @@ func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {stopped: make(chan struct{}),}- js.RegisterBusWatcher(manualWatcher, true)- js.RegisterBusWatcher(automaticOnlyWatcher, false)+ require.NoError(t, js.RegisterBusWatcher(manualWatcher, true))+ require.NoError(t, js.RegisterBusWatcher(automaticOnlyWatcher, false))runCtx := context.Background()require.NoError(t, js.RunBusWatcherNow(runCtx, manualWatcher.Name()))@@ -143,7 +211,7 @@ func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {}func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)enabled := truejob := &conditionalTestSchedulerJob{@@ -155,17 +223,21 @@ func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {}require.NoError(t, js.RescheduleJob(context.Background(), job))- require.Contains(t, js.entryIDs, job.Name())+ state, ok := js.GetJobRuntimeState(job.Name())+ require.True(t, ok)+ require.True(t, state.Scheduled)enabled = falserequire.NoError(t, js.RescheduleJob(context.Background(), job))- require.NotContains(t, js.entryIDs, job.Name())+ state, ok = js.GetJobRuntimeState(job.Name())+ require.True(t, ok)+ require.False(t, state.Scheduled)require.Empty(t, js.cron.Entries())}func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)enabled := falsejob := &conditionalTestSchedulerJob{@@ -177,33 +249,39 @@ func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {}require.NoError(t, js.RescheduleJob(context.Background(), job))- require.NotContains(t, js.entryIDs, job.Name())+ state, ok := js.GetJobRuntimeState(job.Name())+ require.True(t, ok)+ require.False(t, state.Scheduled)enabled = truerequire.NoError(t, js.RescheduleJob(context.Background(), job))- require.Contains(t, js.entryIDs, job.Name())+ state, ok = js.GetJobRuntimeState(job.Name())+ require.True(t, ok)+ require.True(t, state.Scheduled)require.Len(t, js.cron.Entries(), 1)}func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)job := &testSchedulerJob{name: "test-non-conditional-startup",schedule: "*/1 * * * * *",}- js.RegisterJob(job)- js.StartScheduler()+ require.NoError(t, js.RegisterJob(job))+ require.NoError(t, js.StartScheduler())defer js.cron.Stop()- require.Contains(t, js.entryIDs, job.Name())+ state, ok := js.GetJobRuntimeState(job.Name())+ require.True(t, ok)+ require.True(t, state.Scheduled)require.Len(t, js.cron.Entries(), 1)}func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)var once sync.OncerunErrCh := make(chan error, 1)@@ -231,7 +309,7 @@ func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T) {lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())- js := NewJobScheduler(lifecycleCtx, nil)+ js := newJobSchedulerForTestInternal(t, lifecycleCtx, nil)startedCh := make(chan struct{}, 1)stoppedCh := make(chan struct{}, 1)@@ -271,22 +349,22 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T}func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)job := &testSchedulerJob{name: "dyn-upsert", schedule: "*/5 * * * * *"}require.NoError(t, js.AddJob(context.Background(), job))require.True(t, js.HasJob(job.Name()))require.Len(t, js.cron.Entries(), 1)- firstEntry := js.entryIDs[job.Name()]+ firstEntry := js.cron.Entries()[0].ID// Re-adding with a changed schedule (e.g. a new sync interval) must replace the// existing cron entry, not leak a second one that keeps firing forever.job.schedule = "*/10 * * * * *"require.NoError(t, js.AddJob(context.Background(), job))require.Len(t, js.cron.Entries(), 1)- require.NotEqual(t, firstEntry, js.entryIDs[job.Name()])+ require.NotEqual(t, firstEntry, js.cron.Entries()[0].ID)- state, ok := js.GetJobRuntimeState(job.Name()).Get()+ state, ok := js.GetJobRuntimeState(job.Name())require.True(t, ok)require.True(t, state.Scheduled)require.Equal(t, "*/10 * * * * *", state.Schedule)@@ -294,28 +372,28 @@ func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {}func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)job := &testSchedulerJob{name: "dyn-invalid-reschedule", schedule: "*/5 * * * * *"}require.NoError(t, js.AddJob(context.Background(), job))require.True(t, js.HasJob(job.Name()))require.Len(t, js.cron.Entries(), 1)- firstEntry := js.entryIDs[job.Name()]+ firstEntry := js.cron.Entries()[0].IDjob.schedule = "not a cron schedule"require.Error(t, js.AddJob(context.Background(), job))require.True(t, js.HasJob(job.Name()))- require.Equal(t, firstEntry, js.entryIDs[job.Name()])+ require.Equal(t, firstEntry, js.cron.Entries()[0].ID)require.Len(t, js.cron.Entries(), 1)- state, ok := js.GetJobRuntimeState(job.Name()).Get()+ state, ok := js.GetJobRuntimeState(job.Name())require.True(t, ok)require.True(t, state.Scheduled)require.Equal(t, "*/5 * * * * *", state.Schedule)}func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)// Removing an unknown job must be a safe no-op (e.g. deleting a sync that never// had auto-sync enabled).@@ -328,12 +406,11 @@ func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {js.RemoveJob(context.Background(), job.Name())require.False(t, js.HasJob(job.Name()))- require.NotContains(t, js.entryIDs, job.Name())require.Empty(t, js.cron.Entries())}func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {- js := NewJobScheduler(context.Background(), nil)+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)job := &schedulertypes.GenericJob{JobName: "generic-dyn",@@ -353,8 +430,8 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {jobFinished := make(chan struct{}, 1)var runOnce sync.Once- js := NewJobScheduler(lifecycleCtx, nil)- js.RegisterJob(&testSchedulerJob{+ js := newJobSchedulerForTestInternal(t, lifecycleCtx, nil)+ require.NoError(t, js.RegisterJob(&testSchedulerJob{name: "test-shutdown-waits",schedule: "*/1 * * * * *",run: func(ctx context.Context) {@@ -366,8 +443,8 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {jobFinished <- struct{}{}})},- })- js.StartScheduler()+ }))+ require.NoError(t, js.StartScheduler())t.Cleanup(func() {cancelLifecycle()select {diff --git a/backend/pkg/utils/waitgroup.go b/backend/pkg/utils/waitgroup.gonew file mode 100644index 000000000..29ce376e9--- /dev/null+++ b/backend/pkg/utils/waitgroup.go@@ -0,0 +1,27 @@+package utils++import (+ "context"+ "sync"++ "emperror.dev/errors"+)++// WaitGroup waits for all work or returns when ctx ends. After a timeout, its+// waiter goroutine remains until the group eventually completes.+func WaitGroup(ctx context.Context, group *sync.WaitGroup) error {+ if ctx == nil || group == nil {+ return errors.New("wait group context unavailable")+ }+ done := make(chan struct{})+ go func() {+ group.Wait()+ close(done)+ }()+ select {+ case <-done:+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+}diff --git a/cli/go.mod b/cli/go.modindex fb473bf07..be2c8963a 100644--- a/cli/go.mod+++ b/cli/go.mod@@ -58,7 +58,7 @@ require (github.com/mattn/go-shellwords v1.0.13 // indirectgithub.com/moby/docker-image-spec v1.3.1 // indirectgithub.com/moby/moby/api v1.55.0 // indirect- github.com/moby/moby/client v0.5.0 // indirect+ github.com/moby/moby/client v0.5.1 // indirectgithub.com/muesli/cancelreader v0.2.2 // indirectgithub.com/muesli/mango v0.2.0 // indirectgithub.com/muesli/mango-cobra v1.3.0 // indirect@@ -92,7 +92,7 @@ require (go.uber.org/multierr v1.11.0 // indirectgo.yaml.in/yaml/v3 v3.0.4 // indirectgo.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect- golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect+ golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirectgolang.org/x/sync v0.22.0 // indirectgolang.org/x/sys v0.47.0 // indirectgolang.org/x/text v0.40.0 // indirectdiff --git a/cli/go.sum b/cli/go.sumindex 5d79d7ee2..582eb5aa2 100644--- a/cli/go.sum+++ b/cli/go.sum@@ -109,8 +109,8 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Ngithub.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=-github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=-github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=+github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw=+github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM=github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=github.com/muesli/mango v0.2.0 h1:iNNc0c5VLQ6fsMgAqGQofByNUBH2Q2nEbD6TaI+5yyQ=@@ -208,8 +208,8 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=-golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q=-golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw=+golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A=+golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=diff --git a/go.work.sum b/go.work.sumindex b370c7937..0ede2f355 100644--- a/go.work.sum+++ b/go.work.sum@@ -517,6 +517,8 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g=github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls=+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc=@@ -544,6 +546,8 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHgithub.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.9 h1:FjwMGxhQ6uY31o+leQBWtvZu8z0vfkDsuG7IpaI6goo=github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.9/go.mod h1:lTA9XMO2us+m6TK8wzbL7cOYmdPLnSV1T5aRz6Sv4rE=+github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.5 h1:7ZFdtE1XEHH+58GZU4Mbhq6SO/UbColleDApOtlv3vo=+github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.5/go.mod h1:M/qt7xBBXqilBwNmrO1yiu0cywZwQx5aqtKpdJN9J2A=github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 h1:IdCLsiiIj5YJ3AFevsewURCPV+YWUlOW8JiPhoAy8vg=github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4/go.mod h1:l4bdfCD7XyyZA9BolKBo1eLqgaJxl0/x91PL4Yqe0ao=github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=@@ -582,6 +586,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254=github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU=+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo=+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=@@ -595,6 +601,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEgithub.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc=github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw=+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0=+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po=github.com/aws/aws-sdk-go-v2/service/kms v1.52.0 h1:QNtg+Mtj1zmepk568+UKBD5DFfqh+ESTUUqQT27JkQc=github.com/aws/aws-sdk-go-v2/service/kms v1.52.0/go.mod h1:Y0+uxvxz6ib4KktRdK0V4X45Vcs/JyYoz8H71pO8xeI=github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw=@@ -603,6 +611,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3 h1:JRseEu/vIDMaWis4bSw0QbXL+cvIGc1XnX076H5ZXLE=github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk=+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s=github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=github.com/aws/aws-sdk-go-v2/service/signin v1.1.5/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ=@@ -699,6 +709,8 @@ github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8 h1:jP7ki8Tzx9github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY=github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM=github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc=+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo=github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=@@ -778,8 +790,6 @@ github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKgithub.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g=github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=-github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=-github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=github.com/compose-spec/compose-go/v2 v2.9.1/go.mod h1:Oky9AZGTRB4E+0VbTPZTUu4Kp+oEMMuwZXZtPPVT1iE=@@ -982,6 +992,7 @@ github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/Xgithub.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=github.com/fernet/fernet-go v0.0.0-20211208181803-9f70042a33ee h1:v6Eju/FhxsACGNipFEPBZZAzGr1F/jlRQr1qiBw2nEE=github.com/fernet/fernet-go v0.0.0-20211208181803-9f70042a33ee/go.mod h1:2H9hjfbpSMHwY503FclkV/lZTBh2YlOmLLSda12uL8c=github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e h1:Ss/B3/5wWRh8+emnK0++g5zQzwDTi30W10pKxKc4JXI=@@ -1138,6 +1149,8 @@ github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xgithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=+github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg=+github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ=github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=@@ -1258,6 +1271,8 @@ github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=+github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=+github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs=github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=@@ -1531,6 +1546,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jgithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=@@ -1550,6 +1567,8 @@ github.com/microsoft/go-mssqldb v1.9.8 h1:d4IFMvF/o+HdpXUqbBfzHvn/NlFA75YGcfHUUvgithub.com/microsoft/go-mssqldb v1.9.8/go.mod h1:eGSRSGAW4hKMy5YcAenhCDjIRm2rhqIdmmwgciMzLus=github.com/microsoft/go-mssqldb v1.10.0 h1:pHEt+Qz6YFPWqREq10mqSE524QQo+/QremwTCQht7TY=github.com/microsoft/go-mssqldb v1.10.0/go.mod h1:mnG7lGa9iYJbzJqGCXyuQCegStKMr3kogDLD6+bmggg=+github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=+github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=@@ -1610,6 +1629,8 @@ github.com/moby/sys/mount v0.3.4 h1:yn5jq4STPztkkzSKpZkLcmjue+bZJ0u2AuQY1iNI1Ww=github.com/moby/sys/mount v0.3.4/go.mod h1:KcQJMbQdJHPlq5lcYT+/CjatWM4PuxKe+XLSVS4J6Os=github.com/moby/sys/mount v0.3.5-0.20260529155943-fc52b7222d0b h1:RB/940R3dFkU1fLLk5rBIvcv/C3HfbjqVkStfi/wlb4=github.com/moby/sys/mount v0.3.5-0.20260529155943-fc52b7222d0b/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M=+github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00=+github.com/moby/sys/mount v0.3.5/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M=github.com/moby/sys/reexec v0.1.0 h1:RrBi8e0EBTLEgfruBOFcxtElzRGTEUkeIFaVXgU7wok=github.com/moby/sys/reexec v0.1.0/go.mod h1:EqjBg8F3X7iZe5pU6nRZnYCMUTXoxsjiIfHup5wYIN8=github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=@@ -2082,10 +2103,10 @@ github.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0github.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U=github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=+github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g=+github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=-github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=github.com/zmap/zcrypto v0.0.0-20210511125630-18f1e0152cfc h1:zkGwegkOW709y0oiAraH/3D8njopUR/pARHv4tZZ6pw=@@ -2103,6 +2124,8 @@ go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVigo.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=+go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=+go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=go.etcd.io/etcd/api/v3 v3.6.8 h1:gqb1VN92TAI6G2FiBvWcqKtHiIjr4SU2GdXxTwyexbM=go.etcd.io/etcd/api/v3 v3.6.8/go.mod h1:qyQj1HZPUV3B5cbAL8scG62+fyz5dSxxu0w8pn28N6Q=go.etcd.io/etcd/client/pkg/v3 v3.5.6 h1:TXQWYceBKqLp4sa87rcPs11SXxUA/mHwH975v+BDvLU=@@ -2189,6 +2212,8 @@ go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDgo.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk=go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18BqlOuz2PWEvs8yM7Q6XE=go.opentelemetry.io/otel/exporters/prometheus v0.65.0/go.mod h1:i1P8pcumauPtUI4YNopea1dhzEMuEqWP1xoUZDylLHo=+go.opentelemetry.io/otel/exporters/prometheus v0.66.0 h1:vkrK8PAznv2NKt2r+kdu252ccGzkEqLc2aSXbQIALYQ=+go.opentelemetry.io/otel/exporters/prometheus v0.66.0/go.mod h1:V/UB6D3vMF/UBOL5igAsAYnk1nG/bzYYTzvsB16cy7o=go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8=go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk=@@ -2283,8 +2308,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6ILgolang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=-golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A=-golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=@@ -2546,6 +2569,7 @@ golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=@@ -2835,6 +2859,8 @@ sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k=software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=+storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI=+storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4=tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=tags.cncf.io/container-device-interface/specs-go v1.0.0 h1:8gLw29hH1ZQP9K1YtAzpvkHCjjyIxHZYzBAvlQ+0vD8=tags.cncf.io/container-device-interface/specs-go v1.0.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=diff --git a/types/scheduler/job.go b/types/scheduler/job.goindex 79e8ad30a..80872402c 100644--- a/types/scheduler/job.go+++ b/types/scheduler/job.go@@ -32,6 +32,41 @@ type BusWatcher interface {RunNow(ctx context.Context) error}+// StoppableBusWatcher lets a continuous watcher perform explicit shutdown work+// before its actor runner is joined.+type StoppableBusWatcher interface {+ BusWatcher+ Stop(ctx context.Context) error+}++// DynamicScheduler owns jobs registered and removed while the application is running.+type DynamicScheduler interface {+ AddJob(ctx context.Context, job Job) error+ RemoveJob(ctx context.Context, name string)+ HasJob(name string) bool+}++// JobController exposes scheduler operations used by job-management services.+type JobController interface {+ GetJob(jobID string) (Job, bool)+ GetJobRuntimeState(jobID string) (JobRuntimeState, bool)+ RescheduleJob(ctx context.Context, job Job) error+ RunBusWatcherNow(ctx context.Context, watcherID string) error+}++// JobScheduler is the shared public contract for the actor-owned scheduler.+// Concrete cron and actor state remains private to the backend implementation.+type JobScheduler interface {+ DynamicScheduler+ JobController++ RegisterJob(job Job) error+ RegisterBusWatcher(watcher BusWatcher, canRunManually bool) error+ StartScheduler() error+ GetLocation() *time.Location+ Stop(ctx context.Context) error+}+// GenericJob is a reusable Job built from closures. It lets a service register a// per-entity dynamic job (e.g. one per GitOps sync or one per environment) without// importing the scheduler package: the service constructs a GenericJob and hands it