Skip to content

Modern Docker Management, Designed for Everyone

refactor: move automation logic to use actors (#3458)

Kyle Mendell3bdb4e67ff48
diff --git a/Justfile b/Justfile
index 086a56934..250e3d6e0 100644
--- a/Justfile
+++ b/Justfile
@@ -316,21 +316,25 @@ _lint-js:
@just _lint-tests
@just _lint-email-templates
-# Lint Go backend
+# Build golangci-lint with the custom linters enabled by the shared config
[group('quality')]
-_lint-backend:
+_build-golangci-lint:
golangci-lint custom
+
+# Lint Go backend
+[group('quality')]
+_lint-backend: _build-golangci-lint
cd backend && ../.bin/golangci-lint-custom run -c ../.github/.golangci.yml ./...
# Lint Go CLI
[group('quality')]
-_lint-cli:
- cd cli && golangci-lint run -c ../.github/.golangci.yml ./...
+_lint-cli: _build-golangci-lint
+ cd cli && ../.bin/golangci-lint-custom run -c ../.github/.golangci.yml ./...
# Lint Types
[group('quality')]
-_lint-types:
- cd types && golangci-lint run -c ../.github/.golangci.yml ./...
+_lint-types: _build-golangci-lint
+ cd types && ../.bin/golangci-lint-custom run -c ../.github/.golangci.yml ./...
# Lint edge tunnel protobuf definitions.
[group('quality')]
diff --git a/backend/api/handlers/activities_test.go b/backend/api/handlers/activities_test.go
index 93665c272..8111fb1ab 100644
--- a/backend/api/handlers/activities_test.go
+++ b/backend/api/handlers/activities_test.go
@@ -83,7 +83,7 @@ func TestActivityHandlerClearHistoryDeletesSelectedEnvironmentOnlyInternal(t *te
func TestActivityHandlerClearHistoryProxiesRemoteEnvironmentInternal(t *testing.T) {
ctx := context.Background()
db := setupActivityHandlerTestDBInternal(t)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
token := "remote-token"
@@ -190,7 +190,7 @@ func TestActivityHandlerStreamAllEmitsEnvironmentScopedEventsInternal(t *testing
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
activityService := services.NewActivityService(db, settingsService)
@@ -236,7 +236,7 @@ func TestActivityHandlerStreamAllReusesRemoteEnvironmentAfterInitialPollInternal
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
activityService := services.NewActivityService(db, settingsService)
@@ -302,7 +302,7 @@ func TestActivityHandlerStreamAllRemoteFailureEmitsErrorAndKeepsStreamingInterna
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
activityService := services.NewActivityService(db, settingsService)
@@ -339,7 +339,7 @@ func TestActivityHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *test
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
activityService := services.NewActivityService(db, settingsService)
_, err = activityService.StartActivity(ctx, services.StartActivityRequest{EnvironmentID: "0", Type: models.ActivityTypeResourceAction})
diff --git a/backend/api/handlers/dashboard_test.go b/backend/api/handlers/dashboard_test.go
index 9aca9e1b8..6212340d7 100644
--- a/backend/api/handlers/dashboard_test.go
+++ b/backend/api/handlers/dashboard_test.go
@@ -34,7 +34,7 @@ func setupDashboardHandlerTestDB(t *testing.T) (*database.DB, *services.Settings
require.NoError(t, db.AutoMigrate(&models.ApiKey{}, &models.Environment{}, &models.ImageUpdateRecord{}, &models.Project{}, &models.SettingVariable{}))
databaseDB := &database.DB{DB: db}
- settingsSvc, err := services.NewSettingsService(context.Background(), databaseDB)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), databaseDB)
require.NoError(t, err)
return databaseDB, settingsSvc
@@ -187,7 +187,7 @@ func TestDashboardHandlerStreamAllEmitsRemoteSnapshotInternal(t *testing.T) {
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
token := "remote-token"
@@ -231,7 +231,7 @@ func TestDashboardHandlerStreamAllLegacyAgentComposesSnapshotFromGranularEndpoin
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
token := "remote-token"
@@ -283,7 +283,7 @@ func TestDashboardHandlerStreamAllLegacyAgent404EmitsIncompatibleErrorInternal(t
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
token := "remote-token"
@@ -326,7 +326,7 @@ func TestDashboardHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *tes
db := setupActivityHandlerTestDBInternal(t)
limitStreamTestDBToSingleConnInternal(t, db)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
allowedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -370,7 +370,7 @@ func TestDashboardHandlerStreamAllFiltersUnauthorizedEnvironmentsInternal(t *tes
func TestDashboardHandlerRemoteFetchReusesLoadedEnvironmentInternal(t *testing.T) {
ctx := context.Background()
db := setupActivityHandlerTestDBInternal(t)
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
diff --git a/backend/api/handlers/remenv_handlers_test.go b/backend/api/handlers/remenv_handlers_test.go
index a218db91c..526d13d57 100644
--- a/backend/api/handlers/remenv_handlers_test.go
+++ b/backend/api/handlers/remenv_handlers_test.go
@@ -141,7 +141,7 @@ func TestSettingsHandler_GetSettings_RemoteFiltersNonAdminVisibility(t *testing.
defer server.Close()
db := setupActivityHandlerTestDBInternal(t)
- settingsService, err := services.NewSettingsService(context.Background(), db)
+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), db)
require.NoError(t, err)
handler := &SettingsHandler{
settingsService: settingsService,
@@ -167,7 +167,7 @@ func TestSettingsHandler_GetSettings_RemotePreservesAdminResponse(t *testing.T)
defer server.Close()
db := setupActivityHandlerTestDBInternal(t)
- settingsService, err := services.NewSettingsService(context.Background(), db)
+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), db)
require.NoError(t, err)
handler := &SettingsHandler{
settingsService: settingsService,
diff --git a/backend/api/handlers/settings_actor_test.go b/backend/api/handlers/settings_actor_test.go
new file mode 100644
index 000000000..a06452800
--- /dev/null
+++ b/backend/api/handlers/settings_actor_test.go
@@ -0,0 +1,32 @@
+package handlers
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*services.SettingsService, error) {
+ t.Helper()
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
+ require.NoError(t, err)
+ effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, executor.Stop(stopCtx))
+ require.NoError(t, effects.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+ })
+ return services.NewSettingsService(ctx, db, executor, effects)
+}
diff --git a/backend/api/handlers/settings_test.go b/backend/api/handlers/settings_test.go
index c749b03c1..c4b4ca100 100644
--- a/backend/api/handlers/settings_test.go
+++ b/backend/api/handlers/settings_test.go
@@ -12,7 +12,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
- "github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
apitypes "github.com/getarcaneapp/arcane/types/v2/settings"
"github.com/libtnb/sqlite"
@@ -76,7 +75,7 @@ func TestSettingsHandler_UpdateLocalEnvironment_RejectsUnreadableProjectsDirecto
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))
- settingsSvc, err := services.NewSettingsService(ctx, &database.DB{DB: db})
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, &database.DB{DB: db})
require.NoError(t, err)
originalDir := settingsSvc.GetSettingsConfig().ProjectsDirectory.Value
diff --git a/backend/api/handlers/variables_test.go b/backend/api/handlers/variables_test.go
index 996da4165..e9e3aaa4a 100644
--- a/backend/api/handlers/variables_test.go
+++ b/backend/api/handlers/variables_test.go
@@ -123,7 +123,7 @@ func setupVariableHandlerServicesInternal(t *testing.T) (*services.VariableServi
&models.SettingVariable{},
))
databaseDB := &database.DB{DB: databaseConnection}
- settingsService, err := services.NewSettingsService(context.Background(), databaseDB)
+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), databaseDB)
require.NoError(t, err)
require.NoError(t, settingsService.UpdateSetting(context.Background(), "projectsDirectory", t.TempDir()))
environmentService := services.NewEnvironmentService(databaseDB, nil, nil, nil, settingsService, nil)
diff --git a/backend/go.mod b/backend/go.mod
index 9dd1464a6..661102656 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -10,6 +10,7 @@ replace (
require (
emperror.dev/emperror v0.33.0
emperror.dev/errors v0.8.1
+ github.com/anthdm/hollywood v1.0.5
github.com/aws/aws-sdk-go-v2/config v1.32.31
github.com/aws/aws-sdk-go-v2/credentials v1.19.30
github.com/aws/aws-sdk-go-v2/service/ecr v1.60.0
@@ -81,6 +82,7 @@ require (
connectrpc.com/connect v1.19.2 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
+ github.com/DataDog/gostackparse v0.7.0 // indirect
github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679 // indirect
github.com/DmitriyVTitov/size v1.5.0 // indirect
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
@@ -218,6 +220,7 @@ require (
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
+ github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.69.0 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 4263599cf..f615a547b 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -11,6 +11,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=
+github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679 h1:qNT7R4qrN+5u5ajSbqSW1opHP4LA8lzA+ASyw5MQZjs=
github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=
@@ -30,6 +32,8 @@ github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9
github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/anthdm/hollywood v1.0.5 h1:SuCTVRRFqx0MZ4E0RijHl4+Xt56PF0C7aqYKguBU6j8=
+github.com/anthdm/hollywood v1.0.5/go.mod h1:wU4WxIRVs++E2PuiVXc8dA2An/Wlom4AhzwQ7e3tDzI=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
@@ -549,6 +553,10 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zU
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.getarcane.app/builds v0.3.0 h1:6fCgHy6PaEr3LpCtiF1iCdT4nuV3D3xvUHnV+E/0cqk=
go.getarcane.app/builds v0.3.0/go.mod h1:yZJDSh9XX/sgZlwx7lghHEucgZeQ74Gr/Fx+XdR2bdk=
go.getarcane.app/docker/convert v0.1.0 h1:prlIOs6dMx6CdnetsJFZ6ak0vcistM3DcH5lrZiFHBU=
diff --git a/backend/internal/actors/actor.go b/backend/internal/actors/actor.go
new file mode 100644
index 000000000..22cc40ef8
--- /dev/null
+++ b/backend/internal/actors/actor.go
@@ -0,0 +1,171 @@
+package actors
+
+import (
+ "context"
+ "sync"
+
+ "emperror.dev/errors"
+ "github.com/anthdm/hollywood/actor"
+)
+
+var errActorRequestNoResponseInternal = errors.Sentinel("actor request completed without response")
+
+// Context exposes the operations available while handling one actor message
+// without leaking Hollywood into actor consumers.
+type Context struct {
+ context context.Context
+ engine *actor.Engine
+ target *actor.PID
+ respond func(any)
+ workers *sync.WaitGroup
+}
+
+// Context returns the current receiver lifetime.
+func (c *Context) Context() context.Context {
+ return c.context
+}
+
+// Respond sends exactly one response to the current request.
+func (c *Context) Respond(message any) {
+ c.respond(message)
+}
+
+// Behavior defines the application behavior hosted by one shared actor.
+type Behavior struct {
+ Initialize func(*Context)
+ Handle func(*Context, any)
+ Cleanup func(*Context)
+}
+
+// Binding connects an ingress to an actor for the duration of its lifetime.
+type Binding interface {
+ bindInternal(handle *handleInternal)
+ unbindInternal(handle *handleInternal)
+ replayInternal()
+}
+
+// Actor owns one Hollywood process, its ingress bindings, and all workers
+// started from its behavior.
+type Actor struct {
+ handle *handleInternal
+ bindings []Binding
+ unbindOnce sync.Once
+}
+
+type behaviorActorInternal struct {
+ behavior Behavior
+ bindings []Binding
+ lifetime lifetimeInternal
+ workers sync.WaitGroup
+}
+
+// NewActor starts one application behavior on the shared actor runtime.
+func NewActor(ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, producer func() Behavior, bindings ...Binding) (*Actor, error) {
+ if ctx == nil {
+ return nil, errors.New("actor context unavailable")
+ }
+ if runtime == nil {
+ return nil, errors.New("actor runtime unavailable")
+ }
+ if producer == nil {
+ return nil, errors.New("actor behavior unavailable")
+ }
+
+ handle, err := runtime.startInternal(
+ func() actor.Receiver {
+ return &behaviorActorInternal{behavior: producer(), bindings: bindings}
+ },
+ kind,
+ id,
+ maxRestarts,
+ actor.WithContext(ctx),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ applicationActor := &Actor{handle: handle, bindings: bindings}
+ for _, binding := range bindings {
+ binding.bindInternal(handle)
+ }
+ return applicationActor, nil
+}
+
+// Send queues a message for the actor.
+func (a *Actor) Send(message any) error {
+ return a.handle.Send(message)
+}
+
+// Done closes after the actor and its workers stop.
+func (a *Actor) Done() <-chan struct{} {
+ return a.handle.Done()
+}
+
+// Stop disconnects ingress, drains the mailbox, cancels workers, and joins
+// them before the actor terminates.
+func (a *Actor) Stop(ctx context.Context) error {
+ a.unbindOnce.Do(func() {
+ for _, binding := range a.bindings {
+ binding.unbindInternal(a.handle)
+ }
+ })
+ return a.handle.Stop(ctx)
+}
+
+// Request sends one typed request and consumes its response.
+func Request[K comparable, Q, R any](ctx context.Context, a *Actor, request Message[K, Q]) (Message[K, R], error) {
+ if a == nil {
+ var zero Message[K, R]
+ return zero, errors.New("actor unavailable")
+ }
+ return requestHandleInternal[K, Q, R](ctx, a.handle, request)
+}
+
+func (a *behaviorActorInternal) Receive(ctx *actor.Context) {
+ switch message := ctx.Message().(type) {
+ case actor.Initialized:
+ return
+ case actor.Started:
+ lifetimeCtx := a.lifetime.startInternal(ctx.Context())
+ for _, binding := range a.bindings {
+ binding.replayInternal()
+ }
+ if a.behavior.Initialize != nil {
+ a.behavior.Initialize(a.contextInternal(lifetimeCtx, ctx))
+ }
+ case actor.Stopped:
+ a.lifetime.stopInternal()
+ if a.behavior.Cleanup != nil {
+ a.behavior.Cleanup(a.contextInternal(ctx.Context(), ctx))
+ }
+ // Strict joining preserves actor serialization across receiver restarts:
+ // a replacement must never run while an old worker can still own state.
+ // Behavior workers are therefore required to honor their lifetime context.
+ a.workers.Wait()
+ default:
+ if request, ok := message.(requestInternal); ok {
+ defer request.reply.Resolve(requestResultInternal{err: errActorRequestNoResponseInternal})
+ if a.behavior.Handle != nil {
+ requestCtx := a.contextInternal(a.lifetime.contextInternal(), ctx)
+ requestCtx.respond = func(message any) {
+ request.reply.Resolve(requestResultInternal{message: message})
+ }
+ a.behavior.Handle(requestCtx, request.message)
+ }
+ return
+ }
+ if a.behavior.Handle != nil {
+ a.behavior.Handle(a.contextInternal(a.lifetime.contextInternal(), ctx), message)
+ }
+ }
+}
+
+func (a *behaviorActorInternal) contextInternal(lifetimeCtx context.Context, ctx *actor.Context) *Context {
+ return &Context{
+ context: lifetimeCtx,
+ engine: ctx.Engine(),
+ target: ctx.PID().CloneVT(),
+ respond: ctx.Respond,
+ workers: &a.workers,
+ }
+}
diff --git a/backend/internal/actors/actor_test.go b/backend/internal/actors/actor_test.go
new file mode 100644
index 000000000..0738b5bd5
--- /dev/null
+++ b/backend/internal/actors/actor_test.go
@@ -0,0 +1,71 @@
+package actors
+
+import (
+ "context"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+type actorBindingFakeInternal struct {
+ binds atomic.Int32
+ unbinds atomic.Int32
+}
+
+func (b *actorBindingFakeInternal) bindInternal(*handleInternal) {
+ b.binds.Add(1)
+}
+
+func (b *actorBindingFakeInternal) unbindInternal(*handleInternal) {
+ b.unbinds.Add(1)
+}
+
+func (b *actorBindingFakeInternal) replayInternal() {}
+
+func TestActorOwnsLifecycleBindingsAndWorkerJoinInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ binding := &actorBindingFakeInternal{}
+ initialized := make(chan struct{})
+ workerExited := make(chan struct{})
+ applicationActor, err := NewActor(
+ t.Context(),
+ runtime,
+ "actor-test",
+ "lifecycle",
+ 1,
+ func() Behavior {
+ return Behavior{
+ Initialize: func(ctx *Context) {
+ close(initialized)
+ Worker[uint8, NoPayload]{
+ Actor: ctx,
+ WorkContext: ctx.Context(),
+ Label: "actor lifecycle test worker",
+ CompletionKind: 1,
+ }.Start(func(workerCtx context.Context) (NoPayload, error) {
+ defer close(workerExited)
+ <-workerCtx.Done()
+ return NoPayload{}, workerCtx.Err()
+ })
+ },
+ }
+ },
+ binding,
+ )
+ require.NoError(t, err)
+ require.Equal(t, int32(1), binding.binds.Load())
+ <-initialized
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, applicationActor.Stop(stopCtx))
+ <-workerExited
+ require.Equal(t, int32(1), binding.unbinds.Load())
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/executor.go b/backend/internal/actors/executor.go
new file mode 100644
index 000000000..6f1c7e443
--- /dev/null
+++ b/backend/internal/actors/executor.go
@@ -0,0 +1,179 @@
+package actors
+
+import (
+ "context"
+ "sync"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+)
+
+// Executor serializes heterogeneous blocking tasks through one actor mailbox.
+// A task must not synchronously submit another task to the same Executor.
+type Executor struct {
+ actor *Actor
+}
+
+// Task is one submitted executor operation whose result may be awaited with a
+// context independent from the operation's own lifetime.
+type Task[T any] struct {
+ result *Promise[executorResultInternal[T]]
+ actor *Actor
+}
+
+type executorTaskInternal interface {
+ executeInternal(actorCtx context.Context)
+}
+
+type executorResultInternal[T any] struct {
+ value T
+ err error
+}
+
+type executorTaskValueInternal[T any] struct {
+ ctx context.Context
+ label string
+ work func(context.Context) (T, error)
+ after func(T, error)
+ result *Promise[executorResultInternal[T]]
+ started *sync.Once
+}
+
+// NewExecutor creates a serial task executor on the shared actor runtime.
+func NewExecutor(ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32) (*Executor, error) {
+ if runtime == nil {
+ return nil, errors.New("actor runtime unavailable")
+ }
+ if ctx == nil {
+ return nil, errors.New("executor context unavailable")
+ }
+ applicationActor, err := NewActor(
+ ctx,
+ runtime,
+ kind,
+ id,
+ maxRestarts,
+ func() Behavior {
+ return Behavior{Handle: receiveExecutorInternal}
+ },
+ )
+ if err != nil {
+ return nil, err
+ }
+ return &Executor{actor: applicationActor}, nil
+}
+
+// Execute queues work, waits for its typed result, and runs after only once the
+// result is available to the caller. Work and after are both panic-contained.
+func Execute[T any](ctx context.Context, executor *Executor, label string, work func(context.Context) (T, error), after func(T, error)) (T, error) {
+ var zero T
+ task, err := Submit(ctx, executor, label, work, after)
+ if err != nil {
+ return zero, err
+ }
+ return task.Wait(ctx)
+}
+
+// Submit queues work synchronously and returns a handle that can be awaited
+// separately. This is useful for terminal cleanup that must stay queued even
+// when the shutdown wait context expires.
+func Submit[T any](ctx context.Context, executor *Executor, label string, work func(context.Context) (T, error), after func(T, error)) (*Task[T], error) {
+ if executor == nil || executor.actor == nil {
+ return nil, errors.New("actor executor unavailable")
+ }
+ if ctx == nil {
+ return nil, errors.New("task context unavailable")
+ }
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if work == nil {
+ return nil, errors.New("actor task unavailable")
+ }
+
+ result := NewPromise[executorResultInternal[T]]()
+ if err := executor.actor.Send(executorTaskValueInternal[T]{
+ ctx: ctx,
+ label: label,
+ work: work,
+ after: after,
+ result: result,
+ started: &sync.Once{},
+ }); err != nil {
+ return nil, err
+ }
+ return &Task[T]{result: result, actor: executor.actor}, nil
+}
+
+// Wait waits for the submitted task result without changing the task lifetime.
+func (t *Task[T]) Wait(ctx context.Context) (T, error) {
+ var zero T
+ if t == nil || t.result == nil || t.actor == nil {
+ return zero, errors.New("actor task unavailable")
+ }
+ if ctx == nil {
+ return zero, errors.New("task wait context unavailable")
+ }
+ completed, err := awaitInternal(ctx, t.actor.handle, t.result, errors.New("actor executor stopped"))
+ if err != nil {
+ return zero, err
+ }
+ return completed.value, completed.err
+}
+
+// Stop drains the executor mailbox and waits for the actor to terminate.
+func (e *Executor) Stop(ctx context.Context) error {
+ if e == nil || e.actor == nil {
+ return nil
+ }
+ return e.actor.Stop(ctx)
+}
+
+// Done closes after the executor actor terminates.
+func (e *Executor) Done() <-chan struct{} {
+ if e == nil || e.actor == nil {
+ return nil
+ }
+ return e.actor.Done()
+}
+
+func receiveExecutorInternal(ctx *Context, rawMessage any) {
+ task, ok := rawMessage.(executorTaskInternal)
+ if !ok {
+ return
+ }
+ task.executeInternal(ctx.Context())
+}
+
+func (t executorTaskValueInternal[T]) executeInternal(actorCtx context.Context) {
+ if t.result == nil || t.started == nil {
+ return
+ }
+ t.started.Do(func() {
+ workCtx, cancel := context.WithCancel(actorCtx)
+ stopTaskCancel := context.AfterFunc(t.ctx, cancel) //nolint:contextcheck // task execution combines caller cancellation with the actor lifetime.
+ defer cancel()
+ defer stopTaskCancel()
+
+ var (
+ value T
+ err error
+ )
+ func() {
+ defer utils.RecoverToError(&err, t.label)
+ if err = t.ctx.Err(); err == nil {
+ value, err = t.work(workCtx)
+ }
+ }()
+ t.result.Resolve(executorResultInternal[T]{value: value, err: err})
+
+ if t.after == nil {
+ return
+ }
+ func() {
+ var afterErr error
+ defer utils.RecoverToError(&afterErr, t.label+" completion")
+ t.after(value, err)
+ }()
+ })
+}
diff --git a/backend/internal/actors/executor_test.go b/backend/internal/actors/executor_test.go
new file mode 100644
index 000000000..10cd1a2e9
--- /dev/null
+++ b/backend/internal/actors/executor_test.go
@@ -0,0 +1,109 @@
+package actors
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+func TestExecutorSerializesHeterogeneousTasksInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ executor, err := NewExecutor(t.Context(), runtime, "executor-test", "serial", 1)
+ require.NoError(t, err)
+
+ firstStarted := make(chan struct{})
+ releaseFirst := make(chan struct{})
+ firstResult := make(chan error, 1)
+ go func() {
+ _, executeErr := Execute(t.Context(), executor, "first", func(context.Context) (int, error) {
+ close(firstStarted)
+ <-releaseFirst
+ return 1, nil
+ }, nil)
+ firstResult <- executeErr
+ }()
+ <-firstStarted
+
+ secondStarted := make(chan struct{})
+ secondResult := make(chan string, 1)
+ go func() {
+ value, executeErr := Execute(t.Context(), executor, "second", func(context.Context) (string, error) {
+ close(secondStarted)
+ return "two", nil
+ }, nil)
+ if executeErr != nil {
+ secondResult <- executeErr.Error()
+ return
+ }
+ secondResult <- value
+ }()
+
+ require.Never(t, func() bool { return len(secondStarted) > 0 }, 20*time.Millisecond, time.Millisecond)
+ close(releaseFirst)
+ require.NoError(t, <-firstResult)
+ require.Equal(t, "two", <-secondResult)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, executor.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestExecutorPublishesResultBeforeCompletionCallbackInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ executor, err := NewExecutor(t.Context(), runtime, "executor-test", "completion-order", 1)
+ require.NoError(t, err)
+
+ afterStarted := make(chan struct{})
+ releaseAfter := make(chan struct{})
+ result := make(chan int, 1)
+ go func() {
+ value, _ := Execute(t.Context(), executor, "ordered", func(context.Context) (int, error) {
+ return 42, nil
+ }, func(int, error) {
+ close(afterStarted)
+ <-releaseAfter
+ })
+ result <- value
+ }()
+
+ <-afterStarted
+ require.Equal(t, 42, <-result)
+ close(releaseAfter)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, executor.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestExecutorContainsTaskPanicAndContinuesInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ executor, err := NewExecutor(t.Context(), runtime, "executor-test", "panic", 1)
+ require.NoError(t, err)
+
+ _, err = Execute(t.Context(), executor, "panic task", func(context.Context) (NoPayload, error) {
+ panic("deliberate executor panic")
+ }, nil)
+ require.ErrorContains(t, err, "panic task panicked")
+
+ value, err := Execute(t.Context(), executor, "following task", func(context.Context) (int, error) {
+ return 7, nil
+ }, nil)
+ require.NoError(t, err)
+ require.Equal(t, 7, value)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, executor.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/gate.go b/backend/internal/actors/gate.go
new file mode 100644
index 000000000..50015a7cc
--- /dev/null
+++ b/backend/internal/actors/gate.go
@@ -0,0 +1,161 @@
+package actors
+
+import (
+ "context"
+ "sync"
+
+ "emperror.dev/errors"
+)
+
+type gateOperationInternal uint8
+
+const (
+ gateAcquireInternal gateOperationInternal = iota
+ gateReleaseInternal
+)
+
+// AdmissionKey namespaces a string identifier shared by the application gate.
+type AdmissionKey struct {
+ Scope string
+ ID string
+}
+
+// Gate admits at most one active lease for each key through one actor-owned map.
+type Gate[K comparable] struct {
+ actor *Actor
+}
+
+// Lease represents one admitted key. Release is idempotent and confirms that
+// the actor processed the completion before returning.
+type Lease[K comparable] struct {
+ gate *Gate[K]
+ key K
+ once sync.Once
+}
+
+type gateCommandInternal[K comparable] struct {
+ key K
+ reply *Promise[bool]
+}
+
+type gateActorInternal[K comparable] struct {
+ active map[K]struct{}
+}
+
+// NewGate creates a keyed admission gate on the shared actor runtime.
+func NewGate[K comparable](ctx context.Context, runtime *Runtime, kind, id string) (*Gate[K], error) {
+ if ctx == nil {
+ return nil, errors.New("gate context unavailable")
+ }
+ if runtime == nil {
+ return nil, errors.New("actor runtime unavailable")
+ }
+
+ active := make(map[K]struct{})
+ applicationActor, err := NewActor(
+ ctx,
+ runtime,
+ kind,
+ id,
+ // Shared gates tolerate transient receiver failures without forgetting
+ // active leases, whose map intentionally survives receiver replacement.
+ 3,
+ func() Behavior {
+ // The map intentionally survives receiver replacement so an actor restart
+ // cannot forget leases held by work that is still running.
+ state := &gateActorInternal[K]{active: active}
+ return Behavior{Handle: state.receiveInternal}
+ },
+ )
+ if err != nil {
+ return nil, err
+ }
+ return &Gate[K]{actor: applicationActor}, nil
+}
+
+// TryAcquire returns a lease when key is idle and refuses immediately when it
+// is already active. Actor failure is reported separately from contention.
+func (g *Gate[K]) TryAcquire(ctx context.Context, key K) (*Lease[K], bool, error) {
+ if ctx == nil {
+ return nil, false, errors.New("gate request context unavailable")
+ }
+ if err := ctx.Err(); err != nil {
+ return nil, false, err
+ }
+ if g == nil || g.actor == nil {
+ return nil, false, errors.New("actor gate unavailable")
+ }
+
+ reply := NewPromise[bool]()
+ if err := g.actor.Send(Message[gateOperationInternal, gateCommandInternal[K]]{
+ Kind: gateAcquireInternal,
+ Value: gateCommandInternal[K]{
+ key: key,
+ reply: reply,
+ },
+ }); err != nil {
+ return nil, false, err
+ }
+
+ // Admission must resolve after it is queued; abandoning an admitted reply on
+ // caller cancellation would leak the key without a lease to release it.
+ admitted, err := awaitInternal(context.WithoutCancel(ctx), g.actor.handle, reply, errors.New("actor gate stopped"))
+ if err != nil {
+ return nil, false, err
+ }
+ if !admitted {
+ return nil, false, nil
+ }
+ return &Lease[K]{gate: g, key: key}, true, nil
+}
+
+// Release returns the lease to its gate exactly once.
+func (l *Lease[K]) Release() {
+ if l == nil || l.gate == nil {
+ return
+ }
+ l.once.Do(func() {
+ reply := NewPromise[bool]()
+ if err := l.gate.actor.Send(Message[gateOperationInternal, gateCommandInternal[K]]{
+ Kind: gateReleaseInternal,
+ Value: gateCommandInternal[K]{
+ key: l.key,
+ reply: reply,
+ },
+ }); err != nil {
+ return
+ }
+ _, _ = awaitInternal(context.Background(), l.gate.actor.handle, reply, errors.New("actor gate stopped"))
+ })
+}
+
+// Stop drains the gate mailbox and waits for the actor to terminate.
+func (g *Gate[K]) Stop(ctx context.Context) error {
+ if g == nil || g.actor == nil {
+ return nil
+ }
+ return g.actor.Stop(ctx)
+}
+
+func (g *gateActorInternal[K]) receiveInternal(_ *Context, rawMessage any) {
+ message, ok := rawMessage.(Message[gateOperationInternal, gateCommandInternal[K]])
+ if !ok || message.Value.reply == nil {
+ return
+ }
+ // Resolve a refusal if future gate logic panics before producing a result.
+ // Promise resolution is idempotent, so successful paths win first.
+ defer message.Value.reply.Resolve(false)
+
+ switch message.Kind {
+ case gateAcquireInternal:
+ if _, active := g.active[message.Value.key]; active {
+ message.Value.reply.Resolve(false)
+ return
+ }
+ g.active[message.Value.key] = struct{}{}
+ message.Value.reply.Resolve(true)
+ case gateReleaseInternal:
+ delete(g.active, message.Value.key)
+ message.Value.reply.Resolve(true)
+ }
+}
diff --git a/backend/internal/actors/gate_test.go b/backend/internal/actors/gate_test.go
new file mode 100644
index 000000000..b15f214e8
--- /dev/null
+++ b/backend/internal/actors/gate_test.go
@@ -0,0 +1,67 @@
+package actors
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+func TestGateSerializesEachKeyAndReleasesIndependentlyInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ gate, err := NewGate[string](t.Context(), runtime, "gate-test", "keyed")
+ require.NoError(t, err)
+
+ first, admitted, err := gate.TryAcquire(t.Context(), "first")
+ require.NoError(t, err)
+ require.True(t, admitted)
+
+ _, admitted, err = gate.TryAcquire(t.Context(), "first")
+ require.NoError(t, err)
+ require.False(t, admitted)
+
+ second, admitted, err := gate.TryAcquire(t.Context(), "second")
+ require.NoError(t, err)
+ require.True(t, admitted)
+ second.Release()
+
+ first.Release()
+ first.Release()
+ replacement, admitted, err := gate.TryAcquire(t.Context(), "first")
+ require.NoError(t, err)
+ require.True(t, admitted)
+ replacement.Release()
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestGateReturnsContextCancellationWithoutLeakingKeyInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ gate, err := NewGate[string](t.Context(), runtime, "gate-test", "cancellation")
+ require.NoError(t, err)
+
+ canceled, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, admitted, err := gate.TryAcquire(canceled, "key")
+ require.ErrorIs(t, err, context.Canceled)
+ require.False(t, admitted)
+
+ lease, admitted, err := gate.TryAcquire(t.Context(), "key")
+ require.NoError(t, err)
+ require.True(t, admitted)
+ lease.Release()
+
+ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
+ defer stopCancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/message.go b/backend/internal/actors/message.go
new file mode 100644
index 000000000..3429b9948
--- /dev/null
+++ b/backend/internal/actors/message.go
@@ -0,0 +1,375 @@
+package actors
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+)
+
+// NoPayload is used by typed actor messages that only carry a kind.
+type NoPayload struct{}
+
+// Message is the shared typed envelope for actor commands, events, and replies.
+// Worker completion handlers must call Acknowledge before returning.
+type Message[K comparable, V any] struct {
+ Kind K
+ Value V
+ Err error
+ ack chan struct{}
+}
+
+// Acknowledge confirms that an actor processed a worker completion.
+func (m Message[K, V]) Acknowledge() {
+ if m.ack == nil {
+ return
+ }
+ select {
+ case m.ack <- struct{}{}:
+ default:
+ }
+}
+
+// Promise is an idempotent one-result completion primitive.
+type Promise[T any] struct {
+ once sync.Once
+ done chan T
+}
+
+// NewPromise creates a promise that accepts exactly one result.
+func NewPromise[T any]() *Promise[T] {
+ return &Promise[T]{done: make(chan T, 1)}
+}
+
+// Resolve publishes value once; later calls are harmless.
+func (p *Promise[T]) Resolve(value T) {
+ if p == nil {
+ return
+ }
+ p.once.Do(func() { p.done <- value })
+}
+
+// Done returns the channel that receives the promised result.
+func (p *Promise[T]) Done() <-chan T {
+ if p == nil {
+ return nil
+ }
+ return p.done
+}
+
+// awaitInternal waits for a promise while preserving a result that races with caller
+// cancellation or actor termination.
+func awaitInternal[T any](ctx context.Context, handle *handleInternal, promise *Promise[T], stoppedErr error) (T, error) {
+ var zero T
+ if ctx == nil || handle == nil || promise == nil {
+ return zero, errors.New("actor wait unavailable")
+ }
+ select {
+ case result := <-promise.Done():
+ return result, nil
+ case <-ctx.Done():
+ select {
+ case result := <-promise.Done():
+ return result, nil
+ default:
+ return zero, ctx.Err()
+ }
+ case <-handle.Done():
+ select {
+ case result := <-promise.Done():
+ return result, nil
+ default:
+ return zero, stoppedErr
+ }
+ }
+}
+
+// lifetimeInternal is an actor-receiver lifecycle context.
+type lifetimeInternal struct {
+ mu sync.Mutex
+ ctx context.Context
+ cancel context.CancelFunc
+}
+
+func (l *lifetimeInternal) startInternal(parent context.Context) context.Context {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if l.cancel != nil {
+ l.cancel()
+ }
+ l.ctx, l.cancel = context.WithCancel(parent)
+ return l.ctx
+}
+
+func (l *lifetimeInternal) contextInternal() context.Context {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return l.ctx
+}
+
+func (l *lifetimeInternal) stopInternal() {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if l.cancel != nil {
+ l.cancel()
+ l.cancel = nil
+ }
+}
+
+// Ingress coalesces concurrent producers into one pending mailbox signal while
+// retaining the newest value. Bind may happen after Send, which lets services
+// accept startup triggers before their actor exists.
+type Ingress[K comparable, V any] struct {
+ mu sync.Mutex
+ kind K
+ value V
+ generation uint64
+ acknowledged uint64
+ queued bool
+ handle *handleInternal
+}
+
+// NewIngress creates an unbound coalescing actor ingress.
+func NewIngress[K comparable, V any](kind K) *Ingress[K, V] {
+ return &Ingress[K, V]{kind: kind}
+}
+
+// Bind delivers future signals to handle and flushes one signal queued before startup.
+func (i *Ingress[K, V]) bindInternal(handle *handleInternal) {
+ if i == nil {
+ return
+ }
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ i.handle = handle
+ if i.queued && handle != nil {
+ _ = handle.Send(Message[K, NoPayload]{Kind: i.kind})
+ }
+}
+
+// Unbind stops delivery to handle without discarding a pending signal.
+func (i *Ingress[K, V]) unbindInternal(handle *handleInternal) {
+ if i == nil {
+ return
+ }
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ if i.handle == handle {
+ i.handle = nil
+ }
+}
+
+func (i *Ingress[K, V]) replayInternal() {
+ if i == nil {
+ return
+ }
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ if i.generation == i.acknowledged || i.queued || i.handle == nil {
+ return
+ }
+ i.queued = true
+ _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})
+}
+
+// Send records value and queues at most one mailbox signal.
+func (i *Ingress[K, V]) Send(value V) {
+ if i == nil {
+ return
+ }
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ i.value = value
+ i.generation++
+ if i.queued {
+ return
+ }
+ i.queued = true
+ if i.handle != nil {
+ _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})
+ }
+}
+
+// Begin returns the newest value and its generation while retaining the work
+// as pending until Acknowledge commits it.
+func (i *Ingress[K, V]) Begin() (V, uint64) {
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ i.queued = false
+ return i.value, i.generation
+}
+
+// Acknowledge commits work through generation and replays a newer pending value.
+func (i *Ingress[K, V]) Acknowledge(generation uint64) {
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ if generation > i.acknowledged {
+ i.acknowledged = generation
+ }
+ if i.generation == i.acknowledged || i.queued || i.handle == nil {
+ return
+ }
+ i.queued = true
+ _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})
+}
+
+// Take returns and immediately acknowledges the newest value.
+func (i *Ingress[K, V]) Take() V {
+ value, generation := i.Begin()
+ i.Acknowledge(generation)
+ return value
+}
+
+// Pending reports whether an unacknowledged value remains.
+func (i *Ingress[K, V]) Pending() bool {
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ return i.generation != i.acknowledged
+}
+
+// Latest returns the newest value without changing admission state.
+func (i *Ingress[K, V]) Latest() V {
+ i.mu.Lock()
+ defer i.mu.Unlock()
+ return i.value
+}
+
+// Timer sends generation-fenced messages to an actor. It is actor-owned and
+// must be stopped when the receiver handles actor.Stopped.
+type Timer[K comparable] struct {
+ timer *time.Timer
+ generation uint64
+}
+
+var timerGenerationInternal atomic.Uint64
+
+// Reset replaces the active timer and returns its generation.
+func (t *Timer[K]) Reset(ctx *Context, kind K, delay time.Duration) uint64 {
+ if t.timer != nil {
+ t.timer.Stop()
+ }
+ generation := timerGenerationInternal.Add(1)
+ t.generation = generation
+ engine := ctx.engine
+ target := ctx.target
+ actorCtx := ctx.Context()
+ t.timer = time.AfterFunc(delay, func() {
+ if actorCtx.Err() == nil {
+ engine.Send(target, Message[K, uint64]{Kind: kind, Value: generation})
+ }
+ })
+ return generation
+}
+
+// Current reports whether generation belongs to the latest Reset call.
+func (t *Timer[K]) Current(generation uint64) bool {
+ return generation == t.generation
+}
+
+// Stop cancels the timer if one is active.
+func (t *Timer[K]) Stop() {
+ t.generation = timerGenerationInternal.Add(1)
+ if t.timer != nil {
+ t.timer.Stop()
+ t.timer = nil
+ }
+}
+
+// requestHandleInternal sends a typed request through Arcane's promise envelope.
+type requestInternal struct {
+ message any
+ reply *Promise[requestResultInternal]
+}
+
+type requestResultInternal struct {
+ message any
+ err error
+}
+
+func requestHandleInternal[K comparable, Q, R any](ctx context.Context, handle *handleInternal, request Message[K, Q]) (Message[K, R], error) {
+ var zero Message[K, R]
+ if ctx == nil || handle == nil {
+ return zero, errors.New("actor request target unavailable")
+ }
+ reply := NewPromise[requestResultInternal]()
+ if err := handle.Send(requestInternal{message: request, reply: reply}); err != nil {
+ return zero, err
+ }
+ result, err := awaitInternal(ctx, handle, reply, errors.New("actor stopped"))
+ if err != nil {
+ return zero, err
+ }
+ if result.err != nil {
+ return zero, result.err
+ }
+ typed, ok := result.message.(Message[K, R])
+ if !ok {
+ return zero, fmt.Errorf("unexpected actor response type %T", result.message)
+ }
+ return typed, nil
+}
+
+// Worker describes blocking work owned by an actor but executed outside Receive.
+type Worker[K comparable, V any] struct {
+ Actor *Context
+ WorkContext context.Context
+ Label string
+ CompletionKind K
+ PanicValue V
+ RetryDelay time.Duration
+ ActorStopped func(V, error)
+}
+
+// Start contains panics, joins the work to WaitGroup, and posts one typed
+// completion back to the actor. Its completion handler must call Acknowledge.
+func (w Worker[K, V]) Start(work func(context.Context) (V, error)) {
+ actorCtx := w.Actor.Context()
+ engine := w.Actor.engine
+ target := w.Actor.target
+ workerCtx, cancel := context.WithCancel(w.WorkContext)
+ stopActorCancel := context.AfterFunc(actorCtx, cancel)
+ w.Actor.workers.Go(func() {
+ var (
+ value = w.PanicValue
+ err error
+ )
+ defer cancel()
+ defer stopActorCancel()
+ defer func() {
+ if actorCtx.Err() == nil {
+ ack := make(chan struct{}, 1)
+ engine.Send(target, Message[K, V]{Kind: w.CompletionKind, Value: value, Err: err, ack: ack})
+ select {
+ case <-ack:
+ return
+ case <-actorCtx.Done():
+ }
+ }
+ if w.ActorStopped != nil {
+ w.ActorStopped(value, err)
+ }
+ }()
+ for {
+ value = w.PanicValue
+ err = nil
+ func() {
+ defer utils.RecoverToError(&err, w.Label)
+ value, err = work(workerCtx)
+ }()
+ if err == nil || w.RetryDelay <= 0 || workerCtx.Err() != nil {
+ return
+ }
+ timer := time.NewTimer(w.RetryDelay)
+ select {
+ case <-workerCtx.Done():
+ timer.Stop()
+ return
+ case <-timer.C:
+ }
+ }
+ })
+}
diff --git a/backend/internal/actors/message_test.go b/backend/internal/actors/message_test.go
new file mode 100644
index 000000000..d930b1050
--- /dev/null
+++ b/backend/internal/actors/message_test.go
@@ -0,0 +1,415 @@
+package actors
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/anthdm/hollywood/actor"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+type messageKindInternal uint8
+
+const (
+ ingressMessageInternal messageKindInternal = iota
+ timerStartMessageInternal
+ timerElapsedMessageInternal
+ requestMessageInternal
+ responseMessageInternal
+ workerCompletedMessageInternal
+)
+
+type ingressReceiverInternal struct {
+ ingress *Ingress[messageKindInternal, int]
+ values chan<- int
+}
+
+func (r *ingressReceiverInternal) Receive(ctx *actor.Context) {
+ if message, ok := ctx.Message().(Message[messageKindInternal, NoPayload]); ok && message.Kind == ingressMessageInternal {
+ r.values <- r.ingress.Take()
+ }
+}
+
+func TestIngressCoalescesBeforeBindingAndRetainsLatestValueInternal(t *testing.T) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ require.NoError(t, err)
+ ingress := NewIngress[messageKindInternal, int](ingressMessageInternal)
+ values := make(chan int, 2)
+ pid := engine.Spawn(
+ func() actor.Receiver { return &ingressReceiverInternal{ingress: ingress, values: values} },
+ "ingress-test",
+ actor.WithID("coalesced"),
+ actor.WithMaxRestarts(1),
+ )
+
+ ingress.Send(1)
+ ingress.Send(2)
+ handle := &handleInternal{engine: engine, pid: pid, done: make(chan struct{})}
+ ingress.bindInternal(handle)
+ require.Equal(t, 2, <-values)
+
+ ingress.Send(3)
+ require.Equal(t, 3, <-values)
+ ingress.unbindInternal(handle)
+ <-engine.Poison(pid).Done()
+}
+
+type timerReceiverInternal struct {
+ timer Timer[messageKindInternal]
+ elapsed chan<- uint64
+}
+
+func (r *timerReceiverInternal) Receive(ctx *actor.Context) {
+ switch message := ctx.Message().(type) {
+ case Message[messageKindInternal, NoPayload]:
+ if message.Kind == timerStartMessageInternal {
+ actorCtx := &Context{
+ context: ctx.Context(),
+ engine: ctx.Engine(),
+ target: ctx.PID().CloneVT(),
+ respond: ctx.Respond,
+ workers: &sync.WaitGroup{},
+ }
+ r.timer.Reset(actorCtx, timerElapsedMessageInternal, time.Hour)
+ r.timer.Reset(actorCtx, timerElapsedMessageInternal, time.Millisecond)
+ }
+ case Message[messageKindInternal, uint64]:
+ if message.Kind == timerElapsedMessageInternal && r.timer.Current(message.Value) {
+ r.elapsed <- message.Value
+ }
+ case actor.Stopped:
+ r.timer.Stop()
+ }
+}
+
+func TestTimerOnlyDeliversLatestGenerationInternal(t *testing.T) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ require.NoError(t, err)
+ elapsed := make(chan uint64, 2)
+ pid := engine.Spawn(
+ func() actor.Receiver { return &timerReceiverInternal{elapsed: elapsed} },
+ "timer-test",
+ actor.WithID("latest"),
+ actor.WithMaxRestarts(1),
+ )
+ engine.Send(pid, Message[messageKindInternal, NoPayload]{Kind: timerStartMessageInternal})
+ require.Equal(t, uint64(2), <-elapsed)
+ require.Never(t, func() bool { return len(elapsed) > 0 }, 20*time.Millisecond, time.Millisecond)
+ <-engine.Poison(pid).Done()
+}
+
+func TestRequestReturnsTypedResponseInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ applicationActor, err := NewActor(
+ t.Context(),
+ runtime,
+ "request-test",
+ "typed",
+ 1,
+ func() Behavior {
+ return Behavior{Handle: func(ctx *Context, rawMessage any) {
+ if message, ok := rawMessage.(Message[messageKindInternal, int]); ok && message.Kind == requestMessageInternal {
+ ctx.Respond(Message[messageKindInternal, string]{Kind: responseMessageInternal, Value: "ok"})
+ }
+ }}
+ },
+ )
+ require.NoError(t, err)
+ for range 10 {
+ response, requestErr := Request[messageKindInternal, int, string](t.Context(), applicationActor, Message[messageKindInternal, int]{Kind: requestMessageInternal, Value: 1})
+ require.NoError(t, requestErr)
+ require.Equal(t, responseMessageInternal, response.Kind)
+ require.Equal(t, "ok", response.Value)
+ }
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, applicationActor.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestRequestFailsWhenHandlerReturnsWithoutResponseInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ applicationActor, err := NewActor(
+ t.Context(),
+ runtime,
+ "request-test",
+ "missing-response",
+ 1,
+ func() Behavior {
+ return Behavior{Handle: func(*Context, any) {}}
+ },
+ )
+ require.NoError(t, err)
+
+ _, err = Request[messageKindInternal, int, string](context.Background(), applicationActor, Message[messageKindInternal, int]{Kind: requestMessageInternal})
+ require.ErrorContains(t, err, "without response")
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, applicationActor.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+type workerReceiverInternal struct {
+ errors chan<- error
+ values chan<- string
+}
+
+func (r workerReceiverInternal) Receive(ctx *actor.Context) {
+ if message, ok := ctx.Message().(Message[messageKindInternal, string]); ok && message.Kind == workerCompletedMessageInternal {
+ defer message.Acknowledge()
+ if r.errors != nil {
+ r.errors <- message.Err
+ }
+ if r.values != nil {
+ r.values <- message.Value
+ }
+ }
+}
+
+func TestWorkerContainsPanicAndDeliversCompletionInternal(t *testing.T) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ require.NoError(t, err)
+ workerErrors := make(chan error, 1)
+ workerValues := make(chan string, 1)
+ pid := engine.Spawn(
+ func() actor.Receiver { return workerReceiverInternal{errors: workerErrors, values: workerValues} },
+ "worker-test",
+ actor.WithID("panic"),
+ actor.WithMaxRestarts(1),
+ )
+
+ var wg sync.WaitGroup
+ Worker[messageKindInternal, string]{
+ Actor: &Context{
+ context: context.Background(),
+ engine: engine,
+ target: pid,
+ workers: &wg,
+ },
+ WorkContext: context.Background(),
+ Label: "deliberate worker test",
+ CompletionKind: workerCompletedMessageInternal,
+ PanicValue: "panic fallback",
+ }.Start(func(context.Context) (string, error) {
+ panic("deliberate worker panic")
+ })
+
+ require.ErrorContains(t, <-workerErrors, "deliberate worker test panicked")
+ require.Equal(t, "panic fallback", <-workerValues)
+ wg.Wait()
+ <-engine.Poison(pid).Done()
+}
+
+func TestWorkerRetriesContainedPanicInternal(t *testing.T) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ require.NoError(t, err)
+ workerErrors := make(chan error, 1)
+ workerValues := make(chan string, 1)
+ pid := engine.Spawn(
+ func() actor.Receiver { return workerReceiverInternal{errors: workerErrors, values: workerValues} },
+ "worker-test",
+ actor.WithID("retry"),
+ actor.WithMaxRestarts(1),
+ )
+
+ var (
+ wg sync.WaitGroup
+ attempts int
+ )
+ Worker[messageKindInternal, string]{
+ Actor: &Context{
+ context: context.Background(),
+ engine: engine,
+ target: pid,
+ workers: &wg,
+ },
+ WorkContext: context.Background(),
+ Label: "retrying worker test",
+ CompletionKind: workerCompletedMessageInternal,
+ PanicValue: "panic fallback",
+ RetryDelay: time.Millisecond,
+ }.Start(func(context.Context) (string, error) {
+ attempts++
+ if attempts == 1 {
+ panic("deliberate retryable panic")
+ }
+ return "completed", nil
+ })
+
+ require.NoError(t, <-workerErrors)
+ require.Equal(t, "completed", <-workerValues)
+ require.Equal(t, 2, attempts)
+ wg.Wait()
+ <-engine.Poison(pid).Done()
+}
+
+type lifetimeReceiverInternal struct {
+ lifetime lifetimeInternal
+ started chan<- context.Context
+}
+
+func (r *lifetimeReceiverInternal) Receive(ctx *actor.Context) {
+ switch ctx.Message().(type) {
+ case actor.Started:
+ r.started <- r.lifetime.startInternal(ctx.Context())
+ case panicActorMessageInternal:
+ panic("deliberate lifetime restart")
+ case actor.Stopped:
+ r.lifetime.stopInternal()
+ }
+}
+
+func TestLifetimeCancelsOldReceiverBeforeHollywoodRestartInternal(t *testing.T) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ require.NoError(t, err)
+ started := make(chan context.Context, 2)
+ pid := engine.Spawn(
+ func() actor.Receiver { return &lifetimeReceiverInternal{started: started} },
+ "lifetime-test",
+ actor.WithID("restart"),
+ actor.WithMaxRestarts(1),
+ actor.WithRestartDelay(0),
+ )
+
+ first := <-started
+ engine.Send(pid, panicActorMessageInternal{})
+ require.Eventually(t, func() bool { return first.Err() != nil }, time.Second, time.Millisecond)
+ second := <-started
+ require.NoError(t, second.Err())
+ <-engine.Poison(pid).Done()
+}
+
+func TestWorkerResolvesFallbackWhenActorStopsBeforeCompletionInternal(t *testing.T) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ require.NoError(t, err)
+ pid := engine.Spawn(
+ func() actor.Receiver { return workerReceiverInternal{errors: make(chan error, 1)} },
+ "worker-test",
+ actor.WithID("stopped"),
+ actor.WithMaxRestarts(1),
+ )
+
+ actorCtx, stopActor := context.WithCancel(context.Background())
+ result := NewPromise[error]()
+ var wg sync.WaitGroup
+ Worker[messageKindInternal, string]{
+ Actor: &Context{
+ context: actorCtx,
+ engine: engine,
+ target: pid,
+ workers: &wg,
+ },
+ WorkContext: context.Background(),
+ Label: "stopped worker test",
+ CompletionKind: workerCompletedMessageInternal,
+ ActorStopped: func(_ string, err error) {
+ result.Resolve(err)
+ },
+ }.Start(func(ctx context.Context) (string, error) {
+ <-ctx.Done()
+ return "", ctx.Err()
+ })
+
+ stopActor()
+ require.ErrorIs(t, <-result.Done(), context.Canceled)
+ wg.Wait()
+ <-engine.Poison(pid).Done()
+}
+
+type noAcknowledgementReceiverInternal struct {
+ received chan<- struct{}
+}
+
+func (r noAcknowledgementReceiverInternal) Receive(ctx *actor.Context) {
+ if message, ok := ctx.Message().(Message[messageKindInternal, string]); ok && message.Kind == workerCompletedMessageInternal {
+ r.received <- struct{}{}
+ }
+}
+
+func TestWorkerWithoutAcknowledgementUnblocksWhenActorLifetimeEndsInternal(t *testing.T) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ require.NoError(t, err)
+ received := make(chan struct{}, 1)
+ pid := engine.Spawn(
+ func() actor.Receiver { return noAcknowledgementReceiverInternal{received: received} },
+ "worker-test",
+ actor.WithID("missing-ack"),
+ )
+ actorCtx, stopActor := context.WithCancel(context.Background())
+ fallback := make(chan error, 1)
+ var workers sync.WaitGroup
+ Worker[messageKindInternal, string]{
+ Actor: &Context{
+ context: actorCtx,
+ engine: engine,
+ target: pid,
+ workers: &workers,
+ },
+ WorkContext: actorCtx,
+ Label: "missing acknowledgement worker",
+ CompletionKind: workerCompletedMessageInternal,
+ ActorStopped: func(_ string, err error) {
+ fallback <- err
+ },
+ }.Start(func(context.Context) (string, error) {
+ return "done", nil
+ })
+
+ <-received
+ stopActor()
+ require.NoError(t, <-fallback)
+ require.NoError(t, utils.WaitGroup(t.Context(), &workers))
+ <-engine.Poison(pid).Done()
+}
+
+func TestIngressReplaysUnacknowledgedValueAfterActorRestartInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ ingress := NewIngress[messageKindInternal, int](ingressMessageInternal)
+ processed := make(chan int, 1)
+ var attempts atomic.Int32
+ applicationActor, err := NewActor(
+ t.Context(),
+ runtime,
+ "ingress-test",
+ "restart-replay",
+ 2,
+ func() Behavior {
+ return Behavior{Handle: func(_ *Context, rawMessage any) {
+ message, ok := rawMessage.(Message[messageKindInternal, NoPayload])
+ if !ok || message.Kind != ingressMessageInternal {
+ return
+ }
+ value, generation := ingress.Begin()
+ if attempts.Add(1) == 1 {
+ panic("restart after ingress begin")
+ }
+ ingress.Acknowledge(generation)
+ processed <- value
+ }}
+ },
+ ingress,
+ )
+ require.NoError(t, err)
+
+ ingress.Send(42)
+ require.Equal(t, 42, <-processed)
+ require.False(t, ingress.Pending())
+ require.Equal(t, int32(2), attempts.Load())
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, applicationActor.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/resource.go b/backend/internal/actors/resource.go
new file mode 100644
index 000000000..08f526751
--- /dev/null
+++ b/backend/internal/actors/resource.go
@@ -0,0 +1,128 @@
+package actors
+
+import (
+ "context"
+ "log/slog"
+ "sync/atomic"
+
+ "emperror.dev/errors"
+)
+
+// ErrResourceStopped reports work submitted after terminal resource shutdown.
+var ErrResourceStopped = errors.New("actor resource stopped")
+
+// Resource serializes the lifecycle and use of one replaceable value through
+// an actor executor. Its value is never accessed outside that mailbox, and its
+// zero value represents the absence of a resource.
+type Resource[T comparable] struct {
+ executor *Executor
+ value T
+ terminal atomic.Bool
+ stop func(T) error
+}
+
+// NewResource creates an actor-owned replaceable resource.
+func NewResource[T comparable](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, stop func(T) error) (*Resource[T], error) {
+ if stop == nil {
+ return nil, errors.New("resource stop function unavailable")
+ }
+ executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)
+ if err != nil {
+ return nil, err
+ }
+ return &Resource[T]{executor: executor, stop: stop}, nil
+}
+
+// Restart stops the current value before building and publishing its replacement.
+// A non-zero failed build is cleaned up.
+func (r *Resource[T]) Restart(ctx context.Context, label string, build func(context.Context) (T, error)) error {
+ if build == nil {
+ return errors.New("resource build function unavailable")
+ }
+ if r.terminal.Load() {
+ return ErrResourceStopped
+ }
+ _, err := Execute(ctx, r.executor, label, func(workCtx context.Context) (NoPayload, error) {
+ if r.terminal.Load() {
+ return NoPayload{}, ErrResourceStopped
+ }
+ stopErr := r.clearInternal()
+ next, buildErr := build(workCtx)
+ if buildErr != nil {
+ var zero T
+ if next != zero {
+ buildErr = errors.Combine(buildErr, r.stop(next))
+ }
+ return NoPayload{}, errors.Combine(stopErr, buildErr)
+ }
+ r.value = next
+ if stopErr != nil {
+ slog.WarnContext(workCtx, "failed to stop replaced actor resource", "label", label, "error", stopErr)
+ }
+ return NoPayload{}, nil
+ }, nil)
+ return err
+}
+
+// Do serializes work with Restart and Stop. Work is skipped after terminal stop.
+func (r *Resource[T]) Do(ctx context.Context, label string, work func(context.Context, T) error) error {
+ if work == nil {
+ return errors.New("resource work function unavailable")
+ }
+ if r.terminal.Load() {
+ return ErrResourceStopped
+ }
+ _, err := Execute(ctx, r.executor, label, func(workCtx context.Context) (NoPayload, error) {
+ if r.terminal.Load() {
+ return NoPayload{}, ErrResourceStopped
+ }
+ return NoPayload{}, work(workCtx, r.value)
+ }, nil)
+ return err
+}
+
+// Clear stops and forgets the current value while keeping the resource reusable.
+func (r *Resource[T]) Clear(ctx context.Context, label string) error {
+ if r.terminal.Load() {
+ return ErrResourceStopped
+ }
+ _, err := Execute(ctx, r.executor, label, func(context.Context) (NoPayload, error) {
+ if r.terminal.Load() {
+ return NoPayload{}, ErrResourceStopped
+ }
+ return NoPayload{}, r.clearInternal()
+ }, nil)
+ return err
+}
+
+// Stop permanently fences replacement, stops the current value, and joins the actor.
+func (r *Resource[T]) Stop(ctx context.Context) error {
+ if ctx == nil {
+ return errors.New("resource stop context unavailable")
+ }
+ r.terminal.Store(true)
+ task, submitErr := Submit(context.WithoutCancel(ctx), r.executor, "stop actor resource", func(context.Context) (NoPayload, error) {
+ return NoPayload{}, r.clearInternal()
+ }, nil)
+ var clearErr error
+ if submitErr == nil {
+ _, clearErr = task.Wait(ctx)
+ }
+ actorErr := r.executor.Stop(ctx)
+ if submitErr != nil {
+ // Submit only fails after the executor can no longer own the value, so
+ // terminal cleanup must stop it directly instead of leaking it.
+ clearErr = r.clearInternal()
+ }
+ return errors.Combine(submitErr, clearErr, actorErr)
+}
+
+func (r *Resource[T]) clearInternal() error {
+ var zero T
+ if r.value == zero {
+ return nil
+ }
+ current := r.value
+ r.value = zero
+ return r.stop(current)
+}
diff --git a/backend/internal/actors/resource_test.go b/backend/internal/actors/resource_test.go
new file mode 100644
index 000000000..081ba426e
--- /dev/null
+++ b/backend/internal/actors/resource_test.go
@@ -0,0 +1,99 @@
+package actors
+
+import (
+ "context"
+ "errors"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+func TestResourceSerializesRestartWorkAndStopInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ var (
+ mu sync.Mutex
+ stopped []int
+ )
+ resource, err := NewResource(t.Context(), runtime, "resource-test", "lifecycle", 1, func(value int) error {
+ mu.Lock()
+ stopped = append(stopped, value)
+ mu.Unlock()
+ return nil
+ })
+ require.NoError(t, err)
+ require.NoError(t, resource.Restart(t.Context(), "start first", func(context.Context) (int, error) { return 1, nil }))
+ require.NoError(t, resource.Restart(t.Context(), "start second", func(context.Context) (int, error) { return 2, nil }))
+
+ var current int
+ require.NoError(t, resource.Do(t.Context(), "read current", func(_ context.Context, value int) error {
+ current = value
+ return nil
+ }))
+ require.Equal(t, 2, current)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, resource.Stop(stopCtx))
+ mu.Lock()
+ require.Equal(t, []int{1, 2}, stopped)
+ mu.Unlock()
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestResourceFailedReplacementCleansPartialValueInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ var stopped []int
+ resource, err := NewResource(t.Context(), runtime, "resource-test", "failed-replacement", 1, func(value int) error {
+ stopped = append(stopped, value)
+ return nil
+ })
+ require.NoError(t, err)
+ require.NoError(t, resource.Restart(t.Context(), "start", func(context.Context) (int, error) { return 1, nil }))
+ require.Error(t, resource.Restart(t.Context(), "replace", func(context.Context) (int, error) {
+ return 2, errors.New("build failed")
+ }))
+ require.Equal(t, []int{1, 2}, stopped)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, resource.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestResourceCanceledStopStillCleansAndFencesInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ stopped := make(chan struct{})
+ resource, err := NewResource(t.Context(), runtime, "resource-test", "canceled-stop", 1, func(int) error {
+ close(stopped)
+ return nil
+ })
+ require.NoError(t, err)
+ require.NoError(t, resource.Restart(t.Context(), "start", func(context.Context) (int, error) { return 1, nil }))
+
+ canceled, cancel := context.WithCancel(context.Background())
+ cancel()
+ require.ErrorIs(t, resource.Stop(canceled), context.Canceled)
+ select {
+ case <-stopped:
+ case <-time.After(time.Second):
+ t.Fatal("resource cleanup was abandoned after stop wait cancellation")
+ }
+ require.ErrorIs(t, resource.Restart(t.Context(), "restart", func(context.Context) (int, error) { return 2, nil }), ErrResourceStopped)
+
+ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
+ defer stopCancel()
+ require.NoError(t, resource.executor.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/runner.go b/backend/internal/actors/runner.go
new file mode 100644
index 000000000..c454145ce
--- /dev/null
+++ b/backend/internal/actors/runner.go
@@ -0,0 +1,84 @@
+package actors
+
+import (
+ "context"
+
+ "emperror.dev/errors"
+)
+
+// Runner owns one long-lived background function and joins it before stopping.
+type Runner struct {
+ actor *Actor
+}
+
+type runnerStateInternal struct {
+ run func(context.Context) error
+ label string
+}
+
+type runnerEventInternal uint8
+
+const runnerExitedInternal runnerEventInternal = iota
+
+// NewRunner starts a long-lived function on the shared actor runtime.
+func NewRunner(ctx context.Context, runtime *Runtime, kind, id, label string, maxRestarts int32, run func(context.Context) error) (*Runner, error) {
+ if ctx == nil {
+ return nil, errors.New("runner context unavailable")
+ }
+ if runtime == nil {
+ return nil, errors.New("actor runtime unavailable")
+ }
+ if run == nil {
+ return nil, errors.New("runner function unavailable")
+ }
+
+ applicationActor, err := NewActor(
+ ctx,
+ runtime,
+ kind,
+ id,
+ maxRestarts,
+ func() Behavior {
+ state := &runnerStateInternal{run: run, label: label}
+ return Behavior{
+ Initialize: state.initializeInternal,
+ Handle: state.receiveInternal,
+ }
+ },
+ )
+ if err != nil {
+ return nil, err
+ }
+ return &Runner{actor: applicationActor}, nil
+}
+
+// Stop cancels the background function and waits for it and its actor to exit.
+func (r *Runner) Stop(ctx context.Context) error {
+ if r == nil || r.actor == nil {
+ return nil
+ }
+ return r.actor.Stop(ctx)
+}
+
+func (r *runnerStateInternal) initializeInternal(ctx *Context) {
+ Worker[runnerEventInternal, NoPayload]{
+ Actor: ctx,
+ WorkContext: ctx.Context(),
+ Label: r.label,
+ CompletionKind: runnerExitedInternal,
+ }.Start(func(workerCtx context.Context) (NoPayload, error) {
+ return NoPayload{}, r.run(workerCtx)
+ })
+}
+
+func (r *runnerStateInternal) receiveInternal(_ *Context, rawMessage any) {
+ message, ok := rawMessage.(Message[runnerEventInternal, NoPayload])
+ if !ok {
+ return
+ }
+ defer message.Acknowledge()
+ if message.Err != nil {
+ panic(message.Err)
+ }
+ panic(errors.New(r.label + " exited unexpectedly"))
+}
diff --git a/backend/internal/actors/runner_test.go b/backend/internal/actors/runner_test.go
new file mode 100644
index 000000000..9da4cab18
--- /dev/null
+++ b/backend/internal/actors/runner_test.go
@@ -0,0 +1,41 @@
+package actors
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+func TestRunnerCancelsAndJoinsBackgroundFunctionInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ started := make(chan struct{})
+ exited := make(chan struct{})
+ runner, err := NewRunner(t.Context(), runtime, "runner-test", "lifecycle", "test runner", 1, func(ctx context.Context) error {
+ close(started)
+ <-ctx.Done()
+ close(exited)
+ return nil
+ })
+ require.NoError(t, err)
+ select {
+ case <-started:
+ case <-time.After(time.Second):
+ t.Fatal("runner did not start")
+ }
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, runner.Stop(stopCtx))
+ select {
+ case <-exited:
+ default:
+ t.Fatal("runner stop returned before background function exited")
+ }
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/runtime.go b/backend/internal/actors/runtime.go
new file mode 100644
index 000000000..f8c9c5714
--- /dev/null
+++ b/backend/internal/actors/runtime.go
@@ -0,0 +1,290 @@
+// Package actors owns Arcane's in-process actor runtime and its shared safety rails.
+package actors
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "math"
+ "sync"
+ "sync/atomic"
+
+ "emperror.dev/errors"
+ "github.com/anthdm/hollywood/actor"
+ "go.uber.org/fx"
+)
+
+const actorMonitorID = "arcane-monitor"
+
+type monitorReadyInternal struct{}
+
+// Runtime owns the shared Hollywood engine and observes actor failures.
+type Runtime struct {
+ engine *actor.Engine
+ monitorPID *actor.PID
+ deadLetters atomic.Uint64
+ restarts atomic.Uint64
+ maxRestartsExceeded atomic.Uint64
+ duplicateIDs atomic.Uint64
+ policyMu sync.Mutex
+ policies map[string]actorPolicyInternal
+}
+
+type actorPolicyInternal struct {
+ maxRestarts int32
+ handle *handleInternal
+}
+
+// handleInternal identifies a spawned actor and reports when its process terminates.
+type handleInternal struct {
+ engine *actor.Engine
+ pid *actor.PID
+ done chan struct{}
+ doneOnce sync.Once
+ stopOnce sync.Once
+}
+
+// Done closes when the actor process permanently stops.
+func (h *handleInternal) Done() <-chan struct{} {
+ if h == nil {
+ return nil
+ }
+ return h.done
+}
+
+// Send queues a message for the actor while hiding Hollywood's engine and PID
+// from services that only need to communicate with their owned process.
+func (h *handleInternal) Send(message any) error {
+ if h == nil || h.engine == nil || h.pid == nil {
+ return errors.New("actor handle unavailable")
+ }
+ select {
+ case <-h.done:
+ return errors.New("actor stopped")
+ default:
+ h.engine.Send(h.pid, message)
+ return nil
+ }
+}
+
+// Stop drains the actor mailbox, initiates termination exactly once, and waits
+// for the process to exit. A canceled wait does not cancel the queued stop.
+func (h *handleInternal) Stop(ctx context.Context) error {
+ if h == nil || h.engine == nil || h.pid == nil {
+ return nil
+ }
+ if ctx == nil {
+ return errors.New("actor stop context unavailable")
+ }
+ h.stopOnce.Do(func() { h.engine.Poison(h.pid) })
+ select {
+ case <-h.done:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+func (h *handleInternal) stopInternal() {
+ if h != nil {
+ h.doneOnce.Do(func() { close(h.done) })
+ }
+}
+
+// NewRuntime constructs the shared actor engine and registers its lifecycle monitor.
+func NewRuntime(appCtx context.Context, lc fx.Lifecycle) (*Runtime, error) {
+ engine, err := actor.NewEngine(actor.NewEngineConfig())
+ if err != nil {
+ return nil, err
+ }
+
+ runtime := &Runtime{
+ engine: engine,
+ policies: make(map[string]actorPolicyInternal),
+ }
+ ready := make(chan struct{})
+ var readyOnce sync.Once
+ runtime.monitorPID = engine.Spawn(
+ func() actor.Receiver {
+ return &runtimeMonitorInternal{
+ runtime: runtime,
+ ready: ready,
+ readyOnce: &readyOnce,
+ }
+ },
+ "runtime-monitor",
+ actor.WithID(actorMonitorID),
+ actor.WithContext(appCtx),
+ actor.WithMaxRestarts(math.MaxInt32),
+ )
+ engine.Subscribe(runtime.monitorPID)
+ engine.BroadcastEvent(monitorReadyInternal{})
+
+ select {
+ case <-ready:
+ case <-appCtx.Done():
+ <-engine.Poison(runtime.monitorPID).Done()
+ return nil, appCtx.Err()
+ }
+
+ lc.Append(fx.Hook{
+ OnStop: func(ctx context.Context) error {
+ engine.Unsubscribe(runtime.monitorPID)
+ stopped := engine.PoisonCtx(ctx, runtime.monitorPID)
+ select {
+ case <-stopped.Done():
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ },
+ })
+
+ return runtime, nil
+}
+
+// startInternal starts a Hollywood receiver with a deterministic ID and an
+// Arcane-enforced restart limit.
+// Hollywood v1.0.5 panics while cleaning up its native max-restart path, so the
+// shared monitor enforces the same ceiling and stops the actor safely instead.
+func (r *Runtime) startInternal(producer actor.Producer, kind, id string, maxRestarts int32, opts ...actor.OptFunc) (*handleInternal, error) {
+ if r == nil || r.engine == nil {
+ return nil, errors.New("actor runtime unavailable")
+ }
+ if producer == nil {
+ return nil, errors.New("actor producer unavailable")
+ }
+
+ policyKey := actor.NewPID(r.engine.Address(), kind+"/"+id).String()
+ r.policyMu.Lock()
+ if _, exists := r.policies[policyKey]; exists || r.engine.Registry.GetPID(kind, id) != nil {
+ r.policyMu.Unlock()
+ return nil, fmt.Errorf("actor ID already in use: %s/%s", kind, id)
+ }
+ handle := &handleInternal{
+ engine: r.engine,
+ pid: actor.NewPID(r.engine.Address(), kind+"/"+id),
+ done: make(chan struct{}),
+ }
+ r.policies[policyKey] = actorPolicyInternal{maxRestarts: maxRestarts, handle: handle}
+ r.policyMu.Unlock()
+
+ options := append([]actor.OptFunc{}, opts...)
+ options = append(options, actor.WithID(id), actor.WithMaxRestarts(math.MaxInt32))
+ handle.pid = r.engine.Spawn(producer, kind, options...)
+ if handle.pid.String() != policyKey {
+ r.policyMu.Lock()
+ delete(r.policies, policyKey)
+ r.policyMu.Unlock()
+ r.engine.Stop(handle.pid)
+ return nil, fmt.Errorf("actor PID mismatch: got %s, expected %s", handle.pid.String(), policyKey)
+ }
+ return handle, nil
+}
+
+// DeadLetterCount returns the number of undeliverable actor messages observed.
+func (r *Runtime) DeadLetterCount() uint64 {
+ if r == nil {
+ return 0
+ }
+ return r.deadLetters.Load()
+}
+
+// RestartCount returns the number of actor restarts observed.
+func (r *Runtime) RestartCount() uint64 {
+ if r == nil {
+ return 0
+ }
+ return r.restarts.Load()
+}
+
+// MaxRestartsExceededCount returns the number of permanently stopped actors observed.
+func (r *Runtime) MaxRestartsExceededCount() uint64 {
+ if r == nil {
+ return 0
+ }
+ return r.maxRestartsExceeded.Load()
+}
+
+// DuplicateIDCount returns the number of duplicate actor registrations observed.
+func (r *Runtime) DuplicateIDCount() uint64 {
+ if r == nil {
+ return 0
+ }
+ return r.duplicateIDs.Load()
+}
+
+type runtimeMonitorInternal struct {
+ runtime *Runtime
+ ready chan struct{}
+ readyOnce *sync.Once
+}
+
+func (m *runtimeMonitorInternal) Receive(ctx *actor.Context) {
+ switch event := ctx.Message().(type) {
+ case actor.Started:
+ ctx.Engine().Subscribe(ctx.PID())
+
+ case monitorReadyInternal:
+ m.readyOnce.Do(func() { close(m.ready) })
+
+ case actor.ActorRestartedEvent:
+ if event.PID == nil {
+ slog.ErrorContext(ctx.Context(), "actor restarted event omitted PID")
+ return
+ }
+ m.runtime.policyMu.Lock()
+ policy, tracked := m.runtime.policies[event.PID.String()]
+ m.runtime.policyMu.Unlock()
+ if tracked && event.Restarts > policy.maxRestarts {
+ m.runtime.maxRestartsExceeded.Add(1)
+ slog.ErrorContext(ctx.Context(), "actor permanently stopped after exceeding restart limit", "pid", event.PID.String(), "reason", event.Reason, "restarts", event.Restarts, "maxRestarts", policy.maxRestarts, "stack", string(event.Stacktrace))
+ ctx.Engine().Stop(event.PID)
+ return
+ }
+ m.runtime.restarts.Add(1)
+ slog.ErrorContext(ctx.Context(), "actor crashed and restarted", "pid", event.PID.String(), "reason", event.Reason, "restarts", event.Restarts, "stack", string(event.Stacktrace))
+
+ case actor.ActorMaxRestartsExceededEvent:
+ if event.PID == nil {
+ slog.ErrorContext(ctx.Context(), "actor max-restarts event omitted PID")
+ return
+ }
+ m.runtime.maxRestartsExceeded.Add(1)
+ slog.ErrorContext(ctx.Context(), "actor permanently stopped after exceeding restart limit", "pid", event.PID.String())
+
+ case actor.ActorStoppedEvent:
+ if event.PID == nil {
+ slog.ErrorContext(ctx.Context(), "actor stopped event omitted PID")
+ return
+ }
+ m.runtime.policyMu.Lock()
+ policy, tracked := m.runtime.policies[event.PID.String()]
+ if tracked {
+ delete(m.runtime.policies, event.PID.String())
+ }
+ m.runtime.policyMu.Unlock()
+ if tracked {
+ policy.handle.stopInternal()
+ }
+
+ case actor.ActorDuplicateIdEvent:
+ m.runtime.duplicateIDs.Add(1)
+ pid := "<nil>"
+ if event.PID != nil {
+ pid = event.PID.String()
+ }
+ slog.ErrorContext(ctx.Context(), "duplicate actor ID rejected", "pid", pid)
+
+ case actor.DeadLetterEvent:
+ m.runtime.deadLetters.Add(1)
+ target := "<nil>"
+ if event.Target != nil {
+ target = event.Target.String()
+ }
+ slog.ErrorContext(ctx.Context(), "actor message sent to dead letter", "target", target, "messageType", fmt.Sprintf("%T", event.Message))
+
+ case actor.Stopped:
+ ctx.Engine().Unsubscribe(ctx.PID())
+ }
+}
diff --git a/backend/internal/actors/runtime_test.go b/backend/internal/actors/runtime_test.go
new file mode 100644
index 000000000..ec01dccae
--- /dev/null
+++ b/backend/internal/actors/runtime_test.go
@@ -0,0 +1,154 @@
+package actors
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/anthdm/hollywood/actor"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+type panicActorInternal struct{}
+
+func (panicActorInternal) Receive(ctx *actor.Context) {
+ if _, ok := ctx.Message().(panicActorMessageInternal); ok {
+ panic("deliberate actor test panic")
+ }
+}
+
+type panicActorMessageInternal struct{}
+
+type stopBlockingActorInternal struct {
+ release <-chan struct{}
+}
+
+func (a stopBlockingActorInternal) Receive(ctx *actor.Context) {
+ if _, ok := ctx.Message().(actor.Stopped); ok {
+ <-a.release
+ }
+}
+
+func TestRuntimeMonitorObservesRestartsMaxRestartsAndDeadLettersInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ handle, err := runtime.startInternal(
+ func() actor.Receiver { return panicActorInternal{} },
+ "panic-test",
+ "restart-limit",
+ 3,
+ actor.WithContext(t.Context()),
+ actor.WithRestartDelay(0),
+ )
+ require.NoError(t, err)
+ pid := handle.pid
+
+ for expectedRestarts := uint64(1); expectedRestarts <= 3; expectedRestarts++ {
+ require.NoError(t, handle.Send(panicActorMessageInternal{}))
+ require.Eventually(t, func() bool {
+ return runtime.RestartCount() == expectedRestarts
+ }, time.Second, time.Millisecond)
+ }
+
+ require.NoError(t, handle.Send(panicActorMessageInternal{}))
+ require.Eventually(t, func() bool {
+ return runtime.MaxRestartsExceededCount() == 1
+ }, time.Second, time.Millisecond)
+ require.Eventually(t, func() bool {
+ select {
+ case <-handle.Done():
+ return true
+ default:
+ return false
+ }
+ }, time.Second, time.Millisecond)
+
+ runtime.engine.Send(pid, panicActorMessageInternal{})
+ require.Eventually(t, func() bool {
+ return runtime.DeadLetterCount() == 1
+ }, time.Second, time.Millisecond)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestHandleCanceledStopStillTerminatesAndCanBeJoinedInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ release := make(chan struct{})
+ handle, err := runtime.startInternal(
+ func() actor.Receiver { return stopBlockingActorInternal{release: release} },
+ "handle-test",
+ "stop",
+ 1,
+ actor.WithContext(t.Context()),
+ )
+ require.NoError(t, err)
+ require.NotNil(t, handle)
+
+ canceled, cancel := context.WithCancel(context.Background())
+ cancel()
+ require.ErrorIs(t, handle.Stop(canceled), context.Canceled)
+ close(release)
+
+ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
+ defer stopCancel()
+ require.NoError(t, handle.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestRuntimeRejectsDuplicateActorIDWithoutOverwritingOwnerInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ first, err := runtime.startInternal(
+ func() actor.Receiver { return panicActorInternal{} },
+ "duplicate-test",
+ "same-id",
+ 1,
+ actor.WithContext(t.Context()),
+ )
+ require.NoError(t, err)
+
+ second, err := runtime.startInternal(
+ func() actor.Receiver { return panicActorInternal{} },
+ "duplicate-test",
+ "same-id",
+ 1,
+ actor.WithContext(t.Context()),
+ )
+ require.Nil(t, second)
+ require.ErrorContains(t, err, "actor ID already in use")
+ require.Equal(t, actor.NewPID(runtime.engine.Address(), "duplicate-test/same-id").String(), first.pid.String())
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, first.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestRuntimeMonitorResubscribesOnStartedInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+
+ runtime.engine.Unsubscribe(runtime.monitorPID)
+ time.Sleep(10 * time.Millisecond)
+ runtime.engine.Send(runtime.monitorPID, actor.Started{})
+ time.Sleep(10 * time.Millisecond)
+ runtime.engine.Send(actor.NewPID(runtime.engine.Address(), "missing/actor"), "dead letter")
+ require.Eventually(t, func() bool {
+ return runtime.DeadLetterCount() == 1
+ }, time.Second, time.Millisecond)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/signal.go b/backend/internal/actors/signal.go
new file mode 100644
index 000000000..2fc9c49b4
--- /dev/null
+++ b/backend/internal/actors/signal.go
@@ -0,0 +1,62 @@
+package actors
+
+import (
+ "sync"
+ "sync/atomic"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+)
+
+// Signal delivers each published value once to every current subscriber.
+// Subscribers run sequentially on the publishing goroutine.
+type Signal[T any] struct {
+ mu sync.RWMutex
+ nextID atomic.Uint64
+ subscribers map[uint64]func(T)
+}
+
+// NewSignal creates an empty typed signal.
+func NewSignal[T any]() *Signal[T] {
+ return &Signal[T]{subscribers: make(map[uint64]func(T))}
+}
+
+// Subscribe registers a callback and returns an idempotent unsubscribe function.
+func (s *Signal[T]) Subscribe(callback func(T)) func() {
+ if s == nil || callback == nil {
+ return func() {}
+ }
+ id := s.nextID.Add(1)
+ s.mu.Lock()
+ s.subscribers[id] = callback
+ s.mu.Unlock()
+
+ var once sync.Once
+ return func() {
+ once.Do(func() {
+ s.mu.Lock()
+ delete(s.subscribers, id)
+ s.mu.Unlock()
+ })
+ }
+}
+
+// Publish synchronously delivers value without holding the subscriber lock.
+func (s *Signal[T]) Publish(value T) {
+ if s == nil {
+ return
+ }
+ s.mu.RLock()
+ subscribers := make([]func(T), 0, len(s.subscribers))
+ for _, callback := range s.subscribers {
+ subscribers = append(subscribers, callback)
+ }
+ s.mu.RUnlock()
+
+ for _, callback := range subscribers {
+ func() {
+ var err error
+ defer utils.RecoverToError(&err, "actor signal subscriber")
+ callback(value)
+ }()
+ }
+}
diff --git a/backend/internal/actors/signal_test.go b/backend/internal/actors/signal_test.go
new file mode 100644
index 000000000..ea9d72f16
--- /dev/null
+++ b/backend/internal/actors/signal_test.go
@@ -0,0 +1,38 @@
+package actors
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestSignalPublishesOnceAndUnsubscribesInternal(t *testing.T) {
+ signal := NewSignal[int]()
+ firstCalls := 0
+ secondCalls := 0
+ unsubscribeFirst := signal.Subscribe(func(value int) {
+ firstCalls += value
+ })
+ signal.Subscribe(func(value int) {
+ secondCalls += value
+ })
+
+ signal.Publish(1)
+ unsubscribeFirst()
+ unsubscribeFirst()
+ signal.Publish(2)
+
+ require.Equal(t, 1, firstCalls)
+ require.Equal(t, 3, secondCalls)
+}
+
+func TestSignalContainsSubscriberPanicInternal(t *testing.T) {
+ signal := NewSignal[int]()
+ called := false
+ signal.Subscribe(func(int) { panic("deliberate signal panic") })
+ signal.Subscribe(func(int) { called = true })
+
+ signal.Publish(1)
+
+ require.True(t, called)
+}
diff --git a/backend/internal/actors/snapshot.go b/backend/internal/actors/snapshot.go
new file mode 100644
index 000000000..4ce5c0c4f
--- /dev/null
+++ b/backend/internal/actors/snapshot.go
@@ -0,0 +1,29 @@
+package actors
+
+import "sync/atomic"
+
+// Snapshot publishes immutable copies of a value for lock-free readers.
+type Snapshot[T any] struct {
+ value atomic.Pointer[T]
+}
+
+// Store replaces the published value with an immutable copy.
+func (s *Snapshot[T]) Store(value T) {
+ if s == nil {
+ return
+ }
+ s.value.Store(&value)
+}
+
+// Load returns a copy of the latest published value.
+func (s *Snapshot[T]) Load() (T, bool) {
+ var zero T
+ if s == nil {
+ return zero, false
+ }
+ value := s.value.Load()
+ if value == nil {
+ return zero, false
+ }
+ return *value, true
+}
diff --git a/backend/internal/actors/state.go b/backend/internal/actors/state.go
new file mode 100644
index 000000000..74e969715
--- /dev/null
+++ b/backend/internal/actors/state.go
@@ -0,0 +1,75 @@
+package actors
+
+import (
+ "context"
+
+ "emperror.dev/errors"
+)
+
+// State serializes mutations through an actor and publishes immutable snapshots.
+type State[T any] struct {
+ executor *Executor
+ value T
+ clone func(T) T
+ snapshot Snapshot[T]
+}
+
+// NewState creates actor-owned state with a caller-defined snapshot clone.
+func NewState[T any](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, initial T, clone func(T) T) (*State[T], error) {
+ if clone == nil {
+ return nil, errors.New("actor state clone unavailable")
+ }
+ executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)
+ if err != nil {
+ return nil, err
+ }
+ state := &State[T]{
+ executor: executor,
+ value: initial,
+ clone: clone,
+ }
+ state.snapshot.Store(clone(initial))
+ return state, nil
+}
+
+// Apply serializes one mutation and publishes the resulting state.
+func (s *State[T]) Apply(ctx context.Context, label string, mutate func(context.Context, *T) error) error {
+ if s == nil || s.executor == nil {
+ return errors.New("actor state unavailable")
+ }
+ if mutate == nil {
+ return errors.New("actor state mutation unavailable")
+ }
+ _, err := Execute(ctx, s.executor, label, func(actorCtx context.Context) (NoPayload, error) {
+ defer func() {
+ s.snapshot.Store(s.clone(s.value))
+ }()
+ return NoPayload{}, mutate(actorCtx, &s.value)
+ }, nil)
+ return err
+}
+
+// Load returns the latest immutable state snapshot.
+func (s *State[T]) Load() (T, bool) {
+ if s == nil {
+ var zero T
+ return zero, false
+ }
+ return s.snapshot.Load()
+}
+
+// Stop drains pending mutations and joins the state actor.
+func (s *State[T]) Stop(ctx context.Context) error {
+ if s == nil || s.executor == nil {
+ return nil
+ }
+ return s.executor.Stop(ctx)
+}
+
+// Done closes after the state actor terminates.
+func (s *State[T]) Done() <-chan struct{} {
+ if s == nil || s.executor == nil {
+ return nil
+ }
+ return s.executor.Done()
+}
diff --git a/backend/internal/actors/state_map.go b/backend/internal/actors/state_map.go
new file mode 100644
index 000000000..e9863be7a
--- /dev/null
+++ b/backend/internal/actors/state_map.go
@@ -0,0 +1,178 @@
+package actors
+
+import (
+ "context"
+ "maps"
+ "sync"
+ "sync/atomic"
+
+ "emperror.dev/errors"
+)
+
+// StateMap owns mutable keyed state while publishing immutable snapshots for readers.
+type StateMap[K comparable, V any] struct {
+ executor *Executor
+ values map[K]V
+ snapshot atomic.Pointer[map[K]V]
+ mu sync.Mutex
+}
+
+type stateMapValueInternal[V any] struct {
+ value V
+ found bool
+}
+
+// NewStateMap creates a locally serialized state map for callers without an actor runtime.
+func NewStateMap[K comparable, V any]() *StateMap[K, V] {
+ state := &StateMap[K, V]{values: make(map[K]V)}
+ state.publishInternal()
+ return state
+}
+
+// NewActorStateMap creates a state map whose mutations use the shared actor runtime.
+func NewActorStateMap[K comparable, V any](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32) (*StateMap[K, V], error) {
+ executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)
+ if err != nil {
+ return nil, err
+ }
+ state := &StateMap[K, V]{executor: executor, values: make(map[K]V)}
+ state.publishInternal()
+ return state, nil
+}
+
+// Get reads one value from the latest immutable snapshot without entering the mailbox.
+func (s *StateMap[K, V]) Get(key K) (V, bool) {
+ var zero V
+ if s == nil {
+ return zero, false
+ }
+ snapshot := s.snapshot.Load()
+ if snapshot == nil {
+ return zero, false
+ }
+ value, ok := (*snapshot)[key]
+ return value, ok
+}
+
+// Values reads every value from the latest immutable snapshot.
+func (s *StateMap[K, V]) Values() []V {
+ if s == nil {
+ return nil
+ }
+ snapshot := s.snapshot.Load()
+ if snapshot == nil {
+ return nil
+ }
+ values := make([]V, 0, len(*snapshot))
+ for _, value := range *snapshot {
+ values = append(values, value)
+ }
+ return values
+}
+
+// Apply serializes a mutation and publishes it atomically when changed is true.
+func (s *StateMap[K, V]) Apply(ctx context.Context, label string, mutate func(map[K]V) (changed bool, err error)) error {
+ if mutate == nil {
+ return errors.New("state map mutation unavailable")
+ }
+ _, err := ApplyStateMap(ctx, s, label, func(values map[K]V) (NoPayload, bool, error) {
+ changed, err := mutate(values)
+ return NoPayload{}, changed, err
+ })
+ return err
+}
+
+// ApplyStateMap serializes a typed mutation and publishes its resulting state.
+func ApplyStateMap[K comparable, V, R any](
+ ctx context.Context,
+ s *StateMap[K, V],
+ label string,
+ mutate func(map[K]V) (result R, changed bool, err error),
+) (R, error) {
+ var zero R
+ if s == nil {
+ return zero, errors.New("state map unavailable")
+ }
+ if mutate == nil {
+ return zero, errors.New("state map mutation unavailable")
+ }
+ work := func(context.Context) (R, error) {
+ completed := false
+ defer func() {
+ if !completed {
+ s.publishInternal()
+ }
+ }()
+ result, changed, err := mutate(s.values)
+ completed = true
+ if changed || err != nil {
+ s.publishInternal()
+ }
+ return result, err
+ }
+ if s.executor != nil {
+ return Execute(ctx, s.executor, label, work, nil)
+ }
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return work(ctx)
+}
+
+// Store publishes a value and returns the value it replaced, if any.
+func (s *StateMap[K, V]) Store(ctx context.Context, label string, key K, value V) (V, bool, error) {
+ result, err := ApplyStateMap(ctx, s, label, func(values map[K]V) (stateMapValueInternal[V], bool, error) {
+ previous, replaced := values[key]
+ values[key] = value
+ return stateMapValueInternal[V]{value: previous, found: replaced}, true, nil
+ })
+ return result.value, result.found, err
+}
+
+// Remove deletes and returns one value.
+func (s *StateMap[K, V]) Remove(ctx context.Context, label string, key K) (V, bool, error) {
+ result, err := ApplyStateMap(ctx, s, label, func(values map[K]V) (stateMapValueInternal[V], bool, error) {
+ value, removed := values[key]
+ if removed {
+ delete(values, key)
+ }
+ return stateMapValueInternal[V]{value: value, found: removed}, removed, nil
+ })
+ return result.value, result.found, err
+}
+
+// RemoveWhere atomically removes every value selected by predicate.
+func (s *StateMap[K, V]) RemoveWhere(ctx context.Context, label string, predicate func(K, V) bool) ([]V, error) {
+ if predicate == nil {
+ return nil, errors.New("state map predicate unavailable")
+ }
+ return ApplyStateMap(ctx, s, label, func(values map[K]V) ([]V, bool, error) {
+ removed := make([]V, 0)
+ for key, value := range values {
+ if predicate(key, value) {
+ delete(values, key)
+ removed = append(removed, value)
+ }
+ }
+ return removed, len(removed) > 0, nil
+ })
+}
+
+// Drain atomically removes and returns every value.
+func (s *StateMap[K, V]) Drain(ctx context.Context, label string) ([]V, error) {
+ return s.RemoveWhere(ctx, label, func(K, V) bool { return true })
+}
+
+// Stop drains actor-backed mutations and joins the shared executor.
+func (s *StateMap[K, V]) Stop(ctx context.Context) error {
+ if s == nil || s.executor == nil {
+ return nil
+ }
+ return s.executor.Stop(ctx)
+}
+
+func (s *StateMap[K, V]) publishInternal() {
+ snapshot := make(map[K]V, len(s.values))
+ maps.Copy(snapshot, s.values)
+ s.snapshot.Store(&snapshot)
+}
diff --git a/backend/internal/actors/state_map_test.go b/backend/internal/actors/state_map_test.go
new file mode 100644
index 000000000..fb792bffc
--- /dev/null
+++ b/backend/internal/actors/state_map_test.go
@@ -0,0 +1,114 @@
+package actors
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+func TestActorStateMapSerializesMutationsAndPublishesSnapshotsInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ state, err := NewActorStateMap[string, int](t.Context(), runtime, "state-map-test", "values", 1)
+ require.NoError(t, err)
+
+ require.NoError(t, state.Apply(t.Context(), "store value", func(values map[string]int) (bool, error) {
+ values["answer"] = 42
+ return true, nil
+ }))
+ value, ok := state.Get("answer")
+ require.True(t, ok)
+ require.Equal(t, 42, value)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, state.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestActorStateMapPublishesPartialMutationBeforeReturningErrorInternal(t *testing.T) {
+ state := NewStateMap[string, int]()
+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
+ values["removed"] = 1
+ values["retained"] = 2
+ return true, nil
+ }))
+
+ expectedErr := errors.New("mutation failed after change")
+ err := state.Apply(t.Context(), "partial mutation", func(values map[string]int) (bool, error) {
+ delete(values, "removed")
+ return true, expectedErr
+ })
+ require.ErrorIs(t, err, expectedErr)
+ _, found := state.Get("removed")
+ require.False(t, found)
+ retained, found := state.Get("retained")
+ require.True(t, found)
+ require.Equal(t, 2, retained)
+}
+
+func TestStateMapPublishesPartialMutationBeforePropagatingPanicInternal(t *testing.T) {
+ state := NewStateMap[string, int]()
+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
+ values["removed"] = 1
+ return true, nil
+ }))
+
+ require.Panics(t, func() {
+ _ = state.Apply(t.Context(), "panic after mutation", func(values map[string]int) (bool, error) {
+ delete(values, "removed")
+ panic("mutation panic")
+ })
+ })
+ _, found := state.Get("removed")
+ require.False(t, found)
+}
+
+func TestActorStateMapCanceledMutationDoesNotShareResultStorageInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ state, err := NewActorStateMap[string, int](t.Context(), runtime, "state-map-test", "canceled-result", 1)
+ require.NoError(t, err)
+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
+ values["first"] = 1
+ values["second"] = 2
+ return true, nil
+ }))
+
+ mutationStarted := make(chan struct{})
+ releaseMutation := make(chan struct{})
+ var started sync.Once
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+ result := make(chan []int, 1)
+ resultErr := make(chan error, 1)
+ go func() {
+ removed, removeErr := state.RemoveWhere(ctx, "blocked removal", func(string, int) bool {
+ started.Do(func() { close(mutationStarted) })
+ <-releaseMutation
+ return true
+ })
+ result <- removed
+ resultErr <- removeErr
+ }()
+
+ <-mutationStarted
+ require.ErrorIs(t, <-resultErr, context.DeadlineExceeded)
+ require.Empty(t, <-result)
+ close(releaseMutation)
+ require.Eventually(t, func() bool {
+ return len(state.Values()) == 0
+ }, time.Second, time.Millisecond)
+
+ stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
+ defer stopCancel()
+ require.NoError(t, state.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/actors/state_test.go b/backend/internal/actors/state_test.go
new file mode 100644
index 000000000..f57629ec0
--- /dev/null
+++ b/backend/internal/actors/state_test.go
@@ -0,0 +1,50 @@
+package actors
+
+import (
+ "context"
+ "maps"
+ "testing"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+type stateTestValueInternal struct {
+ values map[string]int
+}
+
+func TestStateSerializesMutationsAndPublishesClonedSnapshotsInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ state, err := NewState(
+ t.Context(),
+ runtime,
+ "state-test",
+ "values",
+ 1,
+ stateTestValueInternal{values: make(map[string]int)},
+ func(value stateTestValueInternal) stateTestValueInternal {
+ value.values = maps.Clone(value.values)
+ return value
+ },
+ )
+ require.NoError(t, err)
+
+ expectedErr := errors.New("mutation failed")
+ err = state.Apply(t.Context(), "mutate state", func(_ context.Context, value *stateTestValueInternal) error {
+ value.values["answer"] = 42
+ return expectedErr
+ })
+ require.ErrorIs(t, err, expectedErr)
+ snapshot, ok := state.Load()
+ require.True(t, ok)
+ require.Equal(t, 42, snapshot.values["answer"])
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, state.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 0ac4597fe..38e875e3b 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -15,6 +15,7 @@ import (
"github.com/subosito/gotenv"
"github.com/getarcaneapp/arcane/backend/v2/api/ws"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/di"
@@ -103,15 +104,15 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
func() context.Context { return appCtx },
newConfiguredHTTPClient,
),
+ di.ActorOptions,
di.ServiceOptions,
di.JobOptions,
serverOptions,
fx.Invoke(
+ registerAppCancelHook,
initializeStartupState,
registerJobs,
- registerAppRollbackCancelHook,
startEdgeTunnelClient,
- registerAppCancelHook,
),
fx.WithLogger(func() fxevent.Logger {
logger := &fxevent.SlogLogger{Logger: slog.Default()}
@@ -330,20 +331,22 @@ func runRoleStartupTasks(ctx context.Context, roleService *services.RoleService,
}
}
-func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, router *echo.Echo, _ *http.Server) {
+func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, cfg *config.Config, router *echo.Echo, _ *http.Server) {
+ var stop func(context.Context) error
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
- startEdgeTunnelClientIfConfigured(appCtx, cfg, router)
+ var startErr error
+ stop, startErr = startEdgeTunnelClientIfConfigured(appCtx, actorRuntime, cfg, router)
+ if startErr != nil {
+ slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", startErr)
+ }
return nil
},
- })
-}
-
-func registerAppRollbackCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {
- lc.Append(fx.Hook{
- OnStop: func(context.Context) error {
- cancelApp()
- return nil
+ OnStop: func(ctx context.Context) error {
+ if stop == nil {
+ return nil
+ }
+ return stop(ctx)
},
})
}
@@ -357,10 +360,10 @@ func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {
})
}
-func startEdgeTunnelClientIfConfigured(appCtx context.Context, cfg *config.Config, router http.Handler) {
+func startEdgeTunnelClientIfConfigured(appCtx context.Context, actorRuntime *actors.Runtime, cfg *config.Config, router http.Handler) (func(context.Context) error, error) {
managerEndpointConfigured := cfg.ManagerApiUrl != ""
if !cfg.EdgeAgent || !managerEndpointConfigured || cfg.AgentToken == "" {
- return
+ return nil, nil
}
edgeCfg := &edge.Config{
@@ -381,18 +384,13 @@ func startEdgeTunnelClientIfConfigured(appCtx context.Context, cfg *config.Confi
}
slog.InfoContext(appCtx, "Starting edge agent session client", edge.StartupLogAttrs(edgeCfg)...)
- errCh, err := edge.StartTunnelClientWithErrors(appCtx, edgeCfg, router)
+ stop, err := edge.StartTunnelClient(appCtx, actorRuntime, edgeCfg, router)
if err != nil {
- slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", err)
- return
+ return nil, errors.WrapIf(err, "failed to start edge tunnel client")
}
slog.InfoContext(appCtx, "Edge tunnel client started", "manager_url", cfg.ManagerApiUrl)
- go func() {
- for err := range errCh {
- slog.ErrorContext(appCtx, "Edge tunnel client error", "error", err)
- }
- }()
+ return stop, nil
}
func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config, httpClient *http.Client) error {
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 61025c28c..928dd3d10 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -3,7 +3,6 @@ package bootstrap
import (
"context"
"crypto/tls"
- "errors"
"io"
"net"
"net/http"
@@ -13,9 +12,11 @@ import (
"time"
"github.com/getarcaneapp/arcane/backend/v2/api"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/labstack/echo/v5"
"github.com/stretchr/testify/assert"
@@ -182,6 +183,7 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {
Config: cfg,
HandlerDeps: api.HandlerDeps{},
AuthMiddleware: middleware.NewAuthMiddleware(nil, cfg),
+ TunnelRegistry: edge.NewTunnelRegistry(),
})
handler, protocols := configureHTTPProtocolsInternal(false, router)
@@ -293,60 +295,37 @@ func TestNewHTTPServerRejectsInvalidTLSCertificateInternal(t *testing.T) {
require.Error(t, err)
}
-func TestRegisterAppCancelHookRunsBeforeEarlierStopHooks(t *testing.T) {
+func TestRegisterAppCancelHookRunsAfterLaterStopHooks(t *testing.T) {
appCtx, cancelApp := context.WithCancel(context.Background())
lifecycle := fxtest.NewLifecycle(t)
- appCanceledBeforeDependencyStop := false
+ registerAppCancelHook(lifecycle, cancelApp)
+ appActiveDuringDependencyStop := false
lifecycle.Append(fx.Hook{
OnStop: func(context.Context) error {
- appCanceledBeforeDependencyStop = errors.Is(appCtx.Err(), context.Canceled)
+ appActiveDuringDependencyStop = appCtx.Err() == nil
return nil
},
})
- registerAppCancelHook(lifecycle, cancelApp)
lifecycle.RequireStart()
lifecycle.RequireStop()
- require.True(t, appCanceledBeforeDependencyStop)
-}
-
-func TestRollbackCancelHookRunsBeforeEarlierStopsInternal(t *testing.T) {
- appCtx, cancelApp := context.WithCancel(context.Background())
- appCanceledBeforeDependencyStop := false
- app := fxtest.New(t,
- fx.Supply(cancelApp),
- fx.Invoke(func(lc fx.Lifecycle) {
- lc.Append(fx.Hook{
- OnStop: func(context.Context) error {
- appCanceledBeforeDependencyStop = errors.Is(appCtx.Err(), context.Canceled)
- return nil
- },
- })
- }),
- fx.Invoke(registerAppRollbackCancelHook),
- fx.Invoke(func(lc fx.Lifecycle) {
- lc.Append(fx.Hook{
- OnStart: func(context.Context) error {
- return errors.New("listen failed")
- },
- })
- }),
- )
-
- require.Error(t, app.Start(context.Background()))
- require.True(t, appCanceledBeforeDependencyStop)
+ require.True(t, appActiveDuringDependencyStop)
+ require.ErrorIs(t, appCtx.Err(), context.Canceled)
}
func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
appCtx := t.Context()
lifecycle := fxtest.NewLifecycle(t)
- jobScheduler := newJobScheduler(appCtx, lifecycle, &config.Config{}, nil, nil, nil)
+ runtime, err := actors.NewRuntime(appCtx, lifecycle)
+ require.NoError(t, err)
+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, runtime, nil, nil, nil, nil)
+ require.NoError(t, err)
watcher := &blockingBusWatcherInternal{
started: make(chan struct{}),
stopped: make(chan struct{}),
}
- jobScheduler.RegisterBusWatcher(watcher, false)
+ require.NoError(t, jobScheduler.RegisterBusWatcher(watcher, false))
lifecycle.RequireStart()
select {
diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.go
index fdce88495..8816c56cc 100644
--- a/backend/internal/bootstrap/edge_bootstrap.go
+++ b/backend/internal/bootstrap/edge_bootstrap.go
@@ -8,12 +8,14 @@ import (
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/labstack/echo/v5"
+ "go.uber.org/fx"
)
// registerEdgeTunnelRoutes configures the manager-side edge tunnel server.
@@ -21,10 +23,13 @@ import (
// Returns the TunnelServer for graceful shutdown.
func registerEdgeTunnelRoutes(
ctx context.Context,
+ lifecycle fx.Lifecycle,
+ actorRuntime *actors.Runtime,
cfg *config.Config,
apiGroup *echo.Group,
environmentService *services.EnvironmentService,
eventService *services.EventService,
+ registry *edge.TunnelRegistry,
) *edge.TunnelServer {
// Resolver that validates API key and returns the environment ID
resolver := func(ctx context.Context, token string) (string, error) {
@@ -90,7 +95,7 @@ func registerEdgeTunnelRoutes(
return nil
}
- server := edge.NewTunnelServerWithRegistry(edge.GetRegistry(), resolver, statusCallback)
+ server := edge.NewTunnelServerWithRegistry(registry, resolver, statusCallback)
server.SetConfig(&edge.Config{
EdgeMTLSMode: cfg.EdgeMTLSMode,
EdgeMTLSCAFile: cfg.EdgeMTLSCAFile,
@@ -135,7 +140,20 @@ func registerEdgeTunnelRoutes(
})
createEdgeMTLSIssueEventsInternal(ctx, eventService, envIDCopy, envNameCopy, remoteAddr, certIssued, caGenerated, reenrolled)
})
- go server.StartCleanupLoop(ctx)
+ var cleanupRunner *actors.Runner
+ lifecycle.Append(fx.Hook{
+ OnStart: func(context.Context) error {
+ var err error
+ cleanupRunner, err = actors.NewRunner(ctx, actorRuntime, "edge-tunnel", "cleanup", "edge tunnel cleanup", 3, func(runCtx context.Context) error {
+ server.StartCleanupLoop(runCtx)
+ return nil
+ })
+ return err
+ },
+ OnStop: func(stopCtx context.Context) error {
+ return cleanupRunner.Stop(stopCtx)
+ },
+ })
apiGroup.POST("/tunnel/poll", server.HandlePoll)
// Rate-limit agent mTLS enrollment per-IP. Enrollment is authenticated
// only by the agent token, so we cap bursts to mitigate brute-force or
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 18d2b4fc6..141df9fe6 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -5,22 +5,36 @@ import (
"encoding/json/v2"
"log/slog"
"net/http"
+ "slices"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/models"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"go.uber.org/fx"
)
-func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *services.SystemUpgradeService) *scheduler.JobScheduler {
+func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *actors.Runtime, _ *actors.Gate[actors.AdmissionKey], imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *services.SystemUpgradeService) (schedulertypes.JobScheduler, error) {
schedulerCtx, cancelScheduler := context.WithCancel(appCtx)
- jobScheduler := scheduler.NewJobScheduler(schedulerCtx, cfg.GetLocation())
+ jobScheduler, err := scheduler.NewJobScheduler(schedulerCtx, runtime, cfg.GetLocation())
+ if err != nil {
+ cancelScheduler()
+ return nil, err
+ }
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
slog.InfoContext(appCtx, "Starting scheduler")
- jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true)
- jobScheduler.StartScheduler()
+ if imageUpdateWatcher != nil {
+ if err := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); err != nil {
+ return err
+ }
+ }
+ if err := jobScheduler.StartScheduler(); err != nil {
+ return err
+ }
if analytics != nil {
go analytics.Run(schedulerCtx)
}
@@ -30,8 +44,8 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
return nil
},
OnStop: func(ctx context.Context) error {
- cancelScheduler()
err := jobScheduler.Stop(ctx)
+ cancelScheduler()
if err != nil {
slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)
return err
@@ -40,21 +54,24 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
return nil
},
})
- return jobScheduler
+ return jobScheduler, nil
}
type registerJobsParams struct {
fx.In
- AppCtx context.Context
- Config *config.Config
- Scheduler *scheduler.JobScheduler
+ Lifecycle fx.Lifecycle
+ AppCtx context.Context
+ Config *config.Config
+ Scheduler schedulertypes.JobScheduler
+ ActorRuntime *actors.Runtime
Activity *services.ActivityService
GitOpsSync *services.GitOpsSyncService
Environment *services.EnvironmentService
JobSchedule *services.JobService
Settings *services.SettingsService
+ Admission *actors.Gate[actors.AdmissionKey]
AutoUpdate *scheduler.AutoUpdateJob
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
@@ -70,7 +87,7 @@ type registerJobsParams struct {
ActivitySweep *scheduler.ActivitySweepJob
}
-func registerJobs(params registerJobsParams) {
+func registerJobs(params registerJobsParams) error {
params.JobSchedule.SetScheduler(params.AppCtx, params.Scheduler)
// Bootstrap owns registration, agent-mode gating, and settings callbacks.
@@ -97,36 +114,47 @@ func registerJobs(params registerJobsParams) {
}
}
- params.Scheduler.RegisterJob(params.AutoUpdate)
- params.Scheduler.RegisterJob(params.DockerClientRefresh)
- params.Scheduler.RegisterJob(params.Analytics)
- params.Scheduler.RegisterJob(params.EventCleanup)
- params.Scheduler.RegisterJob(params.PruningVolumeHelper)
- params.Scheduler.RegisterJob(params.ExpiredSessionsCleanup)
- params.Scheduler.RegisterJob(params.ScheduledPrune)
+ for _, job := range []schedulertypes.Job{
+ params.AutoUpdate,
+ params.DockerClientRefresh,
+ params.Analytics,
+ params.EventCleanup,
+ params.PruningVolumeHelper,
+ params.ExpiredSessionsCleanup,
+ params.ScheduledPrune,
+ params.VulnerabilityScan,
+ params.AutoHeal,
+ params.ActivitySweep,
+ } {
+ if err := params.Scheduler.RegisterJob(job); err != nil {
+ return err
+ }
+ }
// FilesystemWatcher is intentionally not scheduler-registered; it watches inline
// and is only rebound on settings changes below.
- params.Scheduler.RegisterJob(params.VulnerabilityScan)
- params.Scheduler.RegisterJob(params.AutoHeal)
// Internal self-healing sweep (managers and agents alike); intentionally
// absent from job_metadata so it stays out of the Jobs UI.
- params.Scheduler.RegisterJob(params.ActivitySweep)
// GitOps sync and environment health are no longer single global jobs; each
// entity registers its own dynamic job.
- registerDynamicJobs(dynamicJobsParams{
+ if err := registerDynamicJobs(dynamicJobsParams{
AppCtx: params.AppCtx,
Config: params.Config,
Scheduler: params.Scheduler,
GitOpsSync: params.GitOpsSync,
Environment: params.Environment,
JobSchedule: params.JobSchedule,
- })
+ Admission: params.Admission,
+ }); err != nil {
+ return err
+ }
- setupSettingsCallbacks(settingsCallbacksParams{
+ if err := setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{
+ Lifecycle: params.Lifecycle,
LifecycleCtx: params.AppCtx,
Config: params.Config,
Scheduler: params.Scheduler,
+ ActorRuntime: params.ActorRuntime,
Settings: params.Settings,
Environment: params.Environment,
AutoUpdate: params.AutoUpdate,
@@ -135,25 +163,31 @@ func registerJobs(params registerJobsParams) {
ScheduledPrune: params.ScheduledPrune,
VulnerabilityScan: params.VulnerabilityScan,
AutoHeal: params.AutoHeal,
- })
+ }); err != nil {
+ return err
+ }
+ return nil
}
type dynamicJobsParams struct {
AppCtx context.Context
Config *config.Config
- Scheduler *scheduler.JobScheduler
+ Scheduler schedulertypes.JobScheduler
GitOpsSync *services.GitOpsSyncService
Environment *services.EnvironmentService
JobSchedule *services.JobService
+ Admission *actors.Gate[actors.AdmissionKey]
}
// registerDynamicJobs injects the scheduler into the services that own per-entity
// jobs and registers the jobs for already-existing entities at startup. AddJob is
// an idempotent upsert, so these run safely before the scheduler is started.
-func registerDynamicJobs(params dynamicJobsParams) {
+func registerDynamicJobs(params dynamicJobsParams) error {
// GitOps: one job per auto-sync-enabled sync (runs on manager and agents).
if params.GitOpsSync != nil {
- params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler)
+ if err := params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
+ return err
+ }
params.GitOpsSync.RegisterAutoSyncJobsOnStartup(params.AppCtx)
}
@@ -161,7 +195,9 @@ func registerDynamicJobs(params dynamicJobsParams) {
// UI still addresses "environment-health" by ID, so bridge its reschedule and
// run-now back to EnvironmentService.
if !params.Config.AgentMode && params.Environment != nil {
- params.Environment.SetScheduler(params.AppCtx, params.Scheduler)
+ if err := params.Environment.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
+ return err
+ }
params.JobSchedule.OnEnvironmentHealthReschedule = func(ctx context.Context) {
params.Environment.RescheduleHealthJobs(ctx)
}
@@ -170,14 +206,17 @@ func registerDynamicJobs(params dynamicJobsParams) {
}
params.Environment.RegisterHealthJobsOnStartup(params.AppCtx)
}
+ return nil
}
-type settingsCallbacksParams struct {
+type settingsSubscriptionsParams struct {
+ Lifecycle fx.Lifecycle
LifecycleCtx context.Context
Config *config.Config
- Scheduler *scheduler.JobScheduler
- Settings *services.SettingsService
- Environment *services.EnvironmentService
+ Scheduler settingsEffectsSchedulerInternal
+ ActorRuntime *actors.Runtime
+ Settings settingsChangeSubscriberInternal
+ Environment timeoutSettingsEnvironmentInternal
AutoUpdate *scheduler.AutoUpdateJob
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
@@ -187,9 +226,30 @@ type settingsCallbacksParams struct {
AutoHeal *scheduler.AutoHealJob
}
-//nolint:contextcheck // callbacks intentionally use the app lifecycle context so reschedules outlive the triggering request context.
-func setupSettingsCallbacks(params settingsCallbacksParams) {
- params.Settings.OnImagePollingSettingsChanged = func(_ context.Context) {
+type settingsChangeSubscriberInternal interface {
+ SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func()
+}
+
+type settingsEffectsSchedulerInternal interface {
+ RescheduleJob(ctx context.Context, job schedulertypes.Job) error
+}
+
+type timeoutSettingsEnvironmentInternal interface {
+ ListRemoteEnvironments(ctx context.Context) ([]models.Environment, error)
+ ProxyRequest(ctx context.Context, envID string, method string, path string, body []byte) ([]byte, int, error)
+}
+
+func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) error {
+ unsubscribes := make([]func(), 0, 8)
+ subscribe := func(keys []string, callback func([]libarcane.SettingUpdate)) {
+ unsubscribes = append(unsubscribes, params.Settings.SubscribeSettingsChanges(keys, callback))
+ }
+ timeoutSyncExecutor, cancelTimeoutSync, err := setupTimeoutSettingsSubscriptionInternal(params, subscribe)
+ if err != nil {
+ return err
+ }
+
+ subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {
if params.ImageUpdateWatcher != nil {
params.ImageUpdateWatcher.RefreshSchedule()
params.ImageUpdateWatcher.Trigger()
@@ -197,53 +257,95 @@ func setupSettingsCallbacks(params settingsCallbacksParams) {
if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoUpdate); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-update job", "error", err)
}
- }
- params.Settings.OnAutoUpdateSettingsChanged = func(ctx context.Context) {
- slog.DebugContext(params.LifecycleCtx, "AutoUpdateSettingsChanged callback triggered", "triggerContextCanceled", ctx.Err() != nil)
+ })
+
+ subscribe([]string{"autoUpdate", "autoUpdateInterval"}, func(_ []libarcane.SettingUpdate) {
if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoUpdate); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-update job", "error", err)
}
- }
- params.Settings.OnProjectsDirectoryChanged = func(_ context.Context) {
+ })
+
+ subscribe([]string{"projectsDirectory", "followProjectSymlinks"}, func(_ []libarcane.SettingUpdate) {
if params.FilesystemWatcher != nil {
if err := params.FilesystemWatcher.RestartProjectsWatcher(params.LifecycleCtx); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to restart projects filesystem watcher", "error", err)
}
}
- }
- params.Settings.OnTemplatesDirectoryChanged = func(_ context.Context) {
+ })
+
+ subscribe([]string{"templatesDirectory"}, func(_ []libarcane.SettingUpdate) {
if params.FilesystemWatcher != nil {
if err := params.FilesystemWatcher.RestartTemplatesWatcher(params.LifecycleCtx); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)
}
}
- }
- params.Settings.OnScheduledPruneSettingsChanged = func(_ context.Context) {
+ })
+
+ subscribe([]string{"scheduledPruneEnabled", "scheduledPruneInterval"}, func(_ []libarcane.SettingUpdate) {
if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.ScheduledPrune); err != nil {
slog.WarnContext(params.LifecycleCtx, "Failed to reschedule scheduled-prune job", "error", err)
}
- }
- params.Settings.OnVulnerabilityScanSettingsChanged = func(_ context.Context) {
- if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.VulnerabilityScan); err != nil {
- slog.WarnContext(params.LifecycleCtx, "Failed to reschedule vulnerability-scan job", "error", err)
- }
- }
- params.Settings.OnAutoHealSettingsChanged = func(ctx context.Context) {
- if err := params.Scheduler.RescheduleJob(ctx, params.AutoHeal); err != nil {
- slog.WarnContext(ctx, "Failed to reschedule auto-heal job", "error", err)
- }
+ })
+
+ subscribe(
+ []string{
+ "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyNetwork", "trivySecurityOpts", "trivyPrivileged",
+ "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers",
+ },
+ func(_ []libarcane.SettingUpdate) {
+ if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.VulnerabilityScan); err != nil {
+ slog.WarnContext(params.LifecycleCtx, "Failed to reschedule vulnerability-scan job", "error", err)
+ }
+ },
+ )
+
+ subscribe(
+ []string{"autoHealEnabled", "autoHealInterval", "autoHealExcludedContainers", "autoHealMaxRestarts", "autoHealRestartWindow"},
+ func(_ []libarcane.SettingUpdate) {
+ if err := params.Scheduler.RescheduleJob(params.LifecycleCtx, params.AutoHeal); err != nil {
+ slog.WarnContext(params.LifecycleCtx, "Failed to reschedule auto-heal job", "error", err)
+ }
+ },
+ )
+
+ params.Lifecycle.Append(fx.Hook{
+ OnStop: func(ctx context.Context) error {
+ for _, unsubscribe := range slices.Backward(unsubscribes) {
+ unsubscribe()
+ }
+ if cancelTimeoutSync != nil {
+ cancelTimeoutSync()
+ }
+ return timeoutSyncExecutor.Stop(ctx)
+ },
+ })
+ return nil
+}
+
+func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*actors.Executor, context.CancelFunc, error) {
+ if params.Config.AgentMode {
+ return nil, nil, nil
}
- // Only set up timeout sync callback on main instance (not in agent mode)
- if !params.Config.AgentMode {
- params.Settings.OnTimeoutSettingsChanged = func(ctx context.Context, timeoutSettings []libarcane.SettingUpdate) {
- go syncTimeoutSettingsToAgentsInternal(context.WithoutCancel(ctx), params.Environment, timeoutSettings)
- }
+ timeoutSyncExecutor, err := actors.NewExecutor(params.LifecycleCtx, params.ActorRuntime, "services", "settings-timeout-sync", 3)
+ if err != nil {
+ return nil, nil, err
}
+ timeoutSyncContext, cancelTimeoutSync := context.WithCancel(params.LifecycleCtx)
+ subscribe(libarcane.TimeoutSettingKeys(), func(updates []libarcane.SettingUpdate) {
+ _, err := actors.Submit(timeoutSyncContext, timeoutSyncExecutor, "sync timeout settings to remote environments", func(ctx context.Context) (actors.NoPayload, error) {
+ syncTimeoutSettingsToAgentsInternal(ctx, params.Environment, updates)
+ return actors.NoPayload{}, nil
+ }, nil)
+ if err != nil && timeoutSyncContext.Err() == nil {
+ slog.ErrorContext(timeoutSyncContext, "Failed to queue timeout settings sync", "error", err)
+ }
+ })
+ return timeoutSyncExecutor, cancelTimeoutSync, nil
}
// syncTimeoutSettingsToAgentsInternal syncs timeout settings to all connected remote environments
-func syncTimeoutSettingsToAgentsInternal(ctx context.Context, environment *services.EnvironmentService, timeoutSettings []libarcane.SettingUpdate) {
+func syncTimeoutSettingsToAgentsInternal(ctx context.Context, environment timeoutSettingsEnvironmentInternal, timeoutSettings []libarcane.SettingUpdate) {
envs, err := environment.ListRemoteEnvironments(ctx)
if err != nil {
slog.WarnContext(ctx, "Failed to list remote environments for timeout sync", "error", err)
@@ -270,13 +372,13 @@ func syncTimeoutSettingsToAgentsInternal(ctx context.Context, environment *servi
slog.InfoContext(ctx, "Syncing environment settings to remote environments", "count", len(envs), "keys", keys)
for _, env := range envs {
- resp, err := environment.ExecuteRemoteRequest(ctx, env.ID, http.MethodPut, "/api/environments/0/settings", body)
+ responseBody, statusCode, err := environment.ProxyRequest(ctx, env.ID, http.MethodPut, "/api/environments/0/settings", body)
if err != nil {
slog.WarnContext(ctx, "Failed to sync timeout settings to environment", "environmentID", env.ID, "environmentName", env.Name, "error", err)
continue
}
- if err := resp.RequireSuccess(); err != nil {
- slog.WarnContext(ctx, "Environment returned non-OK status for timeout sync", "environmentID", env.ID, "environmentName", env.Name, "statusCode", resp.StatusCode, "response", string(resp.Body))
+ if statusCode < http.StatusOK || statusCode >= http.StatusMultipleChoices {
+ slog.WarnContext(ctx, "Environment returned non-OK status for timeout sync", "environmentID", env.ID, "environmentName", env.Name, "statusCode", statusCode, "response", string(responseBody))
continue
}
slog.DebugContext(ctx, "Successfully synced timeout settings to environment", "environmentID", env.ID, "environmentName", env.Name)
diff --git a/backend/internal/bootstrap/jobs_bootstrap_test.go b/backend/internal/bootstrap/jobs_bootstrap_test.go
new file mode 100644
index 000000000..5466912e7
--- /dev/null
+++ b/backend/internal/bootstrap/jobs_bootstrap_test.go
@@ -0,0 +1,112 @@
+package bootstrap
+
+import (
+ "context"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/models"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+type settingsSubscriptionStubInternal struct {
+ pollingCallback func([]libarcane.SettingUpdate)
+ timeoutCallback func([]libarcane.SettingUpdate)
+}
+
+func (s *settingsSubscriptionStubInternal) SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func() {
+ if slices.Contains(keys, "pollingEnabled") {
+ s.pollingCallback = callback
+ }
+ if slices.Contains(keys, "dockerApiTimeout") {
+ s.timeoutCallback = callback
+ }
+ return func() {}
+}
+
+type settingsSubscriptionSchedulerStubInternal struct {
+ rescheduled chan struct{}
+}
+
+func (s *settingsSubscriptionSchedulerStubInternal) RescheduleJob(context.Context, schedulertypes.Job) error {
+ select {
+ case <-s.rescheduled:
+ default:
+ close(s.rescheduled)
+ }
+ return nil
+}
+
+type timeoutSyncEnvironmentStubInternal struct {
+ started chan struct{}
+}
+
+func (s *timeoutSyncEnvironmentStubInternal) ListRemoteEnvironments(context.Context) ([]models.Environment, error) {
+ return []models.Environment{{
+ BaseModel: models.BaseModel{ID: "remote"},
+ Name: "remote",
+ }}, nil
+}
+
+func (s *timeoutSyncEnvironmentStubInternal) ProxyRequest(ctx context.Context, _ string, _ string, _ string, _ []byte) ([]byte, int, error) {
+ select {
+ case <-s.started:
+ default:
+ close(s.started)
+ }
+ <-ctx.Done()
+ return nil, 0, ctx.Err()
+}
+
+func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(context.Background(), lifecycle)
+ require.NoError(t, err)
+
+ settings := &settingsSubscriptionStubInternal{}
+ scheduler := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}
+ environment := &timeoutSyncEnvironmentStubInternal{started: make(chan struct{})}
+ require.NoError(t, setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{
+ Lifecycle: lifecycle,
+ LifecycleCtx: context.Background(),
+ Config: &config.Config{},
+ Scheduler: scheduler,
+ ActorRuntime: runtime,
+ Settings: settings,
+ Environment: environment,
+ }))
+
+ timeoutCallbackDone := make(chan struct{})
+ go func() {
+ settings.timeoutCallback([]libarcane.SettingUpdate{{Key: "dockerApiTimeout", Value: "30"}})
+ close(timeoutCallbackDone)
+ }()
+
+ select {
+ case <-environment.started:
+ case <-time.After(time.Second):
+ t.Fatal("timeout sync did not start")
+ }
+ select {
+ case <-timeoutCallbackDone:
+ case <-time.After(time.Second):
+ t.Fatal("timeout settings callback remained blocked by remote sync")
+ }
+
+ settings.pollingCallback([]libarcane.SettingUpdate{{Key: "pollingEnabled", Value: "true"}})
+ select {
+ case <-scheduler.rescheduled:
+ case <-time.After(time.Second):
+ t.Fatal("local settings effect was blocked by remote timeout sync")
+ }
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 0cc697927..e58d84476 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -18,6 +18,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/api/handlers"
"github.com/getarcaneapp/arcane/backend/v2/api/ws"
"github.com/getarcaneapp/arcane/backend/v2/frontend"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
@@ -160,9 +161,12 @@ type RouterParams struct {
fx.In
Context context.Context
+ Lifecycle fx.Lifecycle
+ ActorRuntime *actors.Runtime
Config *config.Config
HandlerDeps api.HandlerDeps
AuthMiddleware *middleware.AuthMiddleware
+ TunnelRegistry *edge.TunnelRegistry
}
func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
@@ -217,8 +221,6 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
))
handlerAppCtx := handlers.NewActivityAppContext(ctx)
- tunnelRegistry := edge.NewTunnelRegistry()
- edge.SetDefaultRegistry(tunnelRegistry)
envResolver := func(ctx context.Context, id string) (string, *string, bool, error) {
env, err := deps.Environment.GetEnvironmentByID(ctx, id)
if err != nil || env == nil {
@@ -240,7 +242,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
envResolver,
createAuthValidatorInternal(deps),
permissionMatcher,
- tunnelRegistry,
+ p.TunnelRegistry,
// Proxied WebSocket upgrades enforce the same Origin policy as the local
// endpoints, so a cross-origin page cannot ride a session cookie into a
// remote environment.
@@ -268,7 +270,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
// This is only registered when NOT in agent mode (i.e., running as manager)
var tunnelServer *edge.TunnelServer
if !cfg.AgentMode {
- tunnelServer = registerEdgeTunnelRoutes(ctx, cfg, apiGroup, deps.Environment, deps.Event)
+ tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, p.ActorRuntime, cfg, apiGroup, deps.Environment, deps.Event, p.TunnelRegistry)
}
if cfg.Environment != "production" {
diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.go
index 806c1abd4..120706299 100644
--- a/backend/internal/bootstrap/server_bootstrap.go
+++ b/backend/internal/bootstrap/server_bootstrap.go
@@ -108,9 +108,6 @@ func NewHTTPServer(lc fx.Lifecycle, p HTTPServerParams) (*http.Server, error) {
if grpcServer != nil {
grpcServer.GracefulStop()
}
- if p.TunnelServer != nil {
- p.TunnelServer.WaitForCleanupDone()
- }
if shutdownErr == nil {
slog.InfoContext(ctx, "Server stopped gracefully")
}
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index df25cb880..204608808 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -2,12 +2,20 @@
package di
import (
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
arcanelogging "github.com/getarcaneapp/arcane/backend/v2/internal/logging"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
"go.uber.org/fx"
)
+// ActorOptions provides the shared in-process actor runtime.
+var ActorOptions = fx.Options(
+ fx.Provide(actors.NewRuntime),
+ fx.Provide(provideAdmissionGateInternal),
+ fx.Provide(provideTunnelRegistryInternal),
+)
+
// ServiceOptions provides the backend service graph.
var ServiceOptions = fx.Options(
fx.Provide(
@@ -18,7 +26,7 @@ var ServiceOptions = fx.Options(
// Services constructed directly through their public constructors.
services.NewEventService,
services.NewActivityService,
- services.NewSettingsService,
+ provideSettingsServiceInternal,
services.NewKVService,
services.NewJobService,
services.NewSettingsSearchService,
diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.go
index f20fed7f9..10f3153b1 100644
--- a/backend/internal/di/di_test.go
+++ b/backend/internal/di/di_test.go
@@ -82,6 +82,7 @@ func TestOptionsValidate(t *testing.T) {
&http.Client{},
),
fx.Provide(func() context.Context { return context.Background() }),
+ ActorOptions,
ServiceOptions,
JobOptions,
fx.Invoke(func(graphParams) {}),
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 57d85560b..33362b35b 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -6,11 +6,14 @@ import (
"log/slog"
"net/http"
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
arcanelogging "github.com/getarcaneapp/arcane/backend/v2/internal/logging"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
"github.com/getarcaneapp/arcane/backend/v2/resources"
"go.uber.org/fx"
@@ -20,16 +23,67 @@ func provideResourcesFSInternal() embed.FS {
return resources.FS
}
-func provideDockerClientServiceInternal(ctx context.Context, lc fx.Lifecycle, db *database.DB, cfg *config.Config, settings *services.SettingsService) *services.DockerClientService {
+func provideSettingsServiceInternal(ctx context.Context, lc fx.Lifecycle, db *database.DB, runtime *actors.Runtime) (*services.SettingsService, error) {
+ executor, err := actors.NewExecutor(ctx, runtime, "services", "settings", 3)
+ if err != nil {
+ return nil, err
+ }
+ effects, err := actors.NewExecutor(ctx, runtime, "services", "settings-effects", 3)
+ if err != nil {
+ return nil, errors.Combine(err, executor.Stop(ctx))
+ }
+ service, err := services.NewSettingsService(ctx, db, executor, effects)
+ if err != nil {
+ return nil, errors.Combine(err, executor.Stop(ctx), effects.Stop(ctx))
+ }
+ lc.Append(fx.Hook{
+ OnStop: func(ctx context.Context) error {
+ return errors.Combine(executor.Stop(ctx), effects.Stop(ctx))
+ },
+ })
+ return service, nil
+}
+
+func provideAdmissionGateInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*actors.Gate[actors.AdmissionKey], error) {
+ gate, err := actors.NewGate[actors.AdmissionKey](ctx, runtime, "admission", "application")
+ if err != nil {
+ return nil, err
+ }
+ lc.Append(fx.Hook{OnStop: gate.Stop})
+ return gate, nil
+}
+
+func provideTunnelRegistryInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*edge.TunnelRegistry, error) {
+ registry, err := edge.NewActorTunnelRegistry(ctx, runtime)
+ if err != nil {
+ return nil, err
+ }
+ edge.SetDefaultRegistry(registry)
+ lc.Append(fx.Hook{
+ OnStop: func(ctx context.Context) error {
+ defer edge.ClearDefaultRegistry(registry)
+ return registry.Stop(ctx)
+ },
+ })
+ return registry, nil
+}
+
+func provideDockerClientServiceInternal(ctx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, db *database.DB, cfg *config.Config, settings *services.SettingsService) *services.DockerClientService {
service := services.NewDockerClientService(ctx, db, cfg, settings)
+ var runner *actors.Runner
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
- go service.WatchEvents(ctx)
- return nil
+ var err error
+ runner, err = actors.NewRunner(ctx, actorRuntime, "docker-events", "stream", "Docker event watcher", 3, func(runCtx context.Context) error {
+ service.WatchEvents(runCtx)
+ return nil
+ })
+ return err
},
- OnStop: func(context.Context) error {
+ OnStop: func(stopCtx context.Context) error {
+ err := runner.Stop(stopCtx)
service.Close()
- return nil
+ return err
},
})
return service
@@ -113,10 +167,30 @@ func provideAuthMiddlewareInternal(auth *services.AuthService, apiKey *services.
WithPermissionResolver(role)
}
-func provideFilesystemWatcherJobInternal(ctx context.Context, project *services.ProjectService, template *services.TemplateService, settings *services.SettingsService, cfg *config.Config) *scheduler.FilesystemWatcherJob {
- job, err := scheduler.RegisterFilesystemWatcherJob(ctx, project, template, settings, cfg.ProjectScanMaxDepth)
+func provideFilesystemWatcherJobInternal(ctx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, project *services.ProjectService, template *services.TemplateService, settings *services.SettingsService, cfg *config.Config) (*scheduler.FilesystemWatcherJob, error) {
+ job, err := scheduler.NewFilesystemWatcherJob(ctx, actorRuntime, project, template, settings, cfg.ProjectScanMaxDepth)
if err != nil {
- slog.ErrorContext(ctx, "Failed to register filesystem watcher job", "error", err)
+ return nil, err
}
- return job
+ var runner *actors.Runner
+ lc.Append(fx.Hook{
+ OnStart: func(context.Context) error {
+ runner, err = actors.NewRunner(ctx, actorRuntime, "filesystem-watcher", "lifecycle", "filesystem watcher", 3, func(runCtx context.Context) error {
+ if startErr := job.Start(runCtx); startErr != nil {
+ return startErr
+ }
+ <-runCtx.Done()
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ slog.InfoContext(ctx, "Filesystem watcher job registered")
+ return nil
+ },
+ OnStop: func(stopCtx context.Context) error {
+ return errors.Combine(runner.Stop(stopCtx), job.Stop(stopCtx))
+ },
+ })
+ return job, nil
}
diff --git a/backend/internal/services/activity_service_test.go b/backend/internal/services/activity_service_test.go
index 2a9ec0aac..322c026b2 100644
--- a/backend/internal/services/activity_service_test.go
+++ b/backend/internal/services/activity_service_test.go
@@ -373,7 +373,7 @@ func setupQueuedActivityServiceInternal(t *testing.T) (*ActivityService, context
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
- settings, err := NewSettingsService(ctx, wrapped)
+ settings, err := newSettingsServiceForTestInternal(t, ctx, wrapped)
require.NoError(t, err)
require.NoError(t, settings.SetIntSetting(ctx, maxConcurrentActivitiesSettingKey, 1))
return NewActivityService(wrapped, settings), ctx
diff --git a/backend/internal/services/auth_service_buildables_test.go b/backend/internal/services/auth_service_buildables_test.go
index 38acdf519..333f2795c 100644
--- a/backend/internal/services/auth_service_buildables_test.go
+++ b/backend/internal/services/auth_service_buildables_test.go
@@ -25,7 +25,7 @@ func TestGetAutoLoginConfig_Enabled(t *testing.T) {
ctx := context.Background()
db := setupAuthServiceTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "authLocalEnabled", true))
@@ -51,7 +51,7 @@ func TestGetAutoLoginConfig_DisabledWhenLocalAuthDisabled(t *testing.T) {
ctx := context.Background()
db := setupAuthServiceTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "authLocalEnabled", false))
diff --git a/backend/internal/services/auth_service_test.go b/backend/internal/services/auth_service_test.go
index 3e56df30a..2f9685905 100644
--- a/backend/internal/services/auth_service_test.go
+++ b/backend/internal/services/auth_service_test.go
@@ -324,7 +324,7 @@ func TestVerifyToken_VersionMismatch(t *testing.T) {
func TestRefreshToken_Valid(t *testing.T) {
db := setupAuthServiceTestDB(t)
userSvc := NewUserService(db)
- settingsSvc, err := NewSettingsService(context.Background(), db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService("")
s.userService = userSvc
@@ -355,7 +355,7 @@ func TestRefreshToken_Valid(t *testing.T) {
func TestRefreshToken_VersionMismatchRotates(t *testing.T) {
db := setupAuthServiceTestDB(t)
userSvc := NewUserService(db)
- settingsSvc, err := NewSettingsService(context.Background(), db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService("")
s.userService = userSvc
@@ -470,7 +470,7 @@ func TestRevokeSessionThenVerifyTokenFails(t *testing.T) {
func TestRefreshToken_RotatesJTI(t *testing.T) {
db := setupAuthServiceTestDB(t)
userSvc := NewUserService(db)
- settingsSvc, err := NewSettingsService(context.Background(), db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService("")
s.userService = userSvc
@@ -499,7 +499,7 @@ func TestRefreshToken_RotatesJTI(t *testing.T) {
func TestRefreshToken_RejectsRevokedSession(t *testing.T) {
db := setupAuthServiceTestDB(t)
userSvc := NewUserService(db)
- settingsSvc, err := NewSettingsService(context.Background(), db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService("")
s.userService = userSvc
@@ -658,7 +658,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailNotVerified_NoExistingUser_Creat
ctx := context.Background()
db := setupAuthServiceTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))
@@ -694,7 +694,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailNotVerified_WithExistingUser_Ret
ctx := context.Background()
db := setupAuthServiceTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))
@@ -736,7 +736,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailVerificationMissing_WithExisting
ctx := context.Background()
db := setupAuthServiceTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))
diff --git a/backend/internal/services/build_workspace_service_test.go b/backend/internal/services/build_workspace_service_test.go
index 7426c5589..491874812 100644
--- a/backend/internal/services/build_workspace_service_test.go
+++ b/backend/internal/services/build_workspace_service_test.go
@@ -12,7 +12,7 @@ import (
func TestBuildWorkspaceServiceDeleteFileRejectsCurrentDirectoryInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
root := t.TempDir()
diff --git a/backend/internal/services/dashboard_service_test.go b/backend/internal/services/dashboard_service_test.go
index 530d5a2be..727a09439 100644
--- a/backend/internal/services/dashboard_service_test.go
+++ b/backend/internal/services/dashboard_service_test.go
@@ -34,7 +34,7 @@ func setupDashboardServiceTestDB(t *testing.T) (*database.DB, *SettingsService)
require.NoError(t, db.AutoMigrate(&models.ApiKey{}, &models.Environment{}, &models.ImageUpdateRecord{}, &models.Project{}, &models.SettingVariable{}))
databaseDB := &database.DB{DB: db}
- settingsSvc, err := NewSettingsService(context.Background(), databaseDB)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), databaseDB)
require.NoError(t, err)
return databaseDB, settingsSvc
diff --git a/backend/internal/services/docker_client_service_test.go b/backend/internal/services/docker_client_service_test.go
index e7ffb871e..f0923ad95 100644
--- a/backend/internal/services/docker_client_service_test.go
+++ b/backend/internal/services/docker_client_service_test.go
@@ -10,6 +10,7 @@ import (
"testing"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/events"
@@ -17,6 +18,7 @@ import (
"github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
)
func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {
@@ -189,6 +191,68 @@ func TestDockerClientService_PublishImageStateResyncNotifiesSubscribers(t *testi
}
}
+func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *testing.T) {
+ streamStarted := make(chan struct{})
+ streamStopped := make(chan struct{})
+ server := newDockerPingTestServerWithHandlerInternal(t, func(w http.ResponseWriter, r *http.Request) {
+ switch dockerTestPathInternal(r.URL.Path) {
+ case "/_ping":
+ w.Header().Set("Api-Version", "1.41")
+ w.WriteHeader(http.StatusOK)
+ case "/events":
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ if flusher, ok := w.(http.Flusher); ok {
+ flusher.Flush()
+ }
+ close(streamStarted)
+ <-r.Context().Done()
+ close(streamStopped)
+ default:
+ http.NotFound(w, r)
+ }
+ })
+
+ lifecycle := fxtest.NewLifecycle(t)
+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ service := newDockerClientServiceForTestInternal(server.URL)
+ t.Cleanup(service.Close)
+
+ eventsCh, unsubscribe := service.EventBus().Subscribe(events.ImageEventType)
+ t.Cleanup(unsubscribe)
+ runner, err := actors.NewRunner(t.Context(), actorRuntime, "docker-events-test", "stream", "Docker event watcher", 3, func(ctx context.Context) error {
+ service.WatchEvents(ctx)
+ return nil
+ })
+ require.NoError(t, err)
+
+ select {
+ case <-streamStarted:
+ case <-time.After(time.Second):
+ t.Fatal("Docker event stream did not start")
+ }
+ select {
+ case message := <-eventsCh:
+ require.Equal(t, dockerImageStateResyncActionInternal, message.Action)
+ case <-time.After(time.Second):
+ t.Fatal("Docker event stream did not publish reconnect resync")
+ }
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, runner.Stop(stopCtx))
+ require.Eventually(t, func() bool {
+ select {
+ case <-streamStopped:
+ return true
+ default:
+ return false
+ }
+ }, time.Second, time.Millisecond)
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
func TestCountImageUsage_UsesContainerImageIDs(t *testing.T) {
images := []image.Summary{
{ID: "sha256:image-a", Containers: -1},
diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.go
index 260df93b1..8527231b5 100644
--- a/backend/internal/services/environment_service.go
+++ b/backend/internal/services/environment_service.go
@@ -15,6 +15,7 @@ import (
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
@@ -54,11 +55,9 @@ type EnvironmentService struct {
// scheduler and lifecycleCtx are injected post-construction via SetScheduler
// (manager-only). Each enabled environment gets its own health-check job; this
// replaces the single global environment-health job.
- scheduler DynamicScheduler
- lifecycleCtx context.Context
- // runningHealthChecks guards against a per-environment health check overlapping
- // with itself (replaces the old single job's atomic "running" guard).
- runningHealthChecks sync.Map
+ scheduler schedulertypes.DynamicScheduler
+ lifecycleCtx context.Context
+ admissionGate *actors.Gate[actors.AdmissionKey]
// variableSyncer is injected post-construction via SetVariableSyncer
// (manager-only) to avoid a wire cycle with VariableService.
@@ -114,18 +113,24 @@ func NewEnvironmentService(db *database.DB, httpClient *http.Client, dockerServi
}
const environmentHealthJobPrefix = "environment-health:"
+const environmentHealthAdmissionScopeInternal = "environment-health"
func environmentHealthJobNameInternal(envID string) string { return environmentHealthJobPrefix + envID }
// SetScheduler injects the job scheduler and app lifecycle context. Called during
// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job
// registration becomes a no-op).
-func (s *EnvironmentService) SetScheduler(ctx context.Context, scheduler DynamicScheduler) { //nolint:contextcheck // health-check jobs must capture the app lifecycle context, not request contexts
+func (s *EnvironmentService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error { //nolint:contextcheck // health-check jobs must capture the app lifecycle context, not request contexts
+ if scheduler == nil || admissionGate == nil {
+ return errors.New("environment scheduler dependencies unavailable")
+ }
if ctx == nil {
ctx = context.Background()
}
s.lifecycleCtx = ctx
s.scheduler = scheduler
+ s.admissionGate = admissionGate
+ return nil
}
// SetVariableSyncer injects the global-variable syncer. Called during
@@ -236,22 +241,19 @@ func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {
return nil
}
-func (s *EnvironmentService) acquireHealthLockInternal(envID string) mo.Option[func()] {
- if _, loaded := s.runningHealthChecks.LoadOrStore(envID, struct{}{}); loaded {
- return mo.None[func()]()
- }
- return mo.Some(func() { s.runningHealthChecks.Delete(envID) })
-}
-
// runHealthCheckInternal tests one environment's connection (updating its DB status)
// and, for online remotes, syncs registries and repositories to it.
func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) {
- release, ok := s.acquireHealthLockInternal(envID).Get()
- if !ok {
+ lease, admitted, err := s.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: envID})
+ if err != nil {
+ slog.ErrorContext(ctx, "environment health check admission failed", "environment_id", envID, "error", err)
+ return
+ }
+ if !admitted {
slog.WarnContext(ctx, "environment health check skipped; previous run still in progress", "environment_id", envID)
return
}
- defer release()
+ defer lease.Release()
status, err := s.TestConnection(ctx, envID, nil)
switch {
diff --git a/backend/internal/services/environment_service_test.go b/backend/internal/services/environment_service_test.go
index 16fc3656c..b441f2ec7 100644
--- a/backend/internal/services/environment_service_test.go
+++ b/backend/internal/services/environment_service_test.go
@@ -18,6 +18,7 @@ import (
"gorm.io/gorm"
"github.com/coder/websocket"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -28,8 +29,31 @@ import (
"github.com/getarcaneapp/arcane/types/v2/environment"
"github.com/getarcaneapp/arcane/types/v2/gitops"
"go.getarcane.app/sys/crypto"
+ "go.uber.org/fx/fxtest"
)
+func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), actorRuntime, "environment-test-admission", "overlap")
+ require.NoError(t, err)
+
+ key := actors.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: "environment-id"}
+ lease, admitted, err := gate.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.True(t, admitted)
+
+ service := &EnvironmentService{admissionGate: gate}
+ service.runHealthCheckInternal(t.Context(), "environment-id")
+ lease.Release()
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
func setupEnvironmentServiceTestDB(t *testing.T) *database.DB {
t.Helper()
@@ -149,7 +173,7 @@ func TestEnvironmentService_DeleteEnvironment_CascadesGitOpsSyncs(t *testing.T)
scheduler := &gitOpsSyncTestSchedulerInternal{}
svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
- svc.SetScheduler(ctx, scheduler)
+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))
require.NoError(t, svc.DeleteEnvironment(ctx, "env-delete-gitops", nil, nil))
diff --git a/backend/internal/services/federated_credential_service_test.go b/backend/internal/services/federated_credential_service_test.go
index 18bbc81f1..a73644279 100644
--- a/backend/internal/services/federated_credential_service_test.go
+++ b/backend/internal/services/federated_credential_service_test.go
@@ -133,7 +133,7 @@ func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTest
roleSvc := NewRoleService(db)
userSvc := NewUserService(db).WithRoleService(roleSvc)
sessionSvc := NewSessionService(db)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventSvc := NewEventService(db, &config.Config{}, nil)
authSvc := NewAuthService(userSvc, settingsSvc, eventSvc, sessionSvc, roleSvc, "test-federated-secret", &config.Config{
diff --git a/backend/internal/services/git_repository_service_test.go b/backend/internal/services/git_repository_service_test.go
index 0df985319..0b9fe7e5c 100644
--- a/backend/internal/services/git_repository_service_test.go
+++ b/backend/internal/services/git_repository_service_test.go
@@ -43,7 +43,7 @@ func setupGitRepositoryServiceTestInternal(t *testing.T) (*GitRepositoryService,
})
wrappedDB := &database.DB{DB: db}
- settingsService, err := NewSettingsService(context.Background(), wrappedDB)
+ settingsService, err := newSettingsServiceForTestInternal(t, context.Background(), wrappedDB)
require.NoError(t, err)
eventService := NewEventService(wrappedDB, &config.Config{}, nil)
diff --git a/backend/internal/services/gitops_sync_service.go b/backend/internal/services/gitops_sync_service.go
index 4f0b0433d..901e3a0e2 100644
--- a/backend/internal/services/gitops_sync_service.go
+++ b/backend/internal/services/gitops_sync_service.go
@@ -11,10 +11,10 @@ import (
"path/filepath"
"slices"
"strings"
- "sync"
"time"
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
@@ -27,22 +27,9 @@ import (
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/swarm"
"github.com/moby/moby/client"
- "github.com/samber/mo"
"gorm.io/gorm"
)
-// DynamicScheduler is the subset of the job scheduler used by services that
-// register per-entity jobs at runtime (GitOps syncs, environment health). It is a
-// consumer-side interface satisfied by *pkg/scheduler.JobScheduler; the scheduler
-// is injected post-construction via SetScheduler because it is created after the
-// service graph is built (and pkg/scheduler imports this package, so it cannot be
-// a wire input here).
-type DynamicScheduler interface {
- AddJob(ctx context.Context, job schedulertypes.Job) error
- RemoveJob(ctx context.Context, name string)
- HasJob(name string) bool
-}
-
type GitOpsSyncService struct {
db *database.DB
repoService *GitRepositoryService
@@ -52,13 +39,9 @@ type GitOpsSyncService struct {
settingsService *SettingsService
// scheduler and lifecycleCtx are injected post-construction via SetScheduler.
- scheduler DynamicScheduler
- lifecycleCtx context.Context
- // runningSyncs guards against overlapping PerformSync runs for the same sync.
- // The previous single global job serialized syncs implicitly; with independent
- // per-sync jobs (plus manual/webhook/startup triggers) a sync could otherwise
- // run concurrently with itself and race the clone/redeploy of one project.
- runningSyncs sync.Map
+ scheduler schedulertypes.DynamicScheduler
+ lifecycleCtx context.Context
+ admissionGate *actors.Gate[actors.AdmissionKey]
}
const defaultGitSyncTimeout = 5 * time.Minute
@@ -367,6 +350,7 @@ func NewGitOpsSyncService(db *database.DB, repoService *GitRepositoryService, pr
}
const gitOpsSyncJobPrefix = "gitops-sync:"
+const gitOpsSyncAdmissionScopeInternal = "gitops-sync"
func gitOpsSyncJobNameInternal(syncID string) string { return gitOpsSyncJobPrefix + syncID }
@@ -374,12 +358,17 @@ func gitOpsSyncJobNameInternal(syncID string) string { return gitOpsSyncJobPrefi
// called during bootstrap (after the service graph is built) before any per-sync
// jobs are registered. The lifecycle context is used for background sync kicks so
// they outlive the request/bootstrap goroutine that triggered them.
-func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler DynamicScheduler) { //nolint:contextcheck // background sync kicks must capture the app lifecycle context, not request contexts
+func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error { //nolint:contextcheck // background sync kicks must capture the app lifecycle context, not request contexts
+ if scheduler == nil || admissionGate == nil {
+ return errors.New("GitOps scheduler dependencies unavailable")
+ }
if ctx == nil {
ctx = context.Background()
}
s.lifecycleCtx = ctx
s.scheduler = scheduler
+ s.admissionGate = admissionGate
+ return nil
}
func (s *GitOpsSyncService) schedulerCtxInternal(ctx context.Context) context.Context {
@@ -495,15 +484,6 @@ func isGitOpsSyncOverdueInternal(sync *models.GitOpsSync) bool {
return time.Now().After(sync.LastSyncAt.Add(time.Duration(interval) * time.Minute))
}
-// acquireSyncLockInternal returns a release func when no sync is currently
-// running for the given id; otherwise the caller should skip.
-func (s *GitOpsSyncService) acquireSyncLockInternal(syncID string) mo.Option[func()] {
- if _, loaded := s.runningSyncs.LoadOrStore(syncID, struct{}{}); loaded {
- return mo.None[func()]()
- }
- return mo.Some(func() { s.runningSyncs.Delete(syncID) })
-}
-
func (s *GitOpsSyncService) getEnvironmentSyncLimits(ctx context.Context) (int, int64, int64) {
if s.settingsService == nil {
return defaultMaxSyncFiles, defaultMaxSyncTotalSize, defaultMaxSyncBinarySize
@@ -922,12 +902,15 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id st
func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id string, actor models.User) (*gitops.SyncResult, error) {
// Coalesce overlapping runs for the same sync (scheduled fire, startup/enable
// kick, manual trigger, webhook) so they don't race the clone/redeploy.
- release, ok := s.acquireSyncLockInternal(id).Get()
- if !ok {
+ lease, admitted, err := s.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: id})
+ if err != nil {
+ return nil, errors.WrapIf(err, "failed to admit GitOps sync")
+ }
+ if !admitted {
slog.InfoContext(ctx, "GitOps sync already in progress; skipping", "syncId", id)
return &gitops.SyncResult{Success: false, Message: "sync already in progress", SyncedAt: time.Now()}, nil
}
- defer release()
+ defer lease.Release()
syncCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
defer cancel()
diff --git a/backend/internal/services/gitops_sync_service_test.go b/backend/internal/services/gitops_sync_service_test.go
index 13a7653de..af547f27d 100644
--- a/backend/internal/services/gitops_sync_service_test.go
+++ b/backend/internal/services/gitops_sync_service_test.go
@@ -9,6 +9,7 @@ import (
"time"
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -19,6 +20,7 @@ import (
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -29,7 +31,7 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *dat
db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsDir := t.TempDir()
@@ -41,6 +43,31 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *dat
return NewGitOpsSyncService(db, nil, projectService, nil, eventService, settingsService), db, projectsDir
}
+func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), actorRuntime, "gitops-test-admission", "overlap")
+ require.NoError(t, err)
+
+ key := actors.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: "sync-id"}
+ lease, admitted, err := gate.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.True(t, admitted)
+
+ service := &GitOpsSyncService{admissionGate: gate}
+ result, err := service.PerformSync(t.Context(), "0", "sync-id", models.User{})
+ require.NoError(t, err)
+ require.False(t, result.Success)
+ require.Equal(t, "sync already in progress", result.Message)
+ lease.Release()
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
type gitOpsSyncTestSchedulerInternal struct {
added []string
removed []string
@@ -178,7 +205,7 @@ func TestGitOpsSyncService_RegisterAutoSyncJobsOnStartup_SkipsOrphans(t *testing
}).Error)
scheduler := &gitOpsSyncTestSchedulerInternal{}
- svc.SetScheduler(ctx, scheduler)
+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))
svc.RegisterAutoSyncJobsOnStartup(ctx)
require.Equal(t, []string{gitOpsSyncJobNameInternal("sync-live")}, scheduler.added)
@@ -216,7 +243,7 @@ func TestGitOpsSyncService_DeleteSync_SucceedsWhenEnvironmentMismatched(t *testi
ctx := context.Background()
svc, db, _ := setupGitOpsSyncDirectoryTestService(t)
scheduler := &gitOpsSyncTestSchedulerInternal{}
- svc.SetScheduler(ctx, scheduler)
+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))
sync := &models.GitOpsSync{
BaseModel: models.BaseModel{ID: "sync-env-mismatch"},
@@ -273,7 +300,7 @@ func TestGitOpsSyncService_RunScheduledSync_UnregistersMissingSync(t *testing.T)
ctx := context.Background()
svc, _, _ := setupGitOpsSyncDirectoryTestService(t)
scheduler := &gitOpsSyncTestSchedulerInternal{}
- svc.SetScheduler(ctx, scheduler)
+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newAdmissionGateForTestInternal(t)))
svc.runScheduledSyncInternal(ctx, "0", "ghost-sync")
@@ -355,7 +382,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision(
func TestGitOpsSyncService_GetOrCreateProject_RefusesDuplicateOnNameCollision(t *testing.T) {
ctx := context.Background()
svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)
- svc.SetScheduler(ctx, &gitOpsSyncTestSchedulerInternal{})
+ require.NoError(t, svc.SetScheduler(ctx, &gitOpsSyncTestSchedulerInternal{}, newAdmissionGateForTestInternal(t)))
require.NoError(t, os.MkdirAll(filepath.Join(projectsDir, "Dozzle"), 0o755))
@@ -1117,7 +1144,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_FailsWhenBoundProjectMissing(t *
ctx := context.Background()
svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)
testScheduler := &gitOpsSyncTestSchedulerInternal{}
- svc.SetScheduler(ctx, testScheduler)
+ require.NoError(t, svc.SetScheduler(ctx, testScheduler, newAdmissionGateForTestInternal(t)))
missingProjectID := "missing-project"
sync := &models.GitOpsSync{
@@ -1172,7 +1199,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_DisablesAutoSyncWhenBoundProject
ctx := context.Background()
svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)
testScheduler := &gitOpsSyncTestSchedulerInternal{}
- svc.SetScheduler(ctx, testScheduler)
+ require.NoError(t, svc.SetScheduler(ctx, testScheduler, newAdmissionGateForTestInternal(t)))
for _, dirName := range []string{"Radarr-3", "Radarr-30"} {
projectPath := filepath.Join(projectsDir, dirName)
@@ -1227,7 +1254,7 @@ func TestGitOpsSyncService_GetOrCreateProjectInternal_FailsWhenBoundProjectMissi
ctx := context.Background()
svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)
testScheduler := &gitOpsSyncTestSchedulerInternal{}
- svc.SetScheduler(ctx, testScheduler)
+ require.NoError(t, svc.SetScheduler(ctx, testScheduler, newAdmissionGateForTestInternal(t)))
missingProjectID := "missing-project"
sync := &models.GitOpsSync{
@@ -1286,7 +1313,7 @@ func TestEnvContentChangedInternal(t *testing.T) {
func TestGitOpsSyncService_GetEnvironmentSyncLimits(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 123))
@@ -1309,7 +1336,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {
t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "")
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 200))
@@ -1364,7 +1391,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {
t.Setenv("GIT_SYNC_MAX_FILES", "10000")
t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "1024")
t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "12")
- settingsSvcEnv, svcErr := NewSettingsService(ctx, db)
+ settingsSvcEnv, svcErr := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, svcErr)
svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}
@@ -1385,7 +1412,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {
t.Setenv("GIT_SYNC_MAX_FILES", "0")
t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "0")
t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "0")
- settingsSvcEnv, svcErr := NewSettingsService(ctx, db)
+ settingsSvcEnv, svcErr := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, svcErr)
svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}
diff --git a/backend/internal/services/job_service.go b/backend/internal/services/job_service.go
index ce4f90dcb..5a3512db7 100644
--- a/backend/internal/services/job_service.go
+++ b/backend/internal/services/job_service.go
@@ -2,7 +2,6 @@ package services
import (
"context"
- stderrors "errors"
"log/slog"
"sort"
"time"
@@ -10,22 +9,13 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/models"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
"github.com/getarcaneapp/arcane/types/v2/meta"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/robfig/cron/v3"
- "github.com/samber/mo"
- "gorm.io/gorm"
)
-type JobRunner interface {
- GetJob(jobID string) mo.Option[schedulertypes.Job]
- GetJobRuntimeState(jobID string) mo.Option[schedulertypes.JobRuntimeState]
- RescheduleJob(ctx context.Context, job schedulertypes.Job) error
- RunBusWatcherNow(ctx context.Context, watcherID string) error
-}
-
// JobService manages configuration for background job schedules.
//
// Intervals are persisted in the existing settings table as individual keys.
@@ -38,7 +28,7 @@ type JobService struct {
db *database.DB
settings *SettingsService
cfg *config.Config
- scheduler JobRunner
+ scheduler schedulertypes.JobController
lifecycleCtx context.Context
location *time.Location // Timezone for cron schedule calculations
@@ -59,7 +49,7 @@ func NewJobService(db *database.DB, settings *SettingsService, cfg *config.Confi
}
}
-func (s *JobService) SetScheduler(ctx context.Context, scheduler JobRunner) { //nolint:contextcheck // scheduler jobs must capture the app lifecycle context, not request contexts
+func (s *JobService) SetScheduler(ctx context.Context, scheduler schedulertypes.JobController) { //nolint:contextcheck // scheduler jobs must capture the app lifecycle context, not request contexts
if ctx == nil {
ctx = context.Background()
}
@@ -121,49 +111,33 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule
}
}
- changed := false
changedKeys := make([]string, 0, len(fields))
previousValues := make(map[string]string, len(fields))
- upsert := func(tx *gorm.DB, key string, v *string, currentVal string) error {
- if v == nil {
- return nil
- }
- if *v == currentVal {
- return nil
+ valuesToUpdate := make([]libarcane.SettingUpdate, 0, len(fields))
+ for _, field := range fields {
+ key := field.key
+ value := field.update
+ currentValue := field.current
+ if value == nil || *value == currentValue {
+ continue
}
- changed = true
changedKeys = append(changedKeys, key)
- previousValues[key] = currentVal
- return tx.Save(&models.SettingVariable{Key: key, Value: *v}).Error
+ previousValues[key] = currentValue
+ valuesToUpdate = append(valuesToUpdate, libarcane.SettingUpdate{Key: key, Value: *value})
}
- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- for _, field := range fields {
- if err := upsert(tx, field.key, field.update, field.current); err != nil {
- return err
- }
- }
- return nil
- })
- if err != nil {
+ if len(valuesToUpdate) == 0 {
+ return s.GetJobSchedules(ctx), nil
+ }
+ if err := s.settings.UpdateSettingValues(ctx, valuesToUpdate); err != nil {
return jobschedule.Config{}, errors.WrapIf(err, "failed to update job schedules")
}
- // Refresh settings cache so jobs reading from SettingsService see new values.
- if changed {
- if err := s.settings.LoadDatabaseSettings(ctx); err != nil {
- restoreErr := s.restoreJobSchedulesInternal(ctx, previousValues, changedKeys)
- return jobschedule.Config{}, stderrors.Join(errors.
- WrapIf(err, "failed to reload settings after job schedule update"), restoreErr,
- )
- }
-
- if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {
- restoreErr := s.restoreJobSchedulesInternal(ctx, previousValues, changedKeys)
- return jobschedule.Config{}, stderrors.Join(errors.
- WrapIf(err, "failed to apply job schedule update"), restoreErr,
- )
- }
+ if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {
+ restoreErr := s.restoreJobSchedulesInternal(ctx, previousValues, changedKeys)
+ return jobschedule.Config{}, errors.Combine(errors.
+ WrapIf(err, "failed to apply job schedule update"), restoreErr,
+ )
}
return s.GetJobSchedules(ctx), nil
@@ -196,7 +170,7 @@ func (s *JobService) RescheduleJobsForSettingKeys(ctx context.Context, changedKe
}
}
- return stderrors.Join(rescheduleErrors...)
+ return errors.Combine(rescheduleErrors...)
}
// jobRescheduleContextInternal prefers the app lifecycle context so cron jobs
@@ -225,13 +199,13 @@ func (s *JobService) rescheduleAffectedJobInternal(ctx context.Context, jobID st
// Continuous bus watchers have no cron entry to reschedule; notify the
// watcher so it re-reads its poll schedule instead.
if jobMeta.IsContinuous {
- if jobID == "image-polling" && s.settings != nil && s.settings.OnImagePollingSettingsChanged != nil {
- s.settings.OnImagePollingSettingsChanged(s.jobRescheduleContextInternal(ctx))
+ if jobID == "image-polling" && s.settings != nil {
+ return s.settings.NotifySettingsChanges(s.jobRescheduleContextInternal(ctx), "pollingInterval")
}
return nil
}
- job, ok := s.scheduler.GetJob(jobID).Get()
+ job, ok := s.scheduler.GetJob(jobID)
if !ok {
return errors.Errorf("job %s not found in scheduler", jobID)
}
@@ -242,7 +216,7 @@ func (s *JobService) rescheduleAffectedJobInternal(ctx context.Context, jobID st
return errors.WrapIff(err, "reschedule job %s", jobID)
}
- runtimeState, ok := s.scheduler.GetJobRuntimeState(jobID).Get()
+ runtimeState, ok := s.scheduler.GetJobRuntimeState(jobID)
if !ok {
return errors.Errorf("job %s has no runtime scheduler state", jobID)
}
@@ -259,19 +233,12 @@ func (s *JobService) restoreJobSchedulesInternal(ctx context.Context, previousVa
return nil
}
- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- for key, value := range previousValues {
- if err := tx.Save(&models.SettingVariable{Key: key, Value: value}).Error; err != nil {
- return err
- }
- }
- return nil
- }); err != nil {
- return errors.WrapIf(err, "failed to restore previous job schedules")
+ updates := make([]libarcane.SettingUpdate, 0, len(previousValues))
+ for key, value := range previousValues {
+ updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})
}
-
- if err := s.settings.LoadDatabaseSettings(ctx); err != nil {
- return errors.WrapIf(err, "failed to reload restored job schedules")
+ if err := s.settings.UpdateSettingValues(ctx, updates); err != nil {
+ return errors.WrapIf(err, "failed to restore previous job schedules")
}
if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {
@@ -310,7 +277,7 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponse
prerequisites := s.evaluatePrerequisitesInternal(ctx, jobMeta)
if s.scheduler != nil && jobMeta.SettingsKey != "" && jobMeta.ID != "environment-health" {
- if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID).Get(); ok && runtimeState.Schedule != "" {
+ if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok && runtimeState.Schedule != "" {
schedule = runtimeState.Schedule
nextRun = runtimeState.NextRun
}
@@ -376,7 +343,7 @@ func (s *JobService) getRunnableJobInternal(jobID string) (schedulertypes.Job, e
return nil, errors.Errorf("job %s is manager-only and cannot run in agent mode", jobID)
}
- job, ok := s.scheduler.GetJob(jobID).Get()
+ job, ok := s.scheduler.GetJob(jobID)
if !ok {
return nil, errors.Errorf("job %s not found in scheduler", jobID)
}
diff --git a/backend/internal/services/job_service_test.go b/backend/internal/services/job_service_test.go
index ef4ba3701..589706064 100644
--- a/backend/internal/services/job_service_test.go
+++ b/backend/internal/services/job_service_test.go
@@ -10,7 +10,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/samber/mo"
"github.com/stretchr/testify/require"
)
@@ -18,7 +17,7 @@ func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testin
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
@@ -32,7 +31,7 @@ func TestJobService_ListJobs_AnalyticsHeartbeatIsManagedInternally(t *testing.T)
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
@@ -51,7 +50,7 @@ func TestJobService_ListJobs_IncludesDisabledAutoHealJob(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", false))
@@ -68,7 +67,7 @@ func TestJobService_ListJobs_IncludesDockerClientRefreshJob(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
@@ -87,7 +86,7 @@ func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
nextRun := time.Date(2026, time.July, 10, 8, 0, 0, 0, time.UTC)
@@ -112,7 +111,7 @@ func TestJobService_ListJobs_ImageUpdateWatcherIsContinuousAndRespectsEnabled(t
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "pollingEnabled", false))
@@ -134,7 +133,7 @@ func TestJobService_UpdateJobSchedules_ReschedulesChangedJob(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
@@ -153,7 +152,7 @@ func TestJobService_UpdateJobSchedules_DeprecatedPollingIntervalDoesNotReschedul
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
@@ -172,7 +171,7 @@ func TestJobService_UpdateJobSchedules_UsesLifecycleContextForReschedule(t *test
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
type lifecycleContextKey struct{}
@@ -199,7 +198,7 @@ func TestJobService_UpdateJobSchedules_RejectsInvalidCronWithoutChangingSetting(
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))
@@ -220,7 +219,7 @@ func TestJobService_UpdateJobSchedules_UnchangedScheduleDoesNotReschedule(t *tes
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
@@ -235,7 +234,7 @@ func TestJobService_UpdateJobSchedules_RestoresPreviousScheduleWhenRescheduleFai
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))
@@ -260,7 +259,7 @@ func TestJobService_UpdateJobSchedules_SkipsManagerOnlyJobsInAgentMode(t *testin
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{AgentMode: true})
@@ -279,7 +278,7 @@ func TestJobService_UpdateJobSchedules_DelegatesEnvironmentHealthReschedule(t *t
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
jobSvc := NewJobService(db, settingsSvc, &config.Config{})
@@ -347,14 +346,14 @@ func newFakeJobSchedulerInternal(jobIDs ...string) *fakeJobSchedulerInternal {
}
}
-func (s *fakeJobSchedulerInternal) GetJob(jobID string) mo.Option[schedulertypes.Job] {
+func (s *fakeJobSchedulerInternal) GetJob(jobID string) (schedulertypes.Job, bool) {
job, ok := s.jobs[jobID]
- return mo.TupleToOption(job, ok)
+ return job, ok
}
-func (s *fakeJobSchedulerInternal) GetJobRuntimeState(jobID string) mo.Option[schedulertypes.JobRuntimeState] {
+func (s *fakeJobSchedulerInternal) GetJobRuntimeState(jobID string) (schedulertypes.JobRuntimeState, bool) {
state, ok := s.runtimeStates[jobID]
- return mo.TupleToOption(state, ok)
+ return state, ok
}
func (s *fakeJobSchedulerInternal) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {
diff --git a/backend/internal/services/lifecycle_service_test.go b/backend/internal/services/lifecycle_service_test.go
index 704006c0f..aaa10fffe 100644
--- a/backend/internal/services/lifecycle_service_test.go
+++ b/backend/internal/services/lifecycle_service_test.go
@@ -32,7 +32,7 @@ func setupLifecycleTestDB(t *testing.T) *database.DB {
func newLifecycleTestService(t *testing.T, db *database.DB) (*LifecycleService, *SettingsService) {
t.Helper()
- settings, err := NewSettingsService(context.Background(), db)
+ settings, err := newSettingsServiceForTestInternal(t, context.Background(), db)
require.NoError(t, err)
events := NewEventService(db, nil, nil)
return NewLifecycleService(db, settings, events, nil), settings
diff --git a/backend/internal/services/oidc_service_test.go b/backend/internal/services/oidc_service_test.go
index 1f2129f6d..277140c64 100644
--- a/backend/internal/services/oidc_service_test.go
+++ b/backend/internal/services/oidc_service_test.go
@@ -61,7 +61,7 @@ func TestGetMobileRedirectAllowlistTrimsWhitespace(t *testing.T) {
func TestGetMobileRedirectAllowlistUsesSettings(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
if err != nil {
t.Fatalf("NewSettingsService: %v", err)
}
diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.go
index ff895e1ce..b179c6d4a 100644
--- a/backend/internal/services/project_service_test.go
+++ b/backend/internal/services/project_service_test.go
@@ -70,7 +70,7 @@ func setupProjectTestDB(t *testing.T) *database.DB {
func TestProjectService_RefreshProjectImageRefs_PersistsBuildMetadata(t *testing.T) {
ctx := context.Background()
db := setupProjectTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectPath := t.TempDir()
@@ -106,7 +106,7 @@ func TestProjectService_RefreshProjectImageRefs_PersistsBuildMetadata(t *testing
func TestProjectService_BackfillProjectImageRefs_RetriesOnlyMissingMetadata(t *testing.T) {
ctx := context.Background()
db := setupProjectTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
validPath := t.TempDir()
@@ -167,7 +167,7 @@ func setupProjectDestroyTestServiceInternal(t *testing.T) (*ProjectService, *dat
ctx := context.Background()
db := setupProjectTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsDir := t.TempDir()
@@ -291,7 +291,7 @@ func TestProjectService_GetProjectFromDatabaseByID(t *testing.T) {
ctx := context.Background()
// Setup dependencies
- settingsService, _ := NewSettingsService(ctx, db)
+ settingsService, _ := newSettingsServiceForTestInternal(t, ctx, db)
svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
// Create test project
@@ -613,7 +613,7 @@ func TestProjectService_PullProjectImages_UpdatesCurrentImageRecordAfterPull(t *
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -711,7 +711,7 @@ func TestProjectService_EnsureImagesPresent_UpdatesCurrentImageRecordAfterPull(t
ctx := context.Background()
db := setupProjectTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
imageRef := "registry.example.com/team/api:2.0.0"
@@ -763,7 +763,7 @@ func TestProjectService_PullImageForService_UpdatesCurrentImageRecordAfterPull(t
ctx := context.Background()
db := setupProjectTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
imageRef := "registry.example.com/team/worker:3.1.4"
@@ -812,7 +812,7 @@ func TestProjectService_ComposePullSelectedServicesInternal_ReconcilesOnlyOnSucc
ctx := context.Background()
db := setupProjectTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
privateImageRef := "registry.example.com/team/app:9.9.9"
@@ -932,7 +932,7 @@ func TestProjectService_ComposePullSelectedServicesInternal_LeavesRecordsWhenPul
ctx := context.Background()
db := setupProjectTestDB(t)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
imageRef := "registry.example.com/team/app:9.9.9"
@@ -991,7 +991,7 @@ func TestProjectService_UpdateProjectServicesHardFailsWhenPullFailsInternal(t *t
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1061,7 +1061,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1151,7 +1151,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenVolumeMigrationPreparationF
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1221,7 +1221,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_AppliesVolumeMigrati
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -1268,7 +1268,7 @@ func TestProjectService_PrepareProjectRenameVolumeMigrationForUpdate_UsesCompose
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1383,7 +1383,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_RollsBackVolumeMigra
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -1432,7 +1432,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_SucceedsCommittedRen
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -1482,7 +1482,7 @@ func TestProjectService_UpdateProject_ClearsJournalForNonRenameWhenRecoveryDocke
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1548,7 +1548,7 @@ func TestProjectService_UpdateProject_AllowsRenameAfterJournalRecoveryDockerUnav
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1611,7 +1611,7 @@ func TestProjectService_UpdateProject_RenamesDirectoryWhenNameChanges(t *testing
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -1660,7 +1660,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenTargetDirectoryExists(t *te
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -1707,7 +1707,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenProjectRunning(t *testing.T
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -1751,7 +1751,7 @@ func TestProjectService_UpdateProject_RenameRejectsStaleStoppedWhenRuntimeIsRunn
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1809,7 +1809,7 @@ func TestProjectService_UpdateProject_RenameResolvesUnknownStoppedStatusBeforeVo
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1867,7 +1867,7 @@ func TestProjectService_UpdateProject_RenameRejectsUnknownWhenRuntimeIsRunning(t
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -1931,7 +1931,7 @@ func TestProjectService_UpdateProject_ValidatesComposeUsingExistingProjectName(t
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -1974,7 +1974,7 @@ func TestProjectService_UpdateProject_AllowsMissingEnvFileDuringComposeValidatio
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2019,7 +2019,7 @@ func TestProjectService_UpdateProject_AllowsMissingLocalIncludeDuringComposeVali
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2084,7 +2084,7 @@ func TestProjectService_UpdateProject_RejectsMissingExternalIncludeDuringCompose
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2129,7 +2129,7 @@ func TestProjectService_CreateProject_RejectsExternalInclude(t *testing.T) {
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2165,7 +2165,7 @@ func TestProjectService_CreateProject_RejectsArrayPathInclude(t *testing.T) {
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2203,7 +2203,7 @@ func TestProjectService_CreateProject_WritesStagedProjectFiles(t *testing.T) {
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2257,7 +2257,7 @@ func TestProjectService_GetProjectDetails_UsesFileTreeMaxDepthForProjectFiles(t
t.Setenv("PROJECT_SCAN_MAX_DEPTH", "3")
t.Setenv("PROJECT_FILE_TREE_MAX_DEPTH", "8")
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2300,7 +2300,7 @@ func TestProjectService_UpdateProject_AppliesStagedProjectFileChanges(t *testing
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2343,7 +2343,7 @@ func TestProjectService_UpdateProject_RejectsStaleProjectFileRevision(t *testing
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2380,7 +2380,7 @@ func TestProjectService_UpdateProject_RejectsStaleDeepProjectFileRevision(t *tes
t.Setenv("PROJECT_SCAN_MAX_DEPTH", "3")
t.Setenv("PROJECT_FILE_TREE_MAX_DEPTH", "8")
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2419,7 +2419,7 @@ func TestProjectService_GetProjectFileContent_RejectsExternalInclude(t *testing.
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2460,7 +2460,7 @@ func TestProjectService_GetProjectFileContent_RejectsSymlinkInclude(t *testing.T
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2504,7 +2504,7 @@ func TestProjectService_GetProjectFileContent_RejectsIntermediateSymlinkInclude(
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2548,7 +2548,7 @@ func TestProjectService_GetProjectFileContent_RejectsIntermediateSymlinkProjectF
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2585,7 +2585,7 @@ func TestProjectService_UpdateProject_UsesExistingEnvFileDuringComposeValidation
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2635,7 +2635,7 @@ func newProjectServiceForOverrideTestInternal(t *testing.T, dirName, baseCompose
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2746,7 +2746,7 @@ func TestProjectService_UpdateProject_UsesProvidedEnvContentDuringComposeValidat
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2793,7 +2793,7 @@ func TestProjectService_UpdateProject_ReturnsEnvParseErrorDuringComposeValidatio
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2838,7 +2838,7 @@ func TestProjectService_UpdateProject_UsesGlobalEnvDuringComposeValidation(t *te
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2887,7 +2887,7 @@ func TestProjectService_UpdateProject_DoesNotResolveHostEnvThroughGlobalEnvDurin
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
t.Setenv("HOST_ONLY_PATH", "/host/secret")
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2931,7 +2931,7 @@ func TestProjectService_UpdateProject_DerivesProjectOverrideEnvWhenGitSourceExis
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -2978,7 +2978,7 @@ func TestProjectService_UpdateProject_UnchangedGitEnvLeavesFilesUntouched(t *tes
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3077,7 +3077,7 @@ func TestProjectService_UpdateProject_DeletingGitBackedKeyFallsBackToGit(t *test
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3127,7 +3127,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_MigratesDirectEnvIntoProjectOve
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3178,7 +3178,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_PreservesGitEnvSyntax(t *testin
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3239,7 +3239,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_NormalizesStaleCopiedGitOverrid
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3287,7 +3287,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_RemovesLegacyDeletedGitMasks(t
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3336,7 +3336,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_RemovesGitEnvSource(t *testing.
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3380,7 +3380,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_WritesAndRemovesComposeOverride
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3431,7 +3431,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_UsesGlobalEnvDuringComposeValid
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3482,7 +3482,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_TolerantOfUndefinedComposeVar(t
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3530,7 +3530,7 @@ func TestProjectService_PersistGitSyncEnvFiles_UsesPreparedState(t *testing.T) {
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3568,7 +3568,7 @@ func TestProjectService_GetProjectDetails_ReturnsEffectiveEnvContent(t *testing.
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
@@ -3694,7 +3694,7 @@ func TestProjectService_GetProjectDetails_IncludesUpdateInfo(t *testing.T) {
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -3742,7 +3742,7 @@ func TestProjectService_GetProjectDetails_RefreshesRuntimeStatusWithoutRuntimeSe
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -3815,7 +3815,7 @@ func TestProjectService_GetProjectDetails_PopulatesRuntimeServicesFromComposePs(
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -3869,7 +3869,7 @@ func TestProjectService_ListProjects_FiltersByUpdateStatus(t *testing.T) {
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -4081,7 +4081,7 @@ func TestProjectService_ListProjects_FiltersArchivedProjects(t *testing.T) {
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
@@ -4143,7 +4143,7 @@ func TestProjectService_ArchiveProject_RequiresStoppedProject(t *testing.T) {
ctx := context.Background()
projectsRoot := t.TempDir()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
@@ -4173,7 +4173,7 @@ func TestProjectService_ArchiveProject_TogglesArchiveFlag(t *testing.T) {
ctx := context.Background()
projectsRoot := t.TempDir()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
@@ -4310,7 +4310,7 @@ func TestProjectService_ListProjects_WithDerivedStatusFilter_AllowsAllPageSizeSe
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4542,7 +4542,7 @@ func TestProjectService_PrepareServiceBuildRequest_GeneratedImageProviderGuardra
func TestProjectService_DeployProject_StopsOnBuildPreparationError(t *testing.T) {
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4580,7 +4580,7 @@ func TestProjectService_DeployProject_StopsOnBuildPreparationError(t *testing.T)
func TestProjectService_DeployProject_BuildsGeneratedImageWithoutPull(t *testing.T) {
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4621,7 +4621,7 @@ func TestProjectService_SyncProjectsFromFileSystem_IgnoresSymlinkedProjectDirsWh
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4647,7 +4647,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DetectsSymlinkedProjectDirsWh
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4673,7 +4673,7 @@ func TestProjectService_CountProjectFolders_RespectsFollowProjectSymlinks(t *tes
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4701,7 +4701,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversNestedProjectsAndRel
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4734,7 +4734,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RespectsConfiguredScanMaxDept
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4758,7 +4758,7 @@ func TestProjectService_ListProjects_LoadsProjectIconFromGlobalEnvInIncludedMeta
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4809,7 +4809,7 @@ func TestProjectService_CountProjectFolders_RecursivelyCountsNestedProjects(t *t
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4830,7 +4830,7 @@ func TestProjectService_CountProjectFolders_RespectsConfiguredScanMaxDepth(t *te
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4851,7 +4851,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(t
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4882,7 +4882,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PrunesLeakedScratchRow(t *tes
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4912,7 +4912,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectsWhenDirector
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// An existing-but-empty projects directory simulates a mis-mapped or unmounted
@@ -4947,7 +4947,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectWithAmbiguous
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -4983,7 +4983,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesProjectsBeyondReducedS
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5029,7 +5029,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesDBRecordsWhenDirecto
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5078,7 +5078,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversReadableProjectsDesp
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5129,7 +5129,7 @@ func TestProjectService_SyncProjectsFromFileSystem_AllowsDuplicateLeafDirectorie
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5157,7 +5157,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DetectsNestedSymlinkedProject
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5188,7 +5188,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesSymlinkedProjectsWhenD
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5222,7 +5222,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RefreshesServiceCountOnCompos
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5249,7 +5249,7 @@ func TestProjectService_SyncProjectsFromFileSystem_AlignsNameToEffectiveComposeN
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5278,7 +5278,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesValidCustomNameWitho
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5311,7 +5311,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesGitOpsProjectWithCus
ctx := context.Background()
require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5361,7 +5361,7 @@ func TestProjectService_GetProjectDetails_UsesGitOpsCustomComposeFilename(t *tes
ctx := context.Background()
require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5414,7 +5414,7 @@ func TestProjectService_UpdateProject_WritesThroughSymlinkedProjectPath(t *testi
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5463,7 +5463,7 @@ func TestProjectService_UpdateProject_WritesThroughExternalEnvSymlink(t *testing
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5521,7 +5521,7 @@ func TestProjectService_UpdateProject_RestoresExternalEnvSymlinkTargetWhenProjec
db := setupProjectTestDB(t)
ctx := context.Background()
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
projectsRoot := t.TempDir()
@@ -5690,7 +5690,7 @@ func TestProjectService_UpdateProject_RenameRollsBackWhenFileRevisionIsStale(t *
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
@@ -7492,7 +7492,7 @@ func TestProjectService_UpdateProject_FileChangeFailureRollsBackScoped(t *testin
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
diff --git a/backend/internal/services/project_service_unix_test.go b/backend/internal/services/project_service_unix_test.go
index ae53da9c9..a53d883f0 100644
--- a/backend/internal/services/project_service_unix_test.go
+++ b/backend/internal/services/project_service_unix_test.go
@@ -29,7 +29,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_TolerantOfPermissionLockedEnv(t
projectsDir := t.TempDir()
t.Setenv("PROJECTS_DIRECTORY", projectsDir)
- settingsService, err := NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
eventService := NewEventService(db, nil, nil)
diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.go
index 7eea62b58..01c3ea87d 100644
--- a/backend/internal/services/settings_service.go
+++ b/backend/internal/services/settings_service.go
@@ -21,6 +21,7 @@ import (
"github.com/samber/mo"
"gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
@@ -35,15 +36,15 @@ type SettingsService struct {
db *database.DB
config atomic.Pointer[models.Settings]
envOverrides []settingsEnvOverride
+ writes *actors.Executor
+ effects *actors.Executor
+ lifecycleCtx context.Context
+ changes *actors.Signal[[]libarcane.SettingUpdate]
+}
- OnImagePollingSettingsChanged func(ctx context.Context)
- OnAutoUpdateSettingsChanged func(ctx context.Context)
- OnProjectsDirectoryChanged func(ctx context.Context)
- OnTemplatesDirectoryChanged func(ctx context.Context)
- OnScheduledPruneSettingsChanged func(ctx context.Context)
- OnVulnerabilityScanSettingsChanged func(ctx context.Context)
- OnAutoHealSettingsChanged func(ctx context.Context)
- OnTimeoutSettingsChanged func(ctx context.Context, timeoutSettings []libarcane.SettingUpdate)
+type settingsUpdateResultInternal struct {
+ settings []models.SettingVariable
+ changes []libarcane.SettingUpdate
}
type settingsEnvOverride struct {
@@ -53,9 +54,22 @@ type settingsEnvOverride struct {
value string
}
-func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService, error) {
+func NewSettingsService(ctx context.Context, db *database.DB, writes, effects *actors.Executor) (*SettingsService, error) {
+ if ctx == nil {
+ return nil, errors.New("settings lifecycle context unavailable")
+ }
+ if writes == nil {
+ return nil, errors.New("settings write executor unavailable")
+ }
+ if effects == nil {
+ return nil, errors.New("settings effects executor unavailable")
+ }
svc := &SettingsService{
- db: db,
+ db: db,
+ writes: writes,
+ effects: effects,
+ lifecycleCtx: ctx,
+ changes: actors.NewSignal[[]libarcane.SettingUpdate](),
}
svc.envOverrides = resolveSettingsEnvOverridesInternal()
if len(svc.envOverrides) > 0 {
@@ -75,6 +89,58 @@ func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService,
return svc, nil
}
+// SubscribeSettingsChanges invokes callback once when any subscribed key is
+// present in a successful update. The callback receives only matching values.
+// Callbacks run serially on the settings-effects actor, outside the write actor.
+func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func() {
+ if callback == nil || len(keys) == 0 {
+ return func() {}
+ }
+ subscribed := make(map[string]struct{}, len(keys))
+ for _, key := range keys {
+ subscribed[key] = struct{}{}
+ }
+ return s.changes.Subscribe(func(updates []libarcane.SettingUpdate) {
+ matching := make([]libarcane.SettingUpdate, 0, len(updates))
+ for _, update := range updates {
+ if _, ok := subscribed[update.Key]; ok {
+ matching = append(matching, update)
+ }
+ }
+ if len(matching) > 0 {
+ _, err := actors.Submit(context.WithoutCancel(s.lifecycleCtx), s.effects, "apply settings change effects", func(context.Context) (actors.NoPayload, error) {
+ callback(matching)
+ return actors.NoPayload{}, nil
+ }, nil)
+ if err != nil && s.lifecycleCtx.Err() == nil {
+ slog.ErrorContext(s.lifecycleCtx, "Failed to queue settings change effects", "error", err)
+ }
+ }
+ })
+}
+
+// NotifySettingsChanges publishes current values for keys through the settings
+// actor. It is used when another settings-table workflow performs persistence.
+func (s *SettingsService) NotifySettingsChanges(ctx context.Context, keys ...string) error {
+ _, err := actors.Execute(ctx, s.writes, "notify settings changes", func(context.Context) ([]libarcane.SettingUpdate, error) {
+ updates := make([]libarcane.SettingUpdate, 0, len(keys))
+ cfg := s.GetSettingsConfig()
+ for _, key := range keys {
+ value, _, _, err := cfg.FieldByKey(key)
+ if err != nil {
+ return nil, errors.WrapIff(err, "load changed setting '%s'", key)
+ }
+ updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})
+ }
+ return updates, nil
+ }, func(updates []libarcane.SettingUpdate, err error) {
+ if err == nil && len(updates) > 0 {
+ s.publishSettingsChangesInternal(updates)
+ }
+ })
+ return err
+}
+
func (s *SettingsService) GetSettingsConfig() *models.Settings {
v := s.config.Load()
if v == nil {
@@ -387,11 +453,32 @@ func (s *SettingsService) getEffectiveSettingsConfigInternal(ctx context.Context
}
func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string) error {
- if err := s.updateSettingValueNoRefreshInternal(ctx, key, value); err != nil {
- return err
+ if err := libarcane.ValidateCronSetting(key, value); err != nil {
+ return errors.WrapIff(err, "invalid cron expression for %s", key)
}
+ _, err := actors.Execute(ctx, s.writes, "update setting", func(writeCtx context.Context) (actors.NoPayload, error) {
+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, key, value); err != nil {
+ return actors.NoPayload{}, err
+ }
+ return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))
+ }, nil)
+ return err
+}
- return s.refreshSettingsCacheInternal(ctx)
+// UpdateSettingValues persists a group of internal setting values atomically
+// and publishes the refreshed snapshot before returning.
+func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []libarcane.SettingUpdate) error {
+ values := make([]models.SettingVariable, 0, len(updates))
+ for _, update := range updates {
+ values = append(values, models.SettingVariable{Key: update.Key, Value: update.Value})
+ }
+ _, err := actors.Execute(ctx, s.writes, "update setting values", func(writeCtx context.Context) (actors.NoPayload, error) {
+ if err := s.persistSettings(writeCtx, values); err != nil {
+ return actors.NoPayload{}, err
+ }
+ return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))
+ }, nil)
+ return err
}
func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Context, key, value string) error {
@@ -404,7 +491,38 @@ func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Contex
})
}
+// UpdateSettings publishes the refreshed snapshot before returning. Each
+// subscriber's effects remain ordered and may finish after this method returns.
func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.Update) ([]models.SettingVariable, error) {
+ result, err := actors.Execute(ctx, s.writes, "update settings", func(writeCtx context.Context) (settingsUpdateResultInternal, error) {
+ return s.updateSettingsInternal(writeCtx, updates)
+ }, func(result settingsUpdateResultInternal, err error) {
+ if err == nil && len(result.changes) > 0 {
+ s.publishSettingsChangesInternal(result.changes)
+ }
+ })
+ return result.settings, err
+}
+
+func (s *SettingsService) publishSettingsChangesInternal(updates []libarcane.SettingUpdate) {
+ safe := make([]libarcane.SettingUpdate, 0, len(updates))
+ cfg := s.GetSettingsConfig()
+ for _, update := range updates {
+ _, _, sensitive, err := cfg.FieldByKey(update.Key)
+ if err != nil {
+ slog.WarnContext(s.lifecycleCtx, "Skipping settings change notification for unknown key", "key", update.Key, "error", err)
+ continue
+ }
+ if !sensitive {
+ safe = append(safe, update)
+ }
+ }
+ if len(safe) > 0 {
+ s.changes.Publish(safe)
+ }
+}
+
+func (s *SettingsService) updateSettingsInternal(ctx context.Context, updates settings.Update) (settingsUpdateResultInternal, error) {
defaultCfg := s.getDefaultSettings()
cfg := s.GetSettingsConfig().Clone()
trivyServerTokenUpdated := updates.TrivyServerToken != nil && *updates.TrivyServerToken != "" && !s.isEnvOverrideActiveInternal("trivyServerToken")
@@ -424,7 +542,7 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
map[string]bool{"oidcClientSecret": cfg.OidcClientSecret.Value != ""},
map[string]bool{"oidcClientSecret": updates.OidcClientSecret != nil && *updates.OidcClientSecret != ""},
); err != nil {
- return nil, err
+ return settingsUpdateResultInternal{}, err
}
if err := validation.ValidateCredentialTargetChange(
@@ -435,12 +553,12 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
map[string]bool{"trivyServerToken": cfg.TrivyServerToken.Value != ""},
map[string]bool{"trivyServerToken": trivyServerTokenUpdated},
); err != nil {
- return nil, err
+ return settingsUpdateResultInternal{}, err
}
- valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, err := s.prepareUpdateValues(updates, cfg, defaultCfg)
+ valuesToUpdate, err := s.prepareUpdateValues(updates, cfg, defaultCfg)
if err != nil {
- return nil, err
+ return settingsUpdateResultInternal{}, err
}
if updates.OidcClientSecret != nil && *updates.OidcClientSecret != "" {
valuesToUpdate = append(valuesToUpdate, models.SettingVariable{Key: "oidcClientSecret", Value: *updates.OidcClientSecret})
@@ -450,59 +568,28 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
}
if err := s.persistSettings(ctx, valuesToUpdate); err != nil {
- return nil, err
+ return settingsUpdateResultInternal{}, err
}
- if err := s.refreshSettingsCacheInternal(ctx); err != nil {
- return nil, err
+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); err != nil {
+ return settingsUpdateResultInternal{}, err
}
settingsCfg := s.GetSettingsConfig()
-
- // Now call callbacks after in-memory config is updated
- if changedPolling && s.OnImagePollingSettingsChanged != nil {
- s.OnImagePollingSettingsChanged(ctx)
- }
- if changedAutoUpdate && s.OnAutoUpdateSettingsChanged != nil {
- s.OnAutoUpdateSettingsChanged(ctx)
- }
- if changedScheduledPrune && s.OnScheduledPruneSettingsChanged != nil {
- s.OnScheduledPruneSettingsChanged(ctx)
- }
- if changedVulnerabilityScan && s.OnVulnerabilityScanSettingsChanged != nil {
- s.OnVulnerabilityScanSettingsChanged(ctx)
- }
- if changedAutoHeal && s.OnAutoHealSettingsChanged != nil {
- s.OnAutoHealSettingsChanged(ctx)
- }
- if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool {
- return sv.Key == "projectsDirectory" || sv.Key == "followProjectSymlinks"
- }) && s.OnProjectsDirectoryChanged != nil {
- s.OnProjectsDirectoryChanged(ctx)
- }
- if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool {
- return sv.Key == "templatesDirectory"
- }) && s.OnTemplatesDirectoryChanged != nil {
- s.OnTemplatesDirectoryChanged(ctx)
- }
- if len(changedTimeouts) > 0 && s.OnTimeoutSettingsChanged != nil {
- s.OnTimeoutSettingsChanged(ctx, changedTimeouts)
- }
-
- return settingsCfg.ToSettingVariableSlice(models.SettingVisibilityNonAdmin, false), nil
+ changes := make([]libarcane.SettingUpdate, 0, len(valuesToUpdate))
+ for _, value := range valuesToUpdate {
+ changes = append(changes, libarcane.SettingUpdate{Key: value.Key, Value: value.Value})
+ }
+ return settingsUpdateResultInternal{
+ settings: settingsCfg.ToSettingVariableSlice(models.SettingVisibilityNonAdmin, false),
+ changes: changes,
+ }, nil
}
-func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, bool, bool, bool, bool, bool, []libarcane.SettingUpdate, error) {
+func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defaultCfg *models.Settings) ([]models.SettingVariable, error) {
rt := reflect.TypeFor[settings.Update]()
rv := reflect.ValueOf(updates)
valuesToUpdate := make([]models.SettingVariable, 0)
- changedPolling := false
- changedAutoUpdate := false
- changedScheduledPrune := false
- changedVulnerabilityScan := false
- changedAutoHeal := false
- changedTimeouts := make([]libarcane.SettingUpdate, 0)
-
for i := range rt.NumField() {
field := rt.Field(i)
fieldValue := rv.Field(i)
@@ -520,19 +607,16 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
}
if err := cfg.UpdateField(key, value, false); err != nil {
- return nil, false, false, false, false, false, nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)
+ return nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)
}
valuesToUpdate = append(valuesToUpdate, models.SettingVariable{Key: key, Value: value})
- if libarcane.IsTimeoutSettingKey(key) {
- changedTimeouts = append(changedTimeouts, libarcane.SettingUpdate{Key: key, Value: value})
- }
continue
}
if err := libarcane.ValidateCronSetting(key, value); err != nil {
- return nil, false, false, false, false, false, nil, errors.WrapIff(err, "invalid cron expression for %s", key)
+ return nil, errors.WrapIff(err, "invalid cron expression for %s", key)
}
var valueToSave string
@@ -551,31 +635,13 @@ func (s *SettingsService) prepareUpdateValues(updates settings.Update, cfg, defa
continue
}
if err != nil {
- return nil, false, false, false, false, false, nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)
+ return nil, errors.WrapIff(err, "failed to update in-memory config for key '%s'", key)
}
valuesToUpdate = append(valuesToUpdate, models.SettingVariable{Key: key, Value: valueToSave})
-
- switch key {
- case "pollingEnabled", "pollingInterval":
- changedPolling = true
- case "autoUpdate", "autoUpdateInterval":
- changedAutoUpdate = true
- case "scheduledPruneEnabled",
- "scheduledPruneInterval":
- changedScheduledPrune = true
- case "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyNetwork", "trivySecurityOpts", "trivyPrivileged", "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers":
- changedVulnerabilityScan = true
- case "autoHealEnabled", "autoHealInterval", "autoHealExcludedContainers", "autoHealMaxRestarts", "autoHealRestartWindow":
- changedAutoHeal = true
- }
-
- if libarcane.IsTimeoutSettingKey(key) {
- changedTimeouts = append(changedTimeouts, libarcane.SettingUpdate{Key: key, Value: valueToSave})
- }
}
- return valuesToUpdate, changedPolling, changedAutoUpdate, changedScheduledPrune, changedVulnerabilityScan, changedAutoHeal, changedTimeouts, nil
+ return valuesToUpdate, nil
}
func extractUpdateValue(field reflect.StructField, fieldValue reflect.Value) (string, string, bool) {
@@ -608,76 +674,80 @@ func (s *SettingsService) persistSettings(ctx context.Context, values []models.S
}
func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {
- defaultSettings := s.getDefaultSettings()
- defaultSettingVars := defaultSettings.ToSettingVariableSlice(models.SettingVisibilityAll, false)
-
- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- for _, defaultSetting := range defaultSettingVars {
- var existing models.SettingVariable
- err := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error
-
- switch {
- case errors.Is(err, gorm.ErrRecordNotFound):
- if err := tx.Create(&defaultSetting).Error; err != nil {
- return errors.WrapIff(err, "failed to create default setting %s", defaultSetting.Key)
- }
- case err != nil:
- return errors.WrapIff(err, "failed to check for existing setting %s", defaultSetting.Key)
- case defaultSetting.Key == "trivyImage" && existing.Value != defaultSetting.Value:
- if err := tx.Model(&existing).Update("value", defaultSetting.Value).Error; err != nil {
- return errors.WrapIff(err, "failed to enforce default setting %s", defaultSetting.Key)
+ _, err := actors.Execute(ctx, s.writes, "ensure default settings", func(writeCtx context.Context) (actors.NoPayload, error) {
+ defaultSettings := s.getDefaultSettings()
+ defaultSettingVars := defaultSettings.ToSettingVariableSlice(models.SettingVisibilityAll, false)
+
+ if err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
+ for _, defaultSetting := range defaultSettingVars {
+ var existing models.SettingVariable
+ err := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error
+
+ switch {
+ case errors.Is(err, gorm.ErrRecordNotFound):
+ if err := tx.Create(&defaultSetting).Error; err != nil {
+ return errors.WrapIff(err, "failed to create default setting %s", defaultSetting.Key)
+ }
+ case err != nil:
+ return errors.WrapIff(err, "failed to check for existing setting %s", defaultSetting.Key)
+ case defaultSetting.Key == "trivyImage" && existing.Value != defaultSetting.Value:
+ if err := tx.Model(&existing).Update("value", defaultSetting.Value).Error; err != nil {
+ return errors.WrapIff(err, "failed to enforce default setting %s", defaultSetting.Key)
+ }
}
}
+ return nil
+ }); err != nil {
+ return actors.NoPayload{}, err
}
- return nil
- }); err != nil {
- return err
- }
-
- return nil
+ return actors.NoPayload{}, nil
+ }, nil)
+ return err
}
func (s *SettingsService) PruneUnknownSettings(ctx context.Context) error {
- allowedKeys := allowedSettingKeys()
- if len(allowedKeys) == 0 {
- return nil
- }
-
- keys := make([]string, 0, len(allowedKeys))
- for key := range allowedKeys {
- keys = append(keys, key)
- }
-
- result := s.db.WithContext(ctx).Where("key NOT IN ?", keys).Delete(&models.SettingVariable{})
- if result.Error != nil {
- return errors.WrapIf(result.Error, "failed to prune unknown settings")
- }
+ _, err := actors.Execute(ctx, s.writes, "prune unknown settings", func(writeCtx context.Context) (actors.NoPayload, error) {
+ allowedKeys := allowedSettingKeys()
+ if len(allowedKeys) == 0 {
+ return actors.NoPayload{}, nil
+ }
- if result.RowsAffected > 0 {
- slog.InfoContext(ctx, "Pruned unknown settings", "count", result.RowsAffected)
- }
+ keys := make([]string, 0, len(allowedKeys))
+ for key := range allowedKeys {
+ keys = append(keys, key)
+ }
- return nil
+ result := s.db.WithContext(writeCtx).Where("key NOT IN ?", keys).Delete(&models.SettingVariable{})
+ if result.Error != nil {
+ return actors.NoPayload{}, errors.WrapIf(result.Error, "failed to prune unknown settings")
+ }
+ if result.RowsAffected > 0 {
+ slog.InfoContext(writeCtx, "Pruned unknown settings", "count", result.RowsAffected)
+ }
+ return actors.NoPayload{}, nil
+ }, nil)
+ return err
}
func (s *SettingsService) PersistEnvSettingsIfMissing(ctx context.Context) error {
- rt := reflect.TypeFor[models.Settings]()
- appCfg := config.Load()
- isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled
-
- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- for field := range rt.Fields() {
- if err := s.processEnvField(ctx, tx, field, isEnvOnlyMode); err != nil {
- return err
+ _, err := actors.Execute(ctx, s.writes, "persist environment settings", func(writeCtx context.Context) (actors.NoPayload, error) {
+ rt := reflect.TypeFor[models.Settings]()
+ appCfg := config.Load()
+ isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled
+
+ if err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
+ for field := range rt.Fields() {
+ if err := s.processEnvField(writeCtx, tx, field, isEnvOnlyMode); err != nil {
+ return err
+ }
}
+ return nil
+ }); err != nil {
+ return actors.NoPayload{}, err
}
- return nil
- }); err != nil {
- return err
- }
-
- // Reload settings after persisting env vars
- return s.LoadDatabaseSettings(ctx)
+ return actors.NoPayload{}, s.LoadDatabaseSettings(context.WithoutCancel(writeCtx))
+ }, nil)
+ return err
}
func allowedSettingKeys() map[string]struct{} {
@@ -842,86 +912,90 @@ func (s *SettingsService) SetStringSetting(ctx context.Context, key, value strin
// the autoUpdateExcludedContainers setting. When excluded is true the container
// is added to the list; when false it is removed.
func (s *SettingsService) SetContainerAutoUpdateExclusionInternal(ctx context.Context, containerName string, excluded bool) error {
- raw := s.GetStringSetting(ctx, "autoUpdateExcludedContainers", "")
- existing := make(map[string]struct{})
- var ordered []string
- for part := range strings.SplitSeq(raw, ",") {
- name := strings.TrimSpace(part)
- if name == "" {
- continue
- }
- if _, ok := existing[name]; !ok {
- existing[name] = struct{}{}
- ordered = append(ordered, name)
+ _, err := actors.Execute(ctx, s.writes, "update container auto-update exclusion", func(writeCtx context.Context) (actors.NoPayload, error) {
+ raw := s.GetStringSetting(writeCtx, "autoUpdateExcludedContainers", "")
+ existing := make(map[string]struct{})
+ var ordered []string
+ for part := range strings.SplitSeq(raw, ",") {
+ name := strings.TrimSpace(part)
+ if name == "" {
+ continue
+ }
+ if _, ok := existing[name]; !ok {
+ existing[name] = struct{}{}
+ ordered = append(ordered, name)
+ }
}
- }
- if excluded {
- if _, ok := existing[containerName]; !ok {
- ordered = append(ordered, containerName)
- }
- } else {
- filtered := ordered[:0]
- for _, name := range ordered {
- if name != containerName {
- filtered = append(filtered, name)
+ if excluded {
+ if _, ok := existing[containerName]; !ok {
+ ordered = append(ordered, containerName)
}
+ } else {
+ filtered := ordered[:0]
+ for _, name := range ordered {
+ if name != containerName {
+ filtered = append(filtered, name)
+ }
+ }
+ ordered = filtered
}
- ordered = filtered
- }
- return s.SetStringSetting(ctx, "autoUpdateExcludedContainers", strings.Join(ordered, ","))
+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, "autoUpdateExcludedContainers", strings.Join(ordered, ",")); err != nil {
+ return actors.NoPayload{}, err
+ }
+ return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))
+ }, nil)
+ return err
}
func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, error) {
- const keyName = "encryptionKey"
- var key string
+ return actors.Execute(ctx, s.writes, "ensure encryption key", func(writeCtx context.Context) (string, error) {
+ const keyName = "encryptionKey"
+ var key string
- err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- var sv models.SettingVariable
- err := tx.Where("key = ?", keyName).First(&sv).Error
+ err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
+ var sv models.SettingVariable
+ err := tx.Where("key = ?", keyName).First(&sv).Error
- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
- return errors.WrapIf(err, "failed to load encryption key")
- }
+ if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.WrapIf(err, "failed to load encryption key")
+ }
- // If already present and non-empty, return it
- if sv.Value != "" {
- key = sv.Value
- return nil
- }
+ if sv.Value != "" {
+ key = sv.Value
+ return nil
+ }
- notFound := errors.Is(err, gorm.ErrRecordNotFound)
+ notFound := errors.Is(err, gorm.ErrRecordNotFound)
+ u, genErr := uuid.NewRandom()
+ if genErr != nil {
+ return errors.WrapIf(genErr, "failed to generate encryption key")
+ }
+ sum := sha256.Sum256([]byte(u.String()))
+ generatedKey := base64.StdEncoding.EncodeToString(sum[:])
+ key = generatedKey
- // Generate uuid -> sha256 -> base64 key (32 bytes raw -> 44 chars base64)
- u, genErr := uuid.NewRandom()
- if genErr != nil {
- return errors.WrapIf(genErr, "failed to generate encryption key")
- }
- sum := sha256.Sum256([]byte(u.String()))
- generatedKey := base64.StdEncoding.EncodeToString(sum[:])
- key = generatedKey
+ if notFound {
+ if createErr := tx.Create(&models.SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {
+ return errors.WrapIf(createErr, "failed to persist encryption key")
+ }
+ return nil
+ }
- if notFound {
- if createErr := tx.Create(&models.SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {
- return errors.WrapIf(createErr, "failed to persist encryption key")
+ if updErr := tx.Model(&models.SettingVariable{}).
+ Where("key = ?", keyName).
+ Update("value", generatedKey).Error; updErr != nil {
+ return errors.WrapIf(updErr, "failed to update encryption key")
}
return nil
+ })
+ if err != nil {
+ return "", err
}
- // Record existed but empty value; update it
- if updErr := tx.Model(&models.SettingVariable{}).
- Where("key = ?", keyName).
- Update("value", generatedKey).Error; updErr != nil {
- return errors.WrapIf(updErr, "failed to update encryption key")
- }
- return nil
- })
- if err != nil {
- return "", err
- }
-
- return key, nil
+ return key, nil
+ }, nil)
}
func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projectsDirEnv string) error {
@@ -930,50 +1004,44 @@ func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projec
return nil
}
- var projectsDirSetting models.SettingVariable
- err := s.db.WithContext(ctx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error
-
- if errors.Is(err, gorm.ErrRecordNotFound) {
- slog.DebugContext(ctx, "No projectsDirectory setting found, skipping normalization")
- return nil
- }
-
- if err != nil {
- return errors.WrapIf(err, "failed to load projectsDirectory setting")
- }
+ _, err := actors.Execute(ctx, s.writes, "normalize projects directory", func(writeCtx context.Context) (string, error) {
+ var projectsDirSetting models.SettingVariable
+ err := s.db.WithContext(writeCtx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ slog.DebugContext(writeCtx, "No projectsDirectory setting found, skipping normalization")
+ return "", nil
+ }
+ if err != nil {
+ return "", errors.WrapIf(err, "failed to load projectsDirectory setting")
+ }
- value := strings.TrimSpace(projectsDirSetting.Value)
- // Detect mapping format (container:host), allowing Windows or Unix container paths.
- isMapping := false
- if strings.Contains(value, ":") {
- // Treat as mapping if the container side looks like an absolute Unix path
- // or a Windows drive path (C:/ or C:\). We purposely avoid splitting on the
- // first colon to not break on Windows drive letters.
- if strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value) {
- isMapping = true
+ value := strings.TrimSpace(projectsDirSetting.Value)
+ isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))
+ if filepath.IsAbs(value) || isMapping {
+ slog.DebugContext(writeCtx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)
+ return "", nil
}
- }
- if !filepath.IsAbs(value) && !isMapping {
- // Resolve relative path using current working directory for transparency.
- // Note: In containers, WORKDIR is set to /app so "data/..." becomes "/app/data/...".
cwd, _ := os.Getwd()
absPath, absErr := filepath.Abs(value)
if absErr != nil {
- return errors.WrapIf(absErr, "failed to resolve relative path to absolute")
+ return "", errors.WrapIf(absErr, "failed to resolve relative path to absolute")
}
- slog.InfoContext(ctx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
-
- if err := s.UpdateSetting(ctx, "projectsDirectory", absPath); err != nil {
- return errors.WrapIf(err, "failed to update projectsDirectory")
+ slog.InfoContext(writeCtx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, "projectsDirectory", absPath); err != nil {
+ return "", errors.WrapIf(err, "failed to update projectsDirectory")
}
-
- slog.InfoContext(ctx, "Successfully normalized projects directory")
- } else {
- slog.DebugContext(ctx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)
- }
-
- return nil
+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx)); err != nil {
+ return "", err
+ }
+ slog.InfoContext(writeCtx, "Successfully normalized projects directory")
+ return absPath, nil
+ }, func(value string, err error) {
+ if err == nil && value != "" {
+ s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: "projectsDirectory", Value: value}})
+ }
+ })
+ return err
}
func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {
@@ -984,40 +1052,41 @@ func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {
return nil
}
- var buildsDirSetting models.SettingVariable
- err := s.db.WithContext(ctx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error
-
- if errors.Is(err, gorm.ErrRecordNotFound) {
- slog.DebugContext(ctx, "No buildsDirectory setting found, skipping normalization")
- return nil
- }
-
- if err != nil {
- return errors.WrapIf(err, "failed to load buildsDirectory setting")
- }
+ _, err := actors.Execute(ctx, s.writes, "normalize builds directory", func(writeCtx context.Context) (string, error) {
+ var buildsDirSetting models.SettingVariable
+ err := s.db.WithContext(writeCtx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ slog.DebugContext(writeCtx, "No buildsDirectory setting found, skipping normalization")
+ return "", nil
+ }
+ if err != nil {
+ return "", errors.WrapIf(err, "failed to load buildsDirectory setting")
+ }
- value := strings.TrimSpace(buildsDirSetting.Value)
- if value == "" {
- slog.DebugContext(ctx, "buildsDirectory is empty, skipping normalization")
- return nil
- }
+ value := strings.TrimSpace(buildsDirSetting.Value)
+ if value == "" || filepath.IsAbs(value) {
+ slog.DebugContext(writeCtx, "Builds directory already normalized or empty, skipping", "value", buildsDirSetting.Value)
+ return "", nil
+ }
- if !filepath.IsAbs(value) {
cwd, _ := os.Getwd()
absPath, absErr := filepath.Abs(value)
if absErr != nil {
- return errors.WrapIf(absErr, "failed to resolve relative path to absolute")
+ return "", errors.WrapIf(absErr, "failed to resolve relative path to absolute")
}
- slog.InfoContext(ctx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
-
- if err := s.UpdateSetting(ctx, buildsKey, absPath); err != nil {
- return errors.WrapIf(err, "failed to update buildsDirectory")
+ slog.InfoContext(writeCtx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
+ if err := s.updateSettingValueNoRefreshInternal(writeCtx, buildsKey, absPath); err != nil {
+ return "", errors.WrapIf(err, "failed to update buildsDirectory")
}
-
- slog.InfoContext(ctx, "Successfully normalized builds directory")
- } else {
- slog.DebugContext(ctx, "Builds directory already normalized or custom, skipping", "value", buildsDirSetting.Value)
- }
-
- return nil
+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx)); err != nil {
+ return "", err
+ }
+ slog.InfoContext(writeCtx, "Successfully normalized builds directory")
+ return absPath, nil
+ }, func(value string, err error) {
+ if err == nil && value != "" {
+ s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: buildsKey, Value: value}})
+ }
+ })
+ return err
}
diff --git a/backend/internal/services/settings_service_test.go b/backend/internal/services/settings_service_test.go
index 4e1313b3f..acd6384c0 100644
--- a/backend/internal/services/settings_service_test.go
+++ b/backend/internal/services/settings_service_test.go
@@ -3,12 +3,17 @@ package services
import (
"context"
"path/filepath"
+ "strings"
+ "sync/atomic"
"testing"
+ "time"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
"gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -25,10 +30,57 @@ func setupSettingsTestDB(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
+func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*SettingsService, error) {
+ t.Helper()
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
+ require.NoError(t, err)
+ effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, executor.Stop(stopCtx))
+ require.NoError(t, effects.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+ })
+ return NewSettingsService(ctx, db, executor, effects)
+}
+
+func newAdmissionGateForTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {
+ t.Helper()
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "services-test-admission", t.Name())
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+ })
+ return gate
+}
+
+func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {
+ t.Helper()
+ _, err := actors.Execute(t.Context(), svc.writes, "wait for settings notification submission", func(context.Context) (actors.NoPayload, error) {
+ return actors.NoPayload{}, nil
+ }, nil)
+ require.NoError(t, err)
+ _, err = actors.Execute(t.Context(), svc.effects, "wait for settings notifications", func(context.Context) (actors.NoPayload, error) {
+ return actors.NoPayload{}, nil
+ }, nil)
+ require.NoError(t, err)
+}
+
func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -91,7 +143,7 @@ func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {
func TestSettingsService_EnsureDefaultSettings_OverridesExistingTrivyImage(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyImage", "ghcr.io/aquasecurity/trivy:latest"))
@@ -105,7 +157,7 @@ func TestSettingsService_EnsureDefaultSettings_OverridesExistingTrivyImage(t *te
func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.db.WithContext(ctx).
@@ -118,7 +170,7 @@ func TestSettingsService_GetSettings_UnknownKeysIgnored(t *testing.T) {
func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.SetStringSetting(ctx, "baseServerUrl", "http://cached"))
@@ -134,7 +186,7 @@ func TestSettingsService_GetSettings_UsesCachedSnapshotWithoutDatabase(t *testin
func TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
current, err := svc.GetSettings(ctx)
@@ -155,7 +207,7 @@ func TestSettingsService_AvatarMaxUploadSizeDefaultAndUpdate(t *testing.T) {
func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))
@@ -175,7 +227,7 @@ func TestSettingsServiceUpdateSettingsRejectsOIDCIssuerChangeWithStoredSecretInt
func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecretInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "oidcIssuerUrl", "https://issuer.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "oidcClientSecret", "old-client-secret"))
@@ -195,7 +247,7 @@ func TestSettingsServiceUpdateSettingsAllowsOIDCIssuerChangeWithReplacementSecre
func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))
@@ -214,7 +266,7 @@ func TestSettingsServiceUpdateSettingsRejectsTrivyServerChangeWithStoredTokenInt
func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementTokenInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))
@@ -234,7 +286,7 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithReplacementToke
func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerToken", "old-trivy-token"))
@@ -253,7 +305,7 @@ func TestSettingsServiceUpdateSettingsAllowsClearingTrivyServerWithStoredTokenIn
func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "trivyServerUrl", "https://trivy.example.com"))
@@ -270,7 +322,7 @@ func TestSettingsServiceUpdateSettingsAllowsTrivyServerChangeWithoutStoredTokenI
func TestSettingsService_PruneUnknownSettings_RemovesStaleKeys(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "/tmp/projects"))
@@ -297,7 +349,7 @@ func TestSettingsService_GetSettings_EnvOverride_OidcMergeAccounts(t *testing.T)
db := setupSettingsTestDB(t)
t.Setenv("OIDC_MERGE_ACCOUNTS", "true")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -311,7 +363,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyScanTimeout(t *testing.T)
db := setupSettingsTestDB(t)
t.Setenv("TRIVY_SCAN_TIMEOUT", "1800")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -327,7 +379,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyResourceLimits(t *testing.
t.Setenv("TRIVY_CPU_LIMIT", "2.5")
t.Setenv("TRIVY_MEMORY_LIMIT_MB", "2048")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -343,7 +395,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyNetwork(t *testing.T) {
db := setupSettingsTestDB(t)
t.Setenv("TRIVY_NETWORK", "arcane-external")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -357,7 +409,7 @@ func TestSettingsService_GetSettings_EnvOverride_FollowProjectSymlinks(t *testin
db := setupSettingsTestDB(t)
t.Setenv("FOLLOW_PROJECT_SYMLINKS", "true")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -372,7 +424,7 @@ func TestSettingsService_GetSettings_EnvOverride_TrivyRuntimeSecurity(t *testing
t.Setenv("TRIVY_SECURITY_OPTS", "label=disable,\nlabel=type:container_runtime_t")
t.Setenv("TRIVY_PRIVILEGED", "true")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -387,7 +439,7 @@ func TestSettingsService_GetStringSetting_EnvOverride_SwarmStackSourcesDirectory
db := setupSettingsTestDB(t)
t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", "/mnt/swarm-from-env")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "swarmStackSourcesDirectory", "/tmp/swarm-from-db"))
@@ -399,17 +451,17 @@ func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.False(t, svc.isEnvOverrideActiveInternal("oidcEnabled"))
t.Setenv("OIDC_ENABLED", "false")
- svcWithOverride, err := NewSettingsService(ctx, db)
+ svcWithOverride, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.True(t, svcWithOverride.isEnvOverrideActiveInternal("oidcEnabled"))
t.Setenv("AUTH_SESSION_TIMEOUT", "120")
- svcWithNonOverrideEnv, err := NewSettingsService(ctx, db)
+ svcWithNonOverrideEnv, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.False(t, svcWithNonOverrideEnv.isEnvOverrideActiveInternal("authSessionTimeout"))
}
@@ -417,7 +469,7 @@ func TestSettingsService_isEnvOverrideActiveInternal(t *testing.T) {
func TestSettingsService_GetSetHelpers(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// Defaults for missing keys
@@ -439,7 +491,7 @@ func TestSettingsService_GetSetHelpers(t *testing.T) {
func TestSettingsService_UpdateSetting(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.UpdateSetting(ctx, "pruneImageMode", "all"))
@@ -449,10 +501,24 @@ func TestSettingsService_UpdateSetting(t *testing.T) {
require.Equal(t, "all", sv.Value)
}
+func TestSettingsService_UpdateSettingRejectsInvalidCronBeforePersistence(t *testing.T) {
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+
+ require.NoError(t, svc.UpdateSetting(ctx, "pollingInterval", "*/5 * * * * *"))
+ require.Error(t, svc.UpdateSetting(ctx, "pollingInterval", "not a cron schedule"))
+
+ var setting models.SettingVariable
+ require.NoError(t, svc.db.WithContext(ctx).Where("key = ?", "pollingInterval").First(&setting).Error)
+ require.Equal(t, "*/5 * * * * *", setting.Value)
+}
+
func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.Equal(t, "http://localhost", svc.GetSettingsConfig().BaseServerURL.Value)
@@ -468,40 +534,174 @@ func TestSettingsService_UpdateSetting_RefreshesCachedSnapshot(t *testing.T) {
func TestSettingsService_UpdateSettings_PruneModesDoNotTriggerScheduledPruneCallback(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
callbackCalls := 0
- svc.OnScheduledPruneSettingsChanged = func(context.Context) {
+ svc.SubscribeSettingsChanges([]string{"scheduledPruneEnabled", "scheduledPruneInterval"}, func([]libarcane.SettingUpdate) {
callbackCalls++
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{
PruneImageMode: new("all"),
PruneContainerUntil: new("24h"),
})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.Equal(t, 0, callbackCalls)
}
func TestSettingsService_UpdateSettings_ScheduledPruneScheduleTriggersCallback(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
callbackCalls := 0
- svc.OnScheduledPruneSettingsChanged = func(context.Context) {
+ svc.SubscribeSettingsChanges([]string{"scheduledPruneEnabled", "scheduledPruneInterval"}, func([]libarcane.SettingUpdate) {
callbackCalls++
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{
ScheduledPruneEnabled: new("true"),
})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.Equal(t, 1, callbackCalls)
}
+func TestSettingsServiceActorSerializesConcurrentUpdatesInternal(t *testing.T) {
+ ctx := t.Context()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+
+ start := make(chan struct{})
+ results := make(chan error, 2)
+ go func() {
+ <-start
+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{ProjectsDirectory: new("/data/projects-a")})
+ results <- updateErr
+ }()
+ go func() {
+ <-start
+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{TemplatesDirectory: new("/data/templates-b")})
+ results <- updateErr
+ }()
+ close(start)
+
+ require.NoError(t, <-results)
+ require.NoError(t, <-results)
+ current := svc.GetSettingsConfig()
+ require.Equal(t, "/data/projects-a", current.ProjectsDirectory.Value)
+ require.Equal(t, "/data/templates-b", current.TemplatesDirectory.Value)
+}
+
+func TestSettingsServiceActorPublishesSnapshotBeforeAsynchronousNotificationInternal(t *testing.T) {
+ ctx := t.Context()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+
+ callbackStarted := make(chan string, 1)
+ releaseCallback := make(chan struct{})
+ svc.SubscribeSettingsChanges([]string{"baseServerUrl"}, func(_ []libarcane.SettingUpdate) {
+ callbackStarted <- svc.GetSettingsConfig().BaseServerURL.Value
+ <-releaseCallback
+ })
+
+ updateResult := make(chan error, 1)
+ go func() {
+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{BaseServerURL: new("https://actor.example")})
+ updateResult <- updateErr
+ }()
+
+ require.Equal(t, "https://actor.example", <-callbackStarted)
+ select {
+ case err := <-updateResult:
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("settings update waited for notification callback")
+ }
+
+ secondUpdate := make(chan error, 1)
+ go func() {
+ _, updateErr := svc.UpdateSettings(ctx, settings.Update{ProjectsDirectory: new("/data/unblocked")})
+ secondUpdate <- updateErr
+ }()
+ select {
+ case err := <-secondUpdate:
+ require.NoError(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("settings subscriber blocked the next settings writer")
+ }
+ close(releaseCallback)
+}
+
+func TestSettingsServiceActorNotifiesSubscriberOnceForMultipleMatchingKeysInternal(t *testing.T) {
+ ctx := t.Context()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+
+ var calls atomic.Int32
+ notified := make(chan []libarcane.SettingUpdate, 1)
+ svc.SubscribeSettingsChanges([]string{"pollingEnabled", "pollingInterval"}, func(updates []libarcane.SettingUpdate) {
+ calls.Add(1)
+ notified <- updates
+ })
+
+ _, err = svc.UpdateSettings(ctx, settings.Update{
+ PollingEnabled: new("false"),
+ PollingInterval: new("0 */5 * * * *"),
+ })
+ require.NoError(t, err)
+ require.Len(t, <-notified, 2)
+ require.Equal(t, int32(1), calls.Load())
+}
+
+func TestSettingsServiceActorDoesNotPublishSensitiveValuesInternal(t *testing.T) {
+ ctx := t.Context()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+
+ var calls atomic.Int32
+ svc.SubscribeSettingsChanges([]string{"oidcClientSecret"}, func([]libarcane.SettingUpdate) {
+ calls.Add(1)
+ })
+ _, err = svc.UpdateSettings(ctx, settings.Update{OidcClientSecret: new("should-not-leave-settings-service")})
+ require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
+ require.Zero(t, calls.Load())
+}
+
+func TestSettingsServiceActorSerializesContainerExclusionUpdatesInternal(t *testing.T) {
+ ctx := t.Context()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+
+ start := make(chan struct{})
+ results := make(chan error, 2)
+ for _, name := range []string{"api", "worker"} {
+ go func() {
+ <-start
+ results <- svc.SetContainerAutoUpdateExclusionInternal(ctx, name, true)
+ }()
+ }
+ close(start)
+
+ require.NoError(t, <-results)
+ require.NoError(t, <-results)
+ require.ElementsMatch(t, []string{"api", "worker"}, strings.Split(svc.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""), ","))
+}
+
func BenchmarkSettingsService_GetSettings(b *testing.B) {
ctx := context.Background()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@@ -512,7 +712,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {
b.Fatal(err)
}
settingsDB := &database.DB{DB: db}
- svc, err := NewSettingsService(ctx, settingsDB)
+ svc, err := newSettingsServiceForTestInternal(b, ctx, settingsDB)
if err != nil {
b.Fatal(err)
}
@@ -534,7 +734,7 @@ func BenchmarkSettingsService_GetSettings(b *testing.B) {
func TestSettingsService_EnsureEncryptionKey(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
k1, err := svc.EnsureEncryptionKey(ctx)
@@ -553,7 +753,7 @@ func TestSettingsService_EnsureEncryptionKey(t *testing.T) {
func TestSettingsService_LoadDatabaseSettings_ReloadsChanges(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// Initially empty DB -> defaults (not persisted yet)
@@ -580,7 +780,7 @@ func TestSettingsService_LoadDatabaseSettings_UIConfigurationDisabled_Env(t *tes
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// Reload explicitly (NewSettingsService already did, but explicit for clarity)
@@ -596,7 +796,7 @@ func TestSettingsService_PersistEnvSettingsIfMissing_DoesNotOverrideForcedTrivyI
db := setupSettingsTestDB(t)
t.Setenv("TRIVY_IMAGE", "ghcr.io/aquasecurity/trivy:latest")
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -611,7 +811,7 @@ func TestSettingsService_PersistEnvSettingsIfMissing_DoesNotOverrideForcedTrivyI
func TestSettingsService_UpdateSettings_RefreshesCache(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -640,7 +840,7 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -662,17 +862,18 @@ func TestSettingsService_UpdateSettings_ReturnsEnvOverriddenValues(t *testing.T)
func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
var callbackPayload []libarcane.SettingUpdate
- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {
+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {
callbackPayload = timeoutSettings
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyScanTimeout: new("1200")})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.NotNil(t, callbackPayload)
require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyScanTimeout", Value: "1200"})
@@ -681,14 +882,14 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyScanTimeout(
func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimits(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
var callbackPayload []libarcane.SettingUpdate
- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {
+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {
callbackPayload = timeoutSettings
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{
TrivyResourceLimitsEnabled: new("false"),
@@ -696,6 +897,7 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimi
TrivyMemoryLimitMb: new("3072"),
})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.NotNil(t, callbackPayload)
require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyResourceLimitsEnabled", Value: "false"})
@@ -706,17 +908,18 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyResourceLimi
func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentScanContainers(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
var callbackPayload []libarcane.SettingUpdate
- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {
+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {
callbackPayload = timeoutSettings
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyConcurrentScanContainers: new("4")})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.NotNil(t, callbackPayload)
require.Contains(t, callbackPayload, libarcane.SettingUpdate{Key: "trivyConcurrentScanContainers", Value: "4"})
@@ -725,81 +928,85 @@ func TestSettingsService_UpdateSettings_TimeoutCallbackIncludesTrivyConcurrentSc
func TestSettingsService_UpdateSettings_TrivyNetworkTriggersVulnerabilityCallback(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
callbackCalled := false
- svc.OnVulnerabilityScanSettingsChanged = func(_ context.Context) {
+ svc.SubscribeSettingsChanges([]string{"trivyNetwork"}, func(_ []libarcane.SettingUpdate) {
callbackCalled = true
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.True(t, callbackCalled)
}
func TestSettingsService_UpdateSettings_TrivyNetworkDoesNotTriggerTimeoutCallback(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
var callbackPayload []libarcane.SettingUpdate
- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {
+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {
callbackPayload = timeoutSettings
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyNetwork: new("arcane-external")})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.Nil(t, callbackPayload)
}
func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityTriggersVulnerabilityCallback(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
callbackCalled := false
- svc.OnVulnerabilityScanSettingsChanged = func(_ context.Context) {
+ svc.SubscribeSettingsChanges([]string{"trivySecurityOpts", "trivyPrivileged"}, func(_ []libarcane.SettingUpdate) {
callbackCalled = true
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{
TrivySecurityOpts: new("label=disable"),
TrivyPrivileged: new("true"),
})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.True(t, callbackCalled)
}
func TestSettingsService_UpdateSettings_TrivyRuntimeSecurityDoesNotTriggerTimeoutCallback(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
var callbackPayload []libarcane.SettingUpdate
- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {
+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {
callbackPayload = timeoutSettings
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{
TrivySecurityOpts: new("label=disable"),
TrivyPrivileged: new("true"),
})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.Nil(t, callbackPayload)
}
func TestSettingsService_UpdateSettings_TrivyPreserveCacheOnVolumePrunePersists(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -819,17 +1026,18 @@ func TestSettingsService_UpdateSettings_TrivyPreserveCacheOnVolumePrunePersists(
func TestSettingsService_UpdateSettings_TrivyPreserveCacheOnVolumePruneDoesNotTriggerTimeoutCallback(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
var callbackPayload []libarcane.SettingUpdate
- svc.OnTimeoutSettingsChanged = func(_ context.Context, timeoutSettings []libarcane.SettingUpdate) {
+ svc.SubscribeSettingsChanges(libarcane.TimeoutSettingKeys(), func(timeoutSettings []libarcane.SettingUpdate) {
callbackPayload = timeoutSettings
- }
+ })
_, err = svc.UpdateSettings(ctx, settings.Update{TrivyPreserveCacheOnVolumePrune: new("false")})
require.NoError(t, err)
+ waitForSettingsNotificationsInternal(t, svc)
require.Nil(t, callbackPayload)
}
@@ -845,7 +1053,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)
internalVal := "test-instance-id"
require.NoError(t, db.DB.Create(&models.SettingVariable{Key: internalKey, Value: internalVal}).Error)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// Reload explicitly to trigger the env loading path
@@ -859,7 +1067,7 @@ func TestSettingsService_LoadDatabaseSettings_InternalKeys_EnvMode(t *testing.T)
func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// Seed with relative path
@@ -882,7 +1090,7 @@ func TestSettingsService_NormalizeProjectsDirectory_ConvertsRelativeToAbsolute(t
func TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// Seed with relative path
@@ -901,7 +1109,7 @@ func TestSettingsService_NormalizeProjectsDirectory_SkipsWhenEnvSet(t *testing.T
func TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
customPath := "/custom/projects/path"
@@ -920,7 +1128,7 @@ func TestSettingsService_NormalizeProjectsDirectory_LeavesOtherPathsUnchanged(t
func TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
// Don't create the setting at all
@@ -933,7 +1141,7 @@ func TestSettingsService_NormalizeProjectsDirectory_HandlesNotFound(t *testing.T
func TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalization(t *testing.T) {
ctx := context.Background()
db := setupSettingsTestDB(t)
- svc, err := NewSettingsService(ctx, db)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, svc.EnsureDefaultSettings(ctx))
@@ -955,3 +1163,25 @@ func TestSettingsService_NormalizeProjectsDirectory_UpdatesCacheAfterNormalizati
require.Equal(t, expectedPath, cfg2.ProjectsDirectory.Value)
require.True(t, filepath.IsAbs(cfg2.ProjectsDirectory.Value), "path should be absolute")
}
+
+func TestSettingsService_NormalizeProjectsDirectoryPublishesChangeInternal(t *testing.T) {
+ ctx := t.Context()
+ db := setupSettingsTestDB(t)
+ svc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, svc.EnsureDefaultSettings(ctx))
+ require.NoError(t, svc.UpdateSetting(ctx, "projectsDirectory", "data/projects"))
+
+ notified := make(chan []libarcane.SettingUpdate, 1)
+ unsubscribe := svc.SubscribeSettingsChanges([]string{"projectsDirectory"}, func(updates []libarcane.SettingUpdate) {
+ notified <- updates
+ })
+ defer unsubscribe()
+
+ require.NoError(t, svc.NormalizeProjectsDirectory(ctx, ""))
+ waitForSettingsNotificationsInternal(t, svc)
+ updates := <-notified
+ require.Len(t, updates, 1)
+ require.Equal(t, "projectsDirectory", updates[0].Key)
+ require.True(t, filepath.IsAbs(updates[0].Value))
+}
diff --git a/backend/internal/services/swarm_service_test.go b/backend/internal/services/swarm_service_test.go
index 8b593b4a1..8efc7b53c 100644
--- a/backend/internal/services/swarm_service_test.go
+++ b/backend/internal/services/swarm_service_test.go
@@ -38,7 +38,7 @@ func TestDefaultSwarmListenAddrInternal(t *testing.T) {
func TestSwarmService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToken(t *testing.T) {
ctx := context.Background()
db := setupEnvironmentServiceTestDB(t)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
envSvc := NewEnvironmentService(db, nil, nil, nil, settingsSvc, nil)
@@ -82,7 +82,7 @@ func TestSwarmService_UpdateAndGetStackSource_UsesStoredFilesWithoutSwarmManager
rootDir := t.TempDir()
t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
svc := NewSwarmService(nil, settingsSvc, nil, nil, nil)
@@ -140,7 +140,7 @@ func TestSwarmService_UpdateAndGetStackSource_RoundTripsOverride(t *testing.T) {
rootDir := t.TempDir()
t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)
- settingsSvc, err := NewSettingsService(ctx, db)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
svc := NewSwarmService(nil, settingsSvc, nil, nil, nil)
diff --git a/backend/internal/services/template_service_test.go b/backend/internal/services/template_service_test.go
index 8ae24d68a..32757bb91 100644
--- a/backend/internal/services/template_service_test.go
+++ b/backend/internal/services/template_service_test.go
@@ -483,7 +483,7 @@ func minimalSettingsServiceForTest(t *testing.T) *SettingsService {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))
- svc, err := NewSettingsService(context.Background(), &database.DB{DB: db})
+ svc, err := newSettingsServiceForTestInternal(t, context.Background(), &database.DB{DB: db})
require.NoError(t, err)
return svc
}
diff --git a/backend/internal/services/variable_service_test.go b/backend/internal/services/variable_service_test.go
index 77e07175e..7500345bb 100644
--- a/backend/internal/services/variable_service_test.go
+++ b/backend/internal/services/variable_service_test.go
@@ -50,7 +50,7 @@ func setupVariableServiceTest(t *testing.T) (*VariableService, *database.DB, str
projectsDir := t.TempDir()
dbWrap := &database.DB{DB: db}
- settingsSvc, err := NewSettingsService(context.Background(), dbWrap)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, context.Background(), dbWrap)
require.NoError(t, err)
require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "projectsDirectory", projectsDir))
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.go
index 924e767e0..aa1a35249 100644
--- a/backend/pkg/libarcane/edge/client.go
+++ b/backend/pkg/libarcane/edge/client.go
@@ -9,13 +9,16 @@ import (
"net"
"net/http"
"strings"
+ "sync"
"time"
"emperror.dev/errors"
"github.com/cenkalti/backoff/v5"
"github.com/coder/websocket"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/google/uuid"
"github.com/samber/mo"
)
@@ -29,7 +32,8 @@ const (
healthyTunnelSessionDuration = 60 * time.Second
// maxPollRetryInterval caps the exponential backoff applied to failed poll
// control requests. Successful polls return to the manager-advertised interval.
- maxPollRetryInterval = 60 * time.Second
+ maxPollRetryInterval = 60 * time.Second
+ tunnelSessionWorkerDrainTimeout = 10 * time.Second
// DefaultHeartbeatInterval is how often the client sends heartbeats
DefaultHeartbeatInterval = 30 * time.Second
@@ -66,14 +70,6 @@ func (t *commandRequestTransfer) stopInternal() {
}
}
-// getConn returns the active tunnel connection, or nil if none is established.
-func (c *TunnelClient) getConn() TunnelConnection {
- if box := c.conn.Load(); box != nil {
- return box.conn
- }
- return nil
-}
-
// NewTunnelClient creates a new tunnel client
func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
reconnectInterval := time.Duration(cfg.EdgeReconnectInterval) * time.Second
@@ -104,8 +100,6 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
managerURL: managerURL,
managerGRPCAddr: managerGRPCAddr,
localPort: localPort,
- httpClient: &http.Client{},
- stopCh: make(chan struct{}),
requestTimeout: DefaultRequestTimeout,
agentInstanceID: uuid.NewString(),
}
@@ -113,11 +107,10 @@ func NewTunnelClient(cfg *Config, handler http.Handler) *TunnelClient {
// StartWithErrorChan runs the tunnel client and optionally emits connection errors.
func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error) {
- slog.InfoContext(ctx, "Starting edge agent session client", StartupLogAttrs(c.cfg)...)
if errCh != nil {
defer close(errCh)
}
-
+ slog.InfoContext(ctx, "Starting edge agent session client", StartupLogAttrs(c.cfg)...)
// A fixed reconnect interval meant a permanently broken or unauthorized agent
// hammered the manager's auth and DB path forever. Back off exponentially
// (with jitter) up to maxReconnectInterval, and reset only once a session has
@@ -131,19 +124,15 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)
case <-ctx.Done():
slog.InfoContext(ctx, "Edge tunnel client shutting down")
return
- case <-c.stopCh:
- slog.InfoContext(ctx, "Edge tunnel client stopped")
- return
default:
sessionStart := time.Now()
if err := c.connectAndServe(ctx); err != nil {
+ slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)
if errCh != nil {
select {
case errCh <- err:
default:
}
- } else {
- slog.WarnContext(ctx, "Edge tunnel disconnected", "error", err)
}
}
@@ -158,9 +147,6 @@ func (c *TunnelClient) StartWithErrorChan(ctx context.Context, errCh chan error)
case <-ctx.Done():
reconnectTimer.Stop()
return
- case <-c.stopCh:
- reconnectTimer.Stop()
- return
case <-reconnectTimer.C:
reconnectTimer.Stop()
slog.InfoContext(ctx, "Attempting to reconnect edge tunnel", "delay", delay)
@@ -393,20 +379,20 @@ func (c *TunnelClient) grpcFailureStreakInternal() int {
func (c *TunnelClient) registerMessageInternal() *TunnelMessage {
capabilities := AdvertisedEdgeCommands()
capabilities = append(capabilities, tunnelCapabilityChunkedRequest, tunnelCapabilityProtoParity)
+ registration, _ := c.registration.Load()
return &TunnelMessage{
Type: MessageTypeRegister,
AgentToken: c.cfg.AgentToken,
AgentInstance: c.agentInstanceID,
Capabilities: capabilities,
- ResumeSession: c.sessionID,
+ ResumeSession: registration.sessionID,
}
}
-func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMessage, error) {
+func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context, conn TunnelConnection) (*TunnelMessage, error) {
if c == nil {
return nil, errors.New("edge tunnel connection is not initialized")
}
- conn := c.getConn()
if conn == nil {
return nil, errors.New("edge tunnel connection is not initialized")
}
@@ -446,8 +432,7 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMe
if !result.msg.Accepted {
return nil, errors.Errorf("manager rejected tunnel registration: %s", result.msg.Error)
}
- c.sessionID = result.msg.SessionID
- c.managerCapabilities = append([]string(nil), result.msg.Capabilities...)
+ c.registration.Store(clientRegistrationInternal{sessionID: result.msg.SessionID})
return result.msg, nil
}
}
@@ -455,19 +440,32 @@ func (c *TunnelClient) awaitRegistrationInternal(ctx context.Context) (*TunnelMe
// serveTunnelSessionInternal runs the shared register/heartbeat/message
// lifecycle on an established tunnel connection, for every transport.
func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn TunnelConnection, managerAddr string) error {
- c.conn.Store(&connBox{conn: conn})
+ box := &connBox{conn: conn}
+ c.conn.Store(box)
setActiveAgentTunnelConn(conn)
- defer clearActiveAgentTunnelConn(conn)
- // Always tear down through the wrapper so the peer sees a clean close
- // (websocket close frame / gRPC half-close) and later Sends hit the
- // closed guard instead of the dead socket.
- defer func() { _ = conn.Close() }()
+ connCtx, connCancel := context.WithCancel(ctx)
+ var workers sync.WaitGroup
+ stopCloseWatch := context.AfterFunc(connCtx, func() { _ = conn.Close() })
+ defer func() {
+ connCancel()
+ c.closeAllStreams()
+ c.clearCommandRequestTransfersInternal(conn)
+ _ = conn.Close()
+ stopCloseWatch()
+ drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(ctx), tunnelSessionWorkerDrainTimeout)
+ if err := utils.WaitGroup(drainCtx, &workers); err != nil {
+ slog.WarnContext(context.WithoutCancel(ctx), "Timed out waiting for edge tunnel session workers", "error", err)
+ }
+ cancelDrain()
+ c.conn.CompareAndSwap(box, nil)
+ clearActiveAgentTunnelConn(conn)
+ }()
if err := conn.Send(c.registerMessageInternal()); err != nil {
return errors.WrapIff(err, "failed to send %s tunnel register message", conn.Transport())
}
- registerMsg, err := c.awaitRegistrationInternal(ctx)
+ registerMsg, err := c.awaitRegistrationInternal(ctx, conn)
if err != nil {
return err
}
@@ -480,21 +478,13 @@ func (c *TunnelClient) serveTunnelSessionInternal(ctx context.Context, conn Tunn
)
c.markTransportConnectedInternal(conn.Transport())
- connCtx, connCancel := context.WithCancel(ctx)
- defer connCancel()
- // A blocked websocket Receive is not ctx-aware; closing the connection is
- // what unblocks messageLoop when the session context is cancelled.
- go func() {
- <-connCtx.Done()
- _ = conn.Close()
- }()
- go c.heartbeatLoop(connCtx)
+ workers.Go(func() { c.heartbeatLoop(connCtx, conn) })
- return c.messageLoop(connCtx)
+ return c.messageLoop(connCtx, conn, &workers)
}
// heartbeatLoop sends periodic heartbeats
-func (c *TunnelClient) heartbeatLoop(ctx context.Context) {
+func (c *TunnelClient) heartbeatLoop(ctx context.Context, conn TunnelConnection) {
ticker := time.NewTicker(c.heartbeatInterval)
defer ticker.Stop()
@@ -503,7 +493,6 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
- conn := c.getConn()
if conn == nil || conn.IsClosed() {
return
}
@@ -527,13 +516,7 @@ func (c *TunnelClient) heartbeatLoop(ctx context.Context) {
}
// messageLoop processes incoming messages from the manager
-func (c *TunnelClient) messageLoop(ctx context.Context) error {
- // Tear down any streams opened on this connection when the loop exits so a
- // reconnect does not leak goroutines, local sockets, or activeStreams entries.
- defer c.closeAllStreams()
-
- conn := c.getConn()
- defer c.clearCommandRequestTransfersInternal(conn)
+func (c *TunnelClient) messageLoop(ctx context.Context, conn TunnelConnection, workers *sync.WaitGroup) error {
for {
select {
case <-ctx.Done():
@@ -546,19 +529,19 @@ func (c *TunnelClient) messageLoop(ctx context.Context) error {
switch msg.Type {
case MessageTypeRequest:
- go c.handleRequest(ctx, conn, msg)
+ workers.Go(func() { c.handleRequest(ctx, conn, msg) })
case MessageTypeCommandRequest:
if transferID := strings.TrimSpace(msg.Metadata[bodyTransferMetadataKey]); transferID != "" {
c.beginCommandRequestTransferInternal(ctx, conn, transferID, msg)
} else {
- go c.handleCommandRequest(ctx, conn, msg)
+ workers.Go(func() { c.handleCommandRequest(ctx, conn, msg) })
}
case MessageTypeFileChunk:
- c.handleCommandRequestChunkInternal(ctx, msg)
+ c.handleCommandRequestChunkInternal(ctx, msg, workers)
case MessageTypeWebSocketStart:
- c.handleWebSocketStart(ctx, conn, msg)
+ c.handleWebSocketStart(ctx, conn, msg, workers)
case MessageTypeStreamOpen:
- c.handleStreamOpen(ctx, conn, msg)
+ c.handleStreamOpen(ctx, conn, msg, workers)
case MessageTypeWebSocketData:
c.handleWebSocketData(ctx, msg)
case MessageTypeStreamData:
@@ -614,7 +597,7 @@ func (c *TunnelClient) beginCommandRequestTransferInternal(ctx context.Context,
}
}
-func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, msg *TunnelMessage) {
+func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, msg *TunnelMessage, workers *sync.WaitGroup) {
if c == nil || msg == nil {
return
}
@@ -647,7 +630,7 @@ func (c *TunnelClient) handleCommandRequestChunkInternal(ctx context.Context, ms
if c.requestTransfers.CompareAndDelete(msg.ID, transfer) {
transfer.stopInternal()
transfer.request.Body = append([]byte(nil), transfer.body.Bytes()...)
- go c.handleCommandRequest(ctx, transfer.conn, transfer.request)
+ workers.Go(func() { c.handleCommandRequest(ctx, transfer.conn, transfer.request) })
}
}
@@ -828,7 +811,7 @@ func (c *TunnelClient) handleRequestStreaming(ctx context.Context, conn TunnelCo
}
// handleWebSocketStart handles a WebSocket stream start request from the manager.
-func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
+func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {
if msg.Command == "" {
if commandName, ok := ResolveEdgeCommandName(http.MethodGet, msg.Path, true).Get(); ok {
msg.Command = commandName
@@ -865,11 +848,11 @@ func (c *TunnelClient) handleWebSocketStart(ctx context.Context, conn TunnelConn
streamCtx, cancel := context.WithCancel(ctx)
stream := c.registerStream(conn, streamID, ws, cancel)
- go c.startLocalWebSocketReadLoop(ctx, streamCtx, streamID, ws, stream)
- go c.startLocalWebSocketWriteLoop(ctx, streamCtx, ws, stream, cancel)
+ workers.Go(func() { c.startLocalWebSocketReadLoop(ctx, streamCtx, streamID, ws, stream) })
+ workers.Go(func() { c.startLocalWebSocketWriteLoop(ctx, streamCtx, ws, stream, cancel) })
}
-func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnection, msg *TunnelMessage) {
+func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnection, msg *TunnelMessage, workers *sync.WaitGroup) {
if !ValidateEdgeCommand(msg.Command, http.MethodGet, msg.Path, true) {
c.sendStreamCloseMessage(conn, msg.ID, "unsupported edge stream")
return
@@ -882,7 +865,7 @@ func (c *TunnelClient) handleStreamOpen(ctx context.Context, conn TunnelConnecti
Path: msg.Path,
Query: msg.Query,
Headers: msg.Headers,
- })
+ }, workers)
}
func (c *TunnelClient) buildLocalWebSocketURLInternal(msg *TunnelMessage) string {
@@ -1439,8 +1422,8 @@ func (r *streamingResponseRecorder) writeHeaderLocked(statusCode int) error {
return nil
}
-// StartTunnelClientWithErrors starts the tunnel client and returns a channel for connection errors.
-func StartTunnelClientWithErrors(ctx context.Context, cfg *Config, handler http.Handler) (<-chan error, error) {
+// StartTunnelClient starts the tunnel client on the shared actor runtime.
+func StartTunnelClient(ctx context.Context, runtime *actors.Runtime, cfg *Config, handler http.Handler) (func(context.Context) error, error) {
if !cfg.EdgeAgent {
return nil, errors.New("edge tunnel disabled")
}
@@ -1471,7 +1454,12 @@ func StartTunnelClientWithErrors(ctx context.Context, cfg *Config, handler http.
}
client := NewTunnelClient(cfg, handler)
- errCh := make(chan error, 1)
- go client.StartWithErrorChan(ctx, errCh)
- return errCh, nil
+ runner, err := actors.NewRunner(ctx, runtime, "edge", "agent-client", "Edge tunnel client", 3, func(runCtx context.Context) error {
+ client.StartWithErrorChan(runCtx, nil)
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ return runner.Stop, nil
}
diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.go
index beca1a404..199c20264 100644
--- a/backend/pkg/libarcane/edge/client_test.go
+++ b/backend/pkg/libarcane/edge/client_test.go
@@ -689,12 +689,10 @@ func TestTunnelClient_HeartbeatLoop_ClosesConnectionOnSendFailure(t *testing.T)
client := &TunnelClient{
heartbeatInterval: 5 * time.Millisecond,
}
- client.conn.Store(&connBox{conn: conn})
-
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
- client.heartbeatLoop(ctx)
+ client.heartbeatLoop(ctx, conn)
assert.True(t, conn.closeCalled)
}
@@ -989,17 +987,17 @@ func TestTunnelClient_useTLSForManagerGRPC(t *testing.T) {
}
}
-func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {
+func TestStartTunnelClient_GRPCValidation(t *testing.T) {
ctx := t.Context()
t.Run("edge mode required", func(t *testing.T) {
- _, err := StartTunnelClientWithErrors(ctx, &Config{}, http.NotFoundHandler())
+ _, err := StartTunnelClient(ctx, nil, &Config{}, http.NotFoundHandler())
require.Error(t, err)
assert.Contains(t, err.Error(), "edge tunnel disabled")
})
t.Run("manager url required for grpc transport", func(t *testing.T) {
- _, err := StartTunnelClientWithErrors(ctx, &Config{
+ _, err := StartTunnelClient(ctx, nil, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
AgentToken: "token",
@@ -1009,7 +1007,7 @@ func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {
})
t.Run("agent token required", func(t *testing.T) {
- _, err := StartTunnelClientWithErrors(ctx, &Config{
+ _, err := StartTunnelClient(ctx, nil, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
ManagerApiUrl: "https://manager.example.com/arcane/api",
@@ -1019,7 +1017,7 @@ func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {
})
t.Run("mtls requires https manager url", func(t *testing.T) {
- _, err := StartTunnelClientWithErrors(ctx, &Config{
+ _, err := StartTunnelClient(ctx, nil, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
ManagerApiUrl: "http://manager.example.com/api",
@@ -1031,7 +1029,7 @@ func TestStartTunnelClientWithErrors_GRPCValidation(t *testing.T) {
})
t.Run("required mtls auto-enrollment failure surfaces", func(t *testing.T) {
- _, err := StartTunnelClientWithErrors(ctx, &Config{
+ _, err := StartTunnelClient(ctx, nil, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
ManagerApiUrl: "https://127.0.0.1:1/api",
@@ -1113,7 +1111,8 @@ func TestTunnelClient_connectAndServeGRPC_TimesOutWithoutRegisterResponse(t *tes
assert.Contains(t, err.Error(), "timed out waiting for tunnel registration response")
assert.Contains(t, err.Error(), "not forwarding gRPC")
assert.EqualValues(t, 1, service.connectCount.Load())
- assert.True(t, client.getConn() == nil || client.getConn().IsClosed())
+ active := client.conn.Load()
+ assert.True(t, active == nil || active.conn.IsClosed())
}
func TestTunnelClient_awaitRegistrationInternal_ClosesConnOnContextDone(t *testing.T) {
@@ -1126,7 +1125,7 @@ func TestTunnelClient_awaitRegistrationInternal_ClosesConnOnContextDone(t *testi
}
client.conn.Store(&connBox{conn: conn})
- msg, err := client.awaitRegistrationInternal(ctx)
+ msg, err := client.awaitRegistrationInternal(ctx, conn)
require.Nil(t, msg)
require.ErrorIs(t, err, context.DeadlineExceeded)
assert.True(t, conn.IsClosed())
@@ -1155,7 +1154,7 @@ func TestTunnelClient_awaitRegistrationInternal_ClosesAttemptConnWhenClientConnC
}
resultCh := make(chan registrationResult, 1)
go func() {
- msg, err := client.awaitRegistrationInternal(ctx)
+ msg, err := client.awaitRegistrationInternal(ctx, firstConn)
resultCh <- registrationResult{msg: msg, err: err}
}()
@@ -1842,14 +1841,13 @@ func TestTunnelClient_pollTunnelControlInternal_UsesConfiguredHTTPClient(t *test
})
client := NewTunnelClient(&Config{AgentToken: "valid-token"}, http.NotFoundHandler())
- client.httpClient = &http.Client{Transport: rewriteTransport}
+ httpClient := &http.Client{Transport: rewriteTransport}
- resp, err := client.pollTunnelControlInternal(context.Background(), "http://127.0.0.1:1/api/tunnel/poll", false)
+ resp, err := client.pollTunnelControlInternal(context.Background(), httpClient, "http://127.0.0.1:1/api/tunnel/poll", false)
require.NoError(t, err)
assert.Equal(t, TunnelStatusIdle, resp.Status)
assert.Equal(t, 1, resp.PollIntervalSeconds)
- assert.NotNil(t, client.httpClient)
- assert.NotSame(t, http.DefaultClient, client.httpClient)
+ assert.NotSame(t, http.DefaultClient, httpClient)
defaultReq, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://127.0.0.1:1/api/tunnel/poll", nil)
require.NoError(t, err)
@@ -2006,9 +2004,19 @@ func TestTunnelClient_syncPollManagedSessionInternal_IdleUsesBoundedStopTimeout(
}
nextSession, err := (&TunnelClient{}).syncPollManagedSessionInternal(context.Background(), session, TunnelStatusIdle)
- require.Nil(t, nextSession)
+ require.Same(t, session, nextSession)
require.Error(t, err)
assert.EqualError(t, err, "timed out waiting for poll-managed websocket session to stop")
+
+ fencedSession, err := (&TunnelClient{}).syncPollManagedSessionInternal(context.Background(), nextSession, TunnelStatusRequired)
+ require.NoError(t, err)
+ require.Same(t, session, fencedSession)
+
+ started := time.Now()
+ stillDraining, err := (&TunnelClient{}).syncPollManagedSessionInternal(context.Background(), nextSession, TunnelStatusIdle)
+ require.NoError(t, err)
+ require.Same(t, session, stillDraining)
+ require.Less(t, time.Since(started), defaultPollManagedSessionStopTimeout)
}
func startTestPollAndGRPCManagerInternal(t *testing.T, ctx context.Context, service tunnelpb.TunnelServiceServer, pollResp TunnelPollResponse) (string, func()) {
diff --git a/backend/pkg/libarcane/edge/client_transport_poll.go b/backend/pkg/libarcane/edge/client_transport_poll.go
index 2be6b6d04..e79ebcb20 100644
--- a/backend/pkg/libarcane/edge/client_transport_poll.go
+++ b/backend/pkg/libarcane/edge/client_transport_poll.go
@@ -19,6 +19,8 @@ const defaultTunnelPollRequestTimeout = 15 * time.Second
var defaultPollManagedSessionStopTimeout = 5 * time.Second
+var errPollManagedSessionStopTimeout = errors.Sentinel("timed out waiting for poll-managed websocket session to stop")
+
func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {
managerBaseURL := strings.TrimRight(strings.TrimSpace(c.cfg.GetManagerBaseURL()), "/")
if managerBaseURL == "" {
@@ -28,8 +30,6 @@ func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {
if err != nil {
return errors.WrapIf(err, "failed to configure edge poll client")
}
- c.httpClient = httpClient
-
pollURL := managerBaseURL + "/api/tunnel/poll"
interval := DefaultTunnelPollInterval
// Failed polls back off exponentially instead of retrying every 2s forever;
@@ -64,7 +64,7 @@ func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {
return ctx.Err()
}
- status, err := c.pollTunnelControlInternal(ctx, pollURL, session != nil)
+ status, err := c.pollTunnelControlInternal(ctx, httpClient, pollURL, session != nil)
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
@@ -93,7 +93,10 @@ func (c *TunnelClient) connectAndServePoll(ctx context.Context) error {
session, err = c.syncPollManagedSessionInternal(ctx, session, status.Status)
if err != nil {
- return err
+ if !errors.Is(err, errPollManagedSessionStopTimeout) {
+ return err
+ }
+ slog.WarnContext(ctx, "Poll-managed tunnel is still draining; replacement remains fenced", "error", err)
}
session, err = waitForNextPollCycleInternal(ctx, session, interval)
@@ -136,13 +139,17 @@ func (c *TunnelClient) syncPollManagedSessionInternal(ctx context.Context, sessi
if session == nil {
return nil, nil
}
+ if session.stopping {
+ return session, nil
+ }
slog.InfoContext(ctx, "Poll control plane marked edge tunnel idle; closing session")
+ session.stopping = true
idleCtx, idleCancel := context.WithTimeout(ctx, defaultPollManagedSessionStopTimeout)
err := c.stopPollManagedSessionInternal(idleCtx, session)
idleCancel()
if err != nil {
- return nil, err
+ return session, err
}
return nil, nil
default:
@@ -173,7 +180,7 @@ func waitForNextPollCycleInternal(ctx context.Context, session *pollManagedTunne
}
}
-func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, pollURL string, connected bool) (*TunnelPollResponse, error) {
+func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, httpClient *http.Client, pollURL string, connected bool) (*TunnelPollResponse, error) {
pollReq := TunnelPollRequest{
Transport: EdgeTransportPoll,
Connected: connected,
@@ -196,7 +203,7 @@ func (c *TunnelClient) pollTunnelControlInternal(ctx context.Context, pollURL st
req.Header.Set(header, value)
}
- resp, err := c.httpClient.Do(req)
+ resp, err := httpClient.Do(req)
if err != nil {
return nil, errors.WrapIf(err, "poll request failed")
}
@@ -234,6 +241,7 @@ func (c *TunnelClient) stopPollManagedSessionInternal(ctx context.Context, sessi
if session == nil {
return nil
}
+ session.stopping = true
session.cancel()
select {
@@ -241,7 +249,7 @@ func (c *TunnelClient) stopPollManagedSessionInternal(ctx context.Context, sessi
return err
case <-ctx.Done():
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
- return errors.New("timed out waiting for poll-managed websocket session to stop")
+ return errPollManagedSessionStopTimeout
}
return ctx.Err()
}
diff --git a/backend/pkg/libarcane/edge/client_transport_websocket.go b/backend/pkg/libarcane/edge/client_transport_websocket.go
index 740ebd58b..c12ffcef6 100644
--- a/backend/pkg/libarcane/edge/client_transport_websocket.go
+++ b/backend/pkg/libarcane/edge/client_transport_websocket.go
@@ -19,8 +19,6 @@ func (c *TunnelClient) connectAndServeWebSocket(ctx context.Context) error {
if managerWSURL == "" {
return errors.New("manager WebSocket URL is empty")
}
- c.managerURL = managerWSURL
-
transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
if strings.HasPrefix(strings.ToLower(managerWSURL), "wss://") {
tlsConfig, err := buildManagerClientTLSConfigInternal(c.cfg)
diff --git a/backend/pkg/libarcane/edge/registry.go b/backend/pkg/libarcane/edge/registry.go
index 30e212d87..861d0bf96 100644
--- a/backend/pkg/libarcane/edge/registry.go
+++ b/backend/pkg/libarcane/edge/registry.go
@@ -1,11 +1,14 @@
package edge
import (
- "github.com/samber/mo"
-
+ "context"
"log/slog"
"sync"
"time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/samber/mo"
)
// NewAgentTunnelWithConn creates a new agent tunnel from a transport-agnostic connection.
@@ -94,78 +97,105 @@ func (t *AgentTunnel) CloseWithReason(reason string) error {
// NewTunnelRegistry creates a new tunnel registry
func NewTunnelRegistry() *TunnelRegistry {
return &TunnelRegistry{
- tunnels: make(map[string]*AgentTunnel),
+ tunnels: actors.NewStateMap[string, *AgentTunnel](),
}
}
+// NewActorTunnelRegistry creates a registry owned by the shared actor runtime.
+func NewActorTunnelRegistry(ctx context.Context, runtime *actors.Runtime) (*TunnelRegistry, error) {
+ tunnels, err := actors.NewActorStateMap[string, *AgentTunnel](ctx, runtime, "edge", "tunnel-registry", 3)
+ if err != nil {
+ return nil, err
+ }
+ return &TunnelRegistry{tunnels: tunnels}, nil
+}
+
// Get retrieves a tunnel by environment ID
func (r *TunnelRegistry) Get(envID string) mo.Option[*AgentTunnel] {
- r.mu.RLock()
- defer r.mu.RUnlock()
- tunnel, ok := r.tunnels[envID]
+ tunnel, ok := r.tunnels.Get(envID)
return mo.TupleToOption(tunnel, ok)
}
// Register adds a tunnel to the registry, closing any existing tunnel for the same env
func (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {
- r.mu.Lock()
- defer r.mu.Unlock()
-
- // Close existing tunnel if any
- if existing, ok := r.tunnels[envID]; ok {
+ if r == nil || tunnel == nil {
+ slog.Error("Failed to register edge agent tunnel", "environment_id", envID, "error", "tunnel is required")
+ return
+ }
+ previous, replaced, err := r.tunnels.Store(context.Background(), "register edge tunnel", envID, tunnel)
+ if err != nil {
+ slog.Error("Failed to register edge agent tunnel", "environment_id", envID, "error", err)
+ return
+ }
+ if replaced && previous != tunnel {
slog.Info("Replacing existing edge tunnel")
- _ = existing.CloseWithReason("")
+ _ = previous.CloseWithReason("")
}
-
- r.tunnels[envID] = tunnel
slog.Info("Edge agent tunnel registered")
}
+type registerSessionResultInternal struct {
+ accepted bool
+ drainPrevious bool
+ reason string
+ previous *AgentTunnel
+}
+
// RegisterSession adds a tunnel with duplicate-session handling.
-func (r *TunnelRegistry) RegisterSession(tunnel *AgentTunnel, staleAfter time.Duration) (accepted bool, drainPrevious bool, reason string) {
+func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunnel, staleAfter time.Duration) (bool, bool, string, error) {
if r == nil || tunnel == nil {
- return false, false, "tunnel is required"
+ return false, false, "tunnel is required", nil
}
- r.mu.Lock()
- defer r.mu.Unlock()
-
envID := tunnel.EnvironmentID
if envID == "" {
- return false, false, "environment ID is required"
+ return false, false, "environment ID is required", nil
}
- if existing, ok := r.tunnels[envID]; ok && existing != nil {
- if existing == tunnel {
- return true, false, ""
- }
-
- existingStale := staleAfter > 0 && time.Since(existing.GetLastHeartbeat()) > staleAfter
- sameAgentInstance := existing.AgentInstance != "" && tunnel.AgentInstance != "" && existing.AgentInstance == tunnel.AgentInstance
+ result, err := actors.ApplyStateMap(ctx, r.tunnels, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResultInternal, bool, error) {
+ var result registerSessionResultInternal
+ if existing := tunnels[envID]; existing != nil {
+ if existing == tunnel {
+ result.accepted = true
+ return result, false, nil
+ }
- if existing.Conn == nil || existing.Conn.IsClosed() || existingStale || sameAgentInstance {
- if sameAgentInstance {
- drainPrevious = true
+ existingMetadata := existing.MetadataSnapshot()
+ tunnelMetadata := tunnel.MetadataSnapshot()
+ existingStale := staleAfter > 0 && time.Since(existingMetadata.LastHeartbeat) > staleAfter
+ sameAgentInstance := existingMetadata.AgentInstance != "" && tunnelMetadata.AgentInstance != "" && existingMetadata.AgentInstance == tunnelMetadata.AgentInstance
+ if existing.Conn != nil && !existing.Conn.IsClosed() && !existingStale && !sameAgentInstance {
+ result.reason = "another edge agent session is already active"
+ return result, false, nil
}
- _ = existing.CloseWithReason("replaced by newer edge tunnel session")
- } else {
- return false, false, "another edge agent session is already active"
+ result.previous = existing
+ result.drainPrevious = sameAgentInstance
}
- }
- r.tunnels[envID] = tunnel
- slog.Info("Edge agent tunnel registered", "environment_id", envID, "session_id", tunnel.SessionID, "security_mode", tunnel.SecurityMode)
- return true, drainPrevious, ""
+ tunnels[envID] = tunnel
+ result.accepted = true
+ return result, true, nil
+ })
+ if err != nil {
+ return false, false, "", err
+ }
+ if result.previous != nil {
+ _ = result.previous.CloseWithReason("replaced by newer edge tunnel session")
+ }
+ metadata := tunnel.MetadataSnapshot()
+ slog.Info("Edge agent tunnel registered", "environment_id", envID, "session_id", metadata.SessionID, "security_mode", metadata.SecurityMode)
+ return result.accepted, result.drainPrevious, result.reason, nil
}
// Unregister removes a tunnel from the registry
func (r *TunnelRegistry) Unregister(envID string) {
- r.mu.Lock()
- defer r.mu.Unlock()
-
- if tunnel, ok := r.tunnels[envID]; ok {
+ tunnel, removed, err := r.tunnels.Remove(context.Background(), "unregister edge tunnel", envID)
+ if err != nil {
+ slog.Error("Failed to unregister edge agent tunnel", "environment_id", envID, "error", err)
+ return
+ }
+ if removed {
_ = tunnel.CloseWithReason("")
- delete(r.tunnels, envID)
slog.Info("Edge agent tunnel unregistered")
}
}
@@ -173,48 +203,76 @@ func (r *TunnelRegistry) Unregister(envID string) {
// UnregisterCurrent removes the active tunnel only when it still matches the
// provided tunnel reference. It reports whether the tunnel was removed and
// whether another active session for the environment is already present.
-func (r *TunnelRegistry) UnregisterCurrent(envID string, current *AgentTunnel) (removed bool, activeReplacement bool) {
+func (r *TunnelRegistry) UnregisterCurrent(ctx context.Context, envID string, current *AgentTunnel) (bool, bool) {
if r == nil || current == nil {
return false, false
}
- r.mu.Lock()
- defer r.mu.Unlock()
-
- existing, ok := r.tunnels[envID]
- if !ok || existing != current {
- if ok && existing != nil && existing.Conn != nil && !existing.Conn.IsClosed() {
- return false, true
+ type resultInternal struct {
+ removed bool
+ activeReplacement bool
+ }
+ result, err := actors.ApplyStateMap(ctx, r.tunnels, "unregister current edge tunnel", func(tunnels map[string]*AgentTunnel) (resultInternal, bool, error) {
+ var result resultInternal
+ existing := tunnels[envID]
+ if existing != current {
+ result.activeReplacement = existing != nil && existing.Conn != nil && !existing.Conn.IsClosed()
+ return result, false, nil
}
+ delete(tunnels, envID)
+ result.removed = true
+ return result, true, nil
+ })
+ if err != nil {
+ slog.ErrorContext(ctx, "Failed to unregister current edge agent tunnel", "environment_id", envID, "error", err)
return false, false
}
+ if !result.removed {
+ return false, result.activeReplacement
+ }
- _ = existing.CloseWithReason("")
- delete(r.tunnels, envID)
- slog.Info("Edge agent tunnel unregistered", "environment_id", envID, "session_id", current.SessionID)
+ _ = current.CloseWithReason("")
+ metadata := current.MetadataSnapshot()
+ slog.Info("Edge agent tunnel unregistered", "environment_id", envID, "session_id", metadata.SessionID)
return true, false
}
// CleanupStale removes tunnels that haven't had a heartbeat within the given duration.
-func (r *TunnelRegistry) CleanupStale(maxAge time.Duration) []*AgentTunnel {
- r.mu.Lock()
- defer r.mu.Unlock()
-
+func (r *TunnelRegistry) CleanupStale(ctx context.Context, maxAge time.Duration) []*AgentTunnel {
now := time.Now()
- removed := make([]*AgentTunnel, 0)
-
- for envID, tunnel := range r.tunnels {
- if now.Sub(tunnel.GetLastHeartbeat()) > maxAge {
- slog.Warn("Removing stale edge tunnel", "last_heartbeat", tunnel.GetLastHeartbeat())
- _ = tunnel.CloseWithReason("edge tunnel heartbeat expired")
- delete(r.tunnels, envID)
- removed = append(removed, tunnel)
+ removed, err := r.tunnels.RemoveWhere(ctx, "clean up stale edge tunnels", func(_ string, tunnel *AgentTunnel) bool {
+ return tunnel == nil || now.Sub(tunnel.GetLastHeartbeat()) > maxAge
+ })
+ if err != nil {
+ slog.ErrorContext(ctx, "Failed to clean up stale edge tunnels", "error", err)
+ return nil
+ }
+ for _, tunnel := range removed {
+ if tunnel == nil {
+ continue
}
+ slog.Warn("Removing stale edge tunnel", "last_heartbeat", tunnel.GetLastHeartbeat())
+ _ = tunnel.CloseWithReason("edge tunnel heartbeat expired")
}
return removed
}
+// Stop drains registry mutations, closes active tunnels, and joins its actor.
+func (r *TunnelRegistry) Stop(ctx context.Context) error {
+ fallback := r.tunnels.Values()
+ removed, err := r.tunnels.Drain(ctx, "stop edge tunnel registry")
+ if err != nil {
+ removed = fallback
+ }
+ for _, tunnel := range removed {
+ if tunnel != nil {
+ err = errors.Combine(err, tunnel.CloseWithReason("edge tunnel registry stopped"))
+ }
+ }
+ return errors.Combine(err, r.tunnels.Stop(ctx))
+}
+
var (
defaultRegistryMu sync.RWMutex
defaultRegistry = NewTunnelRegistry()
@@ -238,3 +296,12 @@ func SetDefaultRegistry(registry *TunnelRegistry) {
defaultRegistry = registry
defaultRegistryMu.Unlock()
}
+
+// ClearDefaultRegistry replaces registry only when it is still the active default.
+func ClearDefaultRegistry(registry *TunnelRegistry) {
+ defaultRegistryMu.Lock()
+ if defaultRegistry == registry {
+ defaultRegistry = NewTunnelRegistry()
+ }
+ defaultRegistryMu.Unlock()
+}
diff --git a/backend/pkg/libarcane/edge/registry_test.go b/backend/pkg/libarcane/edge/registry_test.go
index 1bb96949f..de442d8da 100644
--- a/backend/pkg/libarcane/edge/registry_test.go
+++ b/backend/pkg/libarcane/edge/registry_test.go
@@ -1,6 +1,7 @@
package edge
import (
+ "context"
"net/http"
"net/http/httptest"
"strings"
@@ -8,10 +9,45 @@ import (
"time"
"github.com/coder/websocket"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
)
+type controlledCloseTunnelConn struct {
+ closeStarted chan struct{}
+ releaseClose chan struct{}
+ closed chan struct{}
+}
+
+func (c *controlledCloseTunnelConn) Send(*TunnelMessage) error { return nil }
+func (c *controlledCloseTunnelConn) Receive() (*TunnelMessage, error) { return nil, nil }
+func (c *controlledCloseTunnelConn) IsExpectedReceiveError(error) bool { return false }
+func (c *controlledCloseTunnelConn) Transport() string { return EdgeTransportWebSocket }
+func (c *controlledCloseTunnelConn) IsClosed() bool {
+ select {
+ case <-c.closed:
+ return true
+ default:
+ return false
+ }
+}
+func (c *controlledCloseTunnelConn) Close() error {
+ select {
+ case <-c.closeStarted:
+ default:
+ close(c.closeStarted)
+ }
+ <-c.releaseClose
+ select {
+ case <-c.closed:
+ default:
+ close(c.closed)
+ }
+ return nil
+}
+
func createTestConn(t *testing.T) *websocket.Conn {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, nil)
@@ -91,7 +127,8 @@ func TestTunnelRegistry_RegisterSessionRejectsCompetingAgent(t *testing.T) {
tunnel1.AgentInstance = "agent-a"
tunnel1.SessionID = "session-a"
- accepted, drainPrevious, reason := r.RegisterSession(tunnel1, TunnelStaleTimeout)
+ accepted, drainPrevious, reason, err := r.RegisterSession(t.Context(), tunnel1, TunnelStaleTimeout)
+ require.NoError(t, err)
assert.True(t, accepted)
assert.False(t, drainPrevious)
assert.Equal(t, "", reason)
@@ -102,7 +139,8 @@ func TestTunnelRegistry_RegisterSessionRejectsCompetingAgent(t *testing.T) {
tunnel2.AgentInstance = "agent-b"
tunnel2.SessionID = "session-b"
- accepted, drainPrevious, reason = r.RegisterSession(tunnel2, TunnelStaleTimeout)
+ accepted, drainPrevious, reason, err = r.RegisterSession(t.Context(), tunnel2, TunnelStaleTimeout)
+ require.NoError(t, err)
assert.False(t, accepted)
assert.False(t, drainPrevious)
assert.Equal(t, "another edge agent session is already active", reason)
@@ -122,7 +160,8 @@ func TestTunnelRegistry_RegisterSessionReplacesSameAgentInstance(t *testing.T) {
tunnel1 := newWebSocketAgentTunnel(envID, conn1)
tunnel1.AgentInstance = "agent-a"
tunnel1.SessionID = "session-a"
- accepted, drainPrevious, reason := r.RegisterSession(tunnel1, TunnelStaleTimeout)
+ accepted, drainPrevious, reason, err := r.RegisterSession(t.Context(), tunnel1, TunnelStaleTimeout)
+ require.NoError(t, err)
assert.True(t, accepted)
assert.False(t, drainPrevious)
assert.Equal(t, "", reason)
@@ -132,7 +171,8 @@ func TestTunnelRegistry_RegisterSessionReplacesSameAgentInstance(t *testing.T) {
tunnel2 := newWebSocketAgentTunnel(envID, conn2)
tunnel2.AgentInstance = "agent-a"
tunnel2.SessionID = "session-b"
- accepted, drainPrevious, reason = r.RegisterSession(tunnel2, TunnelStaleTimeout)
+ accepted, drainPrevious, reason, err = r.RegisterSession(t.Context(), tunnel2, TunnelStaleTimeout)
+ require.NoError(t, err)
assert.True(t, accepted)
assert.True(t, drainPrevious)
assert.Equal(t, "", reason)
@@ -143,6 +183,80 @@ func TestTunnelRegistry_RegisterSessionReplacesSameAgentInstance(t *testing.T) {
assert.Equal(t, tunnel2, got)
}
+func TestActorTunnelRegistryPublishesReplacementBeforeClosingPreviousInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ registry, err := NewActorTunnelRegistry(t.Context(), runtime)
+ require.NoError(t, err)
+
+ previousConn := &controlledCloseTunnelConn{
+ closeStarted: make(chan struct{}),
+ releaseClose: make(chan struct{}),
+ closed: make(chan struct{}),
+ }
+ previous := NewAgentTunnelWithConn("env-actor-registry", previousConn)
+ previous.AgentInstance = "agent-a"
+ accepted, _, reason, err := registry.RegisterSession(t.Context(), previous, TunnelStaleTimeout)
+ require.NoError(t, err)
+ require.True(t, accepted, reason)
+
+ replacementConn := &controlledCloseTunnelConn{
+ closeStarted: make(chan struct{}),
+ releaseClose: make(chan struct{}),
+ closed: make(chan struct{}),
+ }
+ close(replacementConn.releaseClose)
+ replacement := NewAgentTunnelWithConn("env-actor-registry", replacementConn)
+ replacement.AgentInstance = "agent-a"
+ registered := make(chan bool, 1)
+ go func() {
+ accepted, _, _, registerErr := registry.RegisterSession(t.Context(), replacement, TunnelStaleTimeout)
+ require.NoError(t, registerErr)
+ registered <- accepted
+ }()
+
+ select {
+ case <-previousConn.closeStarted:
+ case <-time.After(time.Second):
+ t.Fatal("replacement did not begin closing the previous tunnel")
+ }
+ active, ok := registry.Get("env-actor-registry").Get()
+ require.True(t, ok)
+ require.Same(t, replacement, active)
+ close(previousConn.releaseClose)
+ require.True(t, <-registered)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, registry.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
+func TestActorTunnelRegistryReportsStoppedExecutorAsInfrastructureFailureInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ registry, err := NewActorTunnelRegistry(t.Context(), runtime)
+ require.NoError(t, err)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, registry.Stop(stopCtx))
+
+ tunnel := NewAgentTunnelWithConn("env-stopped-registry", &controlledCloseTunnelConn{
+ closeStarted: make(chan struct{}),
+ releaseClose: make(chan struct{}),
+ closed: make(chan struct{}),
+ })
+ accepted, drainPrevious, reason, err := registry.RegisterSession(t.Context(), tunnel, TunnelStaleTimeout)
+ require.Error(t, err)
+ require.False(t, accepted)
+ require.False(t, drainPrevious)
+ require.Empty(t, reason)
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
+
func TestTunnelRegistry_CleanupStale(t *testing.T) {
r := NewTunnelRegistry()
envID := "env-1"
@@ -159,7 +273,7 @@ func TestTunnelRegistry_CleanupStale(t *testing.T) {
r.Register(envID, tunnel)
// Cleanup
- removed := r.CleanupStale(5 * time.Minute)
+ removed := r.CleanupStale(t.Context(), 5*time.Minute)
require.Len(t, removed, 1)
assert.Same(t, tunnel, removed[0])
diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.go
index 61cada4f6..a473b690a 100644
--- a/backend/pkg/libarcane/edge/server.go
+++ b/backend/pkg/libarcane/edge/server.go
@@ -378,7 +378,16 @@ func tokenFromHeadersWithSourceInternal(req *http.Request) (string, string) {
}
func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx context.Context, tunnel *AgentTunnel) {
- accepted, drainPrevious, rejectReason := s.registry.RegisterSession(tunnel, TunnelStaleTimeout)
+ accepted, drainPrevious, rejectReason, err := s.registry.RegisterSession(callbackCtx, tunnel, TunnelStaleTimeout)
+ if err != nil {
+ slog.ErrorContext(ctx, "Failed to register edge agent session",
+ "environment_id", tunnel.EnvironmentID,
+ "agent_instance_id", tunnel.AgentInstance,
+ "error", err,
+ )
+ _ = tunnel.CloseWithReason("edge agent session registration unavailable")
+ return
+ }
if !accepted {
slog.WarnContext(ctx, "Rejected duplicate edge agent session",
"environment_id", tunnel.EnvironmentID,
@@ -421,7 +430,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx co
}); err != nil {
slog.WarnContext(ctx, "Failed to send register response", "environment_id", tunnel.EnvironmentID, "error", err)
_ = tunnel.CloseWithReason("")
- removed, active := s.registry.UnregisterCurrent(tunnel.EnvironmentID, tunnel)
+ removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)
if removed && !active {
s.updateConnectionStatusInternal(callbackCtx, tunnel, false)
}
@@ -431,7 +440,7 @@ func (s *TunnelServer) manageConnectedTunnel(ctx context.Context, callbackCtx co
s.updateConnectionStatusInternal(callbackCtx, tunnel, true)
defer func() {
- removed, active := s.registry.UnregisterCurrent(tunnel.EnvironmentID, tunnel)
+ removed, active := s.registry.UnregisterCurrent(callbackCtx, tunnel.EnvironmentID, tunnel)
if !removed {
return
}
@@ -587,7 +596,7 @@ func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
- removed := s.registry.CleanupStale(TunnelStaleTimeout)
+ removed := s.registry.CleanupStale(ctx, TunnelStaleTimeout)
for _, tunnel := range removed {
s.updateConnectionStatusInternal(context.WithoutCancel(ctx), tunnel, false)
}
diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.go
index 0c0f279c9..93d629539 100644
--- a/backend/pkg/libarcane/edge/server_test.go
+++ b/backend/pkg/libarcane/edge/server_test.go
@@ -16,9 +16,11 @@ import (
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/labstack/echo/v5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
@@ -644,3 +646,25 @@ func TestTunnelServer_ManageConnectedTunnel_RegistersBeforeSendingGRPCRegisterRe
_, ok := server.registry.Get(envID).Get()
assert.False(t, ok)
}
+
+func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCancellationInternal(t *testing.T) {
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ registry, err := NewActorTunnelRegistry(t.Context(), runtime)
+ require.NoError(t, err)
+ server := NewTunnelServerWithRegistry(registry, nil, nil)
+ tunnel := NewAgentTunnelWithConn("env-cancelled-disconnect", &registerResponseOrderConn{recvErr: context.Canceled})
+
+ connectionCtx, cancelConnection := context.WithCancel(context.Background())
+ cancelConnection()
+ server.manageConnectedTunnel(connectionCtx, context.WithoutCancel(connectionCtx), tunnel)
+
+ _, ok := registry.Get(tunnel.EnvironmentID).Get()
+ require.False(t, ok)
+
+ stopCtx, cancelStop := context.WithTimeout(context.Background(), time.Second)
+ defer cancelStop()
+ require.NoError(t, registry.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/pkg/libarcane/edge/tunnel.go b/backend/pkg/libarcane/edge/tunnel.go
index f425eb7cd..2986e597a 100644
--- a/backend/pkg/libarcane/edge/tunnel.go
+++ b/backend/pkg/libarcane/edge/tunnel.go
@@ -368,11 +368,21 @@ func (t *GRPCAgentTunnelConn) Close() error {
return nil
}
+ // Preserve a clean half-close unless an in-flight send must be cancelled first.
+ locked := t.mu.TryLock()
+ if !locked {
+ if t.cancel != nil {
+ t.cancel()
+ }
+ t.mu.Lock()
+ }
+ defer t.mu.Unlock()
+
var err error
if t.stream != nil {
err = t.stream.CloseSend()
}
- if t.cancel != nil {
+ if locked && t.cancel != nil {
t.cancel()
}
return err
diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.go
index be8aa4f1e..c5a9b5ccc 100644
--- a/backend/pkg/libarcane/edge/types.go
+++ b/backend/pkg/libarcane/edge/types.go
@@ -9,6 +9,7 @@ import (
"time"
"github.com/coder/websocket"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"google.golang.org/grpc"
)
@@ -47,9 +48,7 @@ type TunnelClient struct {
managerURL string
managerGRPCAddr string
localPort string // Port the agent is running on locally
- httpClient *http.Client
conn atomic.Pointer[connBox]
- stopCh chan struct{}
requestTimeout time.Duration
activeStreams sync.Map // map[string]*activeWSStream
requestTransfers sync.Map // map[string]*commandRequestTransfer
@@ -57,8 +56,11 @@ type TunnelClient struct {
preferWebSocketUntil time.Time
grpcFailureStreak int
agentInstanceID string
- sessionID string
- managerCapabilities []string
+ registration actors.Snapshot[clientRegistrationInternal]
+}
+
+type clientRegistrationInternal struct {
+ sessionID string
}
// connBox wraps the active TunnelConnection so it can be swapped atomically on
@@ -106,8 +108,9 @@ type streamingResponseRecorder struct {
}
type pollManagedTunnelSession struct {
- cancel context.CancelFunc
- done chan error
+ cancel context.CancelFunc
+ done chan error
+ stopping bool
}
type CommandRequest struct {
@@ -213,8 +216,7 @@ type AgentTunnel struct {
// TunnelRegistry manages active edge agent tunnel connections
type TunnelRegistry struct {
- tunnels map[string]*AgentTunnel // environmentID -> tunnel
- mu sync.RWMutex
+ tunnels *actors.StateMap[string, *AgentTunnel]
}
type internalTunnelRequestContextKey struct{}
diff --git a/backend/pkg/libarcane/settings.go b/backend/pkg/libarcane/settings.go
index 0644a74b4..7eff3522a 100644
--- a/backend/pkg/libarcane/settings.go
+++ b/backend/pkg/libarcane/settings.go
@@ -44,8 +44,9 @@ var cronSettingKeys = []string{
var cronParser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
-func IsTimeoutSettingKey(key string) bool {
- return slices.Contains(timeoutSettingKeys, key)
+// TimeoutSettingKeys returns the settings propagated to remote environments.
+func TimeoutSettingKeys() []string {
+ return slices.Clone(timeoutSettingKeys)
}
func IsCronSettingKey(key string) bool {
diff --git a/backend/pkg/scheduler/actor_test.go b/backend/pkg/scheduler/actor_test.go
new file mode 100644
index 000000000..bec3b5573
--- /dev/null
+++ b/backend/pkg/scheduler/actor_test.go
@@ -0,0 +1,65 @@
+package scheduler
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/services"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
+)
+
+func newTestAdmissionGateInternal(t *testing.T) *actors.Gate[actors.AdmissionKey] {
+ t.Helper()
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "scheduler-test-admission", t.Name())
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+ })
+ return gate
+}
+
+func newJobSchedulerForTestInternal(t testing.TB, ctx context.Context, location *time.Location) *jobSchedulerInternal {
+ t.Helper()
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ created, err := NewJobScheduler(ctx, runtime, location)
+ require.NoError(t, err)
+ scheduler := created.(*jobSchedulerInternal)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, scheduler.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+ })
+ return scheduler
+}
+
+func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*services.SettingsService, error) {
+ t.Helper()
+ lifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
+ require.NoError(t, err)
+ effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, executor.Stop(stopCtx))
+ require.NoError(t, effects.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+ })
+ return services.NewSettingsService(ctx, db, executor, effects)
+}
diff --git a/backend/pkg/scheduler/analytics_job_test.go b/backend/pkg/scheduler/analytics_job_test.go
index 3fb806089..576f26ec4 100644
--- a/backend/pkg/scheduler/analytics_job_test.go
+++ b/backend/pkg/scheduler/analytics_job_test.go
@@ -30,7 +30,7 @@ func setupAnalyticsStateServicesInternal(t *testing.T) (*database.DB, *services.
require.NoError(t, db.AutoMigrate(&models.KVEntry{}))
wrappedDB := &database.DB{DB: db}
- settingsService, err := services.NewSettingsService(ctx, wrappedDB)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, wrappedDB)
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "instanceId", "test-instance"))
@@ -157,7 +157,7 @@ func TestAnalyticsJob_Run_SkipsWithinHeartbeatWindowAfterRestart(t *testing.T) {
job.now = func() time.Time { return firstAttemptAt }
job.Run(ctx)
- reloadedSettingsService, err := services.NewSettingsService(ctx, wrappedDB)
+ reloadedSettingsService, err := newSettingsServiceForTestInternal(t, ctx, wrappedDB)
require.NoError(t, err)
restartedJob := NewAnalyticsJob(reloadedSettingsService, services.NewKVService(wrappedDB), server.Client(), cfg)
restartedJob.heartbeatURL = server.URL
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index 116c3d191..6dbeda741 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -6,14 +6,15 @@ import (
"slices"
"strings"
"sync"
- "sync/atomic"
"time"
+ "emperror.dev/errors"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
"github.com/robfig/cron/v3"
"golang.org/x/sync/errgroup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
@@ -24,6 +25,7 @@ import (
const AutoHealJobName = "auto-heal"
const autoHealInspectConcurrency = 4
+const autoHealAdmissionScopeInternal = "auto-heal"
// restartRecord tracks restart timestamps for a single container.
type restartRecord struct {
@@ -35,11 +37,7 @@ type AutoHealJob struct {
settingsService *services.SettingsService
eventService *services.EventService
notificationService *services.NotificationService
-
- // running guards against overlapping cron ticks: inspecting every candidate
- // can outlast the (as low as 30s) interval, and stacked runs re-evaluate the
- // same unhealthy containers concurrently.
- running atomic.Bool
+ admissionGate *actors.Gate[actors.AdmissionKey]
mu sync.Mutex
restarts map[string]*restartRecord
@@ -59,14 +57,19 @@ func NewAutoHealJob(
settingsService *services.SettingsService,
eventService *services.EventService,
notificationService *services.NotificationService,
-) *AutoHealJob {
+ admissionGate *actors.Gate[actors.AdmissionKey],
+) (*AutoHealJob, error) {
+ if admissionGate == nil {
+ return nil, errors.New("auto-heal admission gate unavailable")
+ }
return &AutoHealJob{
dockerClientService: dockerClientService,
settingsService: settingsService,
eventService: eventService,
notificationService: notificationService,
+ admissionGate: admissionGate,
restarts: make(map[string]*restartRecord),
- }
+ }, nil
}
func (j *AutoHealJob) Name() string {
@@ -99,11 +102,16 @@ func (j *AutoHealJob) Run(ctx context.Context) {
return
}
- if !j.running.CompareAndSwap(false, true) {
+ lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoHealAdmissionScopeInternal})
+ if err != nil {
+ slog.ErrorContext(ctx, "auto-heal admission failed", "error", err)
+ return
+ }
+ if !admitted {
slog.WarnContext(ctx, "auto-heal run still in progress; skipping overlapping run")
return
}
- defer j.running.Store(false)
+ defer lease.Release()
dockerClient, err := j.getDockerClientInternal(ctx)
if err != nil {
diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.go
index 85345de3f..dcd6e9766 100644
--- a/backend/pkg/scheduler/auto_heal_job_test.go
+++ b/backend/pkg/scheduler/auto_heal_job_test.go
@@ -8,7 +8,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/models"
- "github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/libtnb/sqlite"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
@@ -133,7 +132,8 @@ func TestAutoHeal_Schedule_Default(t *testing.T) {
func TestAutoHeal_ShouldSchedule(t *testing.T) {
ctx := context.Background()
_, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
- job := NewAutoHealJob(nil, settingsSvc, nil, nil)
+ job, err := NewAutoHealJob(nil, settingsSvc, nil, nil, newTestAdmissionGateInternal(t))
+ require.NoError(t, err)
require.False(t, job.ShouldSchedule(ctx))
@@ -164,12 +164,13 @@ func TestAutoHeal_Run_UsesBoundedConcurrency(t *testing.T) {
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))
- settingsSvc, err := services.NewSettingsService(ctx, &database.DB{DB: db})
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, &database.DB{DB: db})
require.NoError(t, err)
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", true))
require.NoError(t, settingsSvc.SetStringSetting(ctx, "autoHealExcludedContainers", "skip-me"))
- job := NewAutoHealJob(nil, settingsSvc, nil, nil)
+ job, err := NewAutoHealJob(nil, settingsSvc, nil, nil, newTestAdmissionGateInternal(t))
+ require.NoError(t, err)
job.getDockerClient = func() (*client.Client, error) { return nil, nil }
job.listContainers = func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {
return []container.Summary{
@@ -219,3 +220,42 @@ func TestAutoHeal_Run_UsesBoundedConcurrency(t *testing.T) {
require.LessOrEqual(t, atomic.LoadInt32(&maxConcurrent), int32(autoHealInspectConcurrency))
require.Equal(t, int32(4), atomic.LoadInt32(&restarts))
}
+
+func TestAutoHealJob_OverlappingRunIsSkippedInternal(t *testing.T) {
+ ctx := context.Background()
+ _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
+ require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", true))
+
+ job, err := NewAutoHealJob(nil, settingsSvc, nil, nil, newTestAdmissionGateInternal(t))
+ require.NoError(t, err)
+ job.getDockerClient = func() (*client.Client, error) { return nil, nil }
+ started := make(chan struct{}, 1)
+ release := make(chan struct{})
+ var calls atomic.Int32
+ job.listContainers = func(context.Context, *client.Client) ([]container.Summary, error) {
+ calls.Add(1)
+ select {
+ case started <- struct{}{}:
+ default:
+ }
+ <-release
+ return nil, nil
+ }
+
+ firstDone := make(chan struct{})
+ go func() {
+ job.Run(ctx)
+ close(firstDone)
+ }()
+ <-started
+
+ job.Run(ctx)
+ require.Equal(t, int32(1), calls.Load())
+
+ close(release)
+ select {
+ case <-firstDone:
+ case <-time.After(time.Second):
+ t.Fatal("first auto-heal run did not finish")
+ }
+}
diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.go
index c1d5c498c..50cdcf331 100644
--- a/backend/pkg/scheduler/auto_update_job.go
+++ b/backend/pkg/scheduler/auto_update_job.go
@@ -3,12 +3,15 @@ package scheduler
import (
"context"
"log/slog"
- "sync/atomic"
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/types/v2/updater"
)
+const autoUpdateAdmissionScopeInternal = "auto-update"
+
// pendingUpdateApplierInternal is the slice of UpdaterService the job needs,
// kept as an interface so the overlap guard is testable with a fake.
type pendingUpdateApplierInternal interface {
@@ -18,17 +21,18 @@ type pendingUpdateApplierInternal interface {
type AutoUpdateJob struct {
updaterService pendingUpdateApplierInternal
settingsService *services.SettingsService
- // running guards against overlapping cron ticks: an update-all run can
- // outlast its schedule interval, and stacked runs exhaust the shared
- // activity slots.
- running atomic.Bool
+ admissionGate *actors.Gate[actors.AdmissionKey]
}
-func NewAutoUpdateJob(updaterService *services.UpdaterService, settingsService *services.SettingsService) *AutoUpdateJob {
+func NewAutoUpdateJob(updaterService *services.UpdaterService, settingsService *services.SettingsService, admissionGate *actors.Gate[actors.AdmissionKey]) (*AutoUpdateJob, error) {
+ if admissionGate == nil {
+ return nil, errors.New("auto-update admission gate unavailable")
+ }
return &AutoUpdateJob{
updaterService: updaterService,
settingsService: settingsService,
- }
+ admissionGate: admissionGate,
+ }, nil
}
func (j *AutoUpdateJob) Name() string {
@@ -58,11 +62,16 @@ func (j *AutoUpdateJob) Run(ctx context.Context) {
return
}
- if !j.running.CompareAndSwap(false, true) {
+ lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})
+ if err != nil {
+ slog.ErrorContext(ctx, "auto-update admission failed", "error", err)
+ return
+ }
+ if !admitted {
slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")
return
}
- defer j.running.Store(false)
+ defer lease.Release()
slog.InfoContext(ctx, "auto-update run started")
diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.go
index f370aa85e..11dc38a5c 100644
--- a/backend/pkg/scheduler/auto_update_job_test.go
+++ b/backend/pkg/scheduler/auto_update_job_test.go
@@ -13,7 +13,8 @@ import (
func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T) {
ctx := context.Background()
_, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)
- job := NewAutoUpdateJob(nil, settingsSvc)
+ job, err := NewAutoUpdateJob(nil, settingsSvc, newTestAdmissionGateInternal(t))
+ require.NoError(t, err)
require.False(t, job.ShouldSchedule(ctx))
@@ -52,7 +53,7 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {
started: make(chan struct{}, 1),
release: make(chan struct{}),
}
- job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc}
+ job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc, admissionGate: newTestAdmissionGateInternal(t)}
firstDone := make(chan struct{})
go func() {
diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.go
index d17341d5e..40ffb3841 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job.go
@@ -4,9 +4,10 @@ import (
"context"
"log/slog"
"runtime"
- "sync"
"time"
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
"github.com/getarcaneapp/arcane/backend/v2/pkg/fswatch"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
@@ -17,160 +18,53 @@ type FilesystemWatcherJob struct {
templateService *services.TemplateService
settingsService *services.SettingsService
projectScanDepth int
- projectsWatcher *fswatch.Watcher
- templatesWatcher *fswatch.Watcher
- mu sync.Mutex
+ lifecycleCtx context.Context
+ projectsWatcher *actors.Resource[*fswatch.Watcher]
+ templatesWatcher *actors.Resource[*fswatch.Watcher]
}
func NewFilesystemWatcherJob(
+ ctx context.Context,
+ actorRuntime *actors.Runtime,
projectService *services.ProjectService,
templateService *services.TemplateService,
settingsService *services.SettingsService,
projectScanDepth int,
-) *FilesystemWatcherJob {
+) (*FilesystemWatcherJob, error) {
+ projectsWatcher, err := actors.NewResource(ctx, actorRuntime, "filesystem-watcher", "projects", 3, (*fswatch.Watcher).Stop)
+ if err != nil {
+ return nil, err
+ }
+ templatesWatcher, err := actors.NewResource(ctx, actorRuntime, "filesystem-watcher", "templates", 3, (*fswatch.Watcher).Stop)
+ if err != nil {
+ return nil, errors.Combine(err, projectsWatcher.Stop(ctx))
+ }
return &FilesystemWatcherJob{
projectService: projectService,
templateService: templateService,
settingsService: settingsService,
projectScanDepth: projectScanDepth,
- }
-}
-
-func RegisterFilesystemWatcherJob(ctx context.Context, projectService *services.ProjectService, templateService *services.TemplateService, settingsService *services.SettingsService, projectScanDepth int) (*FilesystemWatcherJob, error) {
- job := NewFilesystemWatcherJob(projectService, templateService, settingsService, projectScanDepth)
-
- go func() {
- if err := job.Start(ctx); err != nil {
- slog.ErrorContext(ctx, "Filesystem watcher failed", "error", err)
- }
- }()
-
- slog.InfoContext(ctx, "Filesystem watcher job registered")
- return job, nil
+ lifecycleCtx: ctx,
+ projectsWatcher: projectsWatcher,
+ templatesWatcher: templatesWatcher,
+ }, nil
}
func (j *FilesystemWatcherJob) Start(ctx context.Context) error {
- settings, err := j.settingsService.GetSettings(ctx)
- if err != nil {
- return err
- }
- projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)
- if err != nil {
- return err
- }
- followProjectSymlinks := settings.FollowProjectSymlinks.IsTrue()
- j.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)
-
- // Watchers are held locally until both are running, then published under the
- // lock: assigning to j.projectsWatcher/j.templatesWatcher as they were built
- // raced with Stop and RestartProjectsWatcher, and left every watcher created
- // on a path that later failed holding its inotify/kqueue descriptors.
- var projectsWatcher, templatesWatcher *fswatch.Watcher
- published := false
- defer func() {
- if published {
- return
- }
- // Startup failed somewhere below; release whatever was built. The job
- // never took ownership of these, so nothing else will free them.
- for _, watcher := range []*fswatch.Watcher{projectsWatcher, templatesWatcher} {
- if watcher == nil {
- continue
- }
- if stopErr := watcher.Stop(); stopErr != nil {
- slog.ErrorContext(ctx, "Failed to stop watcher after startup error", "error", stopErr)
- }
- }
- }()
-
- projectsWatcher, err = fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))
- if err != nil {
- return err
- }
-
- templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)
- if err != nil {
+ if err := j.projectsWatcher.Restart(ctx, "start projects filesystem watcher", j.startProjectsWatcherInternal); err != nil {
return err
}
-
- if j.templateService != nil {
- if directoriesOverlapInternal(projectsDirectory, templatesDir) {
- slog.ErrorContext(ctx,
- "Templates and projects directories overlap; templates watcher disabled to prevent compose files being treated as projects",
- "projectsDirectory", projectsDirectory,
- "templatesDirectory", templatesDir)
- } else {
- templatesWatcher, err = fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{
- Debounce: 3 * time.Second,
- OnChange: j.handleTemplatesChange,
- MaxDepth: 1,
- })
- if err != nil {
- return err
- }
- }
- }
-
- if err := projectsWatcher.Start(ctx); err != nil {
- return err
- }
- if templatesWatcher != nil {
- if err := templatesWatcher.Start(ctx); err != nil {
- return err
- }
- }
-
- j.mu.Lock()
- j.projectsWatcher = projectsWatcher
- j.templatesWatcher = templatesWatcher
- j.mu.Unlock()
- published = true
-
- slog.InfoContext(ctx, "Filesystem watcher started for projects directory",
- "path", projectsDirectory)
- if templatesWatcher != nil {
- slog.InfoContext(ctx, "Filesystem watcher started for templates directory",
- "path", templatesDir)
- }
-
- // Initial sync to surface pre-existing resources
- if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {
- slog.ErrorContext(ctx, "Initial project sync failed", "error", err)
+ if err := j.templatesWatcher.Restart(ctx, "start templates filesystem watcher", j.startTemplatesWatcherInternal); err != nil {
+ return errors.Combine(err, j.projectsWatcher.Clear(ctx, "clear projects watcher after template startup failure"))
}
- if j.templateService != nil {
- if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
- slog.ErrorContext(ctx, "Initial template sync failed", "error", err)
- }
- }
-
- <-ctx.Done()
-
- return j.Stop()
+ return nil
}
-func (j *FilesystemWatcherJob) Stop() error {
- j.mu.Lock()
- projectsWatcher := j.projectsWatcher
- templatesWatcher := j.templatesWatcher
- j.projectsWatcher = nil
- j.templatesWatcher = nil
- j.mu.Unlock()
-
- var firstErr error
- if projectsWatcher != nil {
- if err := projectsWatcher.Stop(); err != nil {
- firstErr = err
- }
- }
- if templatesWatcher != nil {
- if err := templatesWatcher.Stop(); err != nil && firstErr == nil {
- firstErr = err
- }
- }
- return firstErr
+func (j *FilesystemWatcherJob) Stop(ctx context.Context) error {
+ return errors.Combine(j.projectsWatcher.Stop(ctx), j.templatesWatcher.Stop(ctx))
}
-func (j *FilesystemWatcherJob) handleFilesystemChange(ctx context.Context) {
+func (j *FilesystemWatcherJob) handleFilesystemChangeInternal(ctx context.Context) {
slog.InfoContext(ctx, "Filesystem change detected, syncing projects")
if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {
@@ -185,74 +79,65 @@ func (j *FilesystemWatcherJob) handleProjectFilePathsChangedInternal(ctx context
if len(paths) == 0 || j.projectService == nil {
return
}
- j.handleFilesystemChange(ctx)
- j.projectService.HandleProjectFilesChanged(ctx, paths)
+ err := j.projectsWatcher.Do(ctx, "sync changed project files", func(workCtx context.Context, _ *fswatch.Watcher) error {
+ j.handleFilesystemChangeInternal(workCtx)
+ j.projectService.HandleProjectFilesChanged(workCtx, paths)
+ return nil
+ })
+ if err != nil && ctx.Err() == nil && !errors.Is(err, actors.ErrResourceStopped) {
+ slog.ErrorContext(ctx, "Failed to dispatch changed project files", "error", err)
+ }
}
-func (j *FilesystemWatcherJob) handleTemplatesChange(ctx context.Context) {
- slog.InfoContext(ctx, "Template directory change detected, syncing templates")
- if j.templateService == nil {
- return
- }
- if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
- slog.ErrorContext(ctx, "Failed to sync templates after filesystem change", "error", err)
- } else {
- slog.InfoContext(ctx, "Template sync completed after filesystem change")
+func (j *FilesystemWatcherJob) handleTemplatesChangeInternal(ctx context.Context) {
+ err := j.templatesWatcher.Do(ctx, "sync changed templates", func(workCtx context.Context, _ *fswatch.Watcher) error {
+ slog.InfoContext(workCtx, "Template directory change detected, syncing templates")
+ if j.templateService == nil {
+ return nil
+ }
+ if syncErr := j.templateService.SyncLocalTemplatesFromFilesystem(workCtx); syncErr != nil {
+ slog.ErrorContext(workCtx, "Failed to sync templates after filesystem change", "error", syncErr)
+ } else {
+ slog.InfoContext(workCtx, "Template sync completed after filesystem change")
+ }
+ return nil
+ })
+ if err != nil && ctx.Err() == nil && !errors.Is(err, actors.ErrResourceStopped) {
+ slog.ErrorContext(ctx, "Failed to dispatch template filesystem change", "error", err)
}
}
func (j *FilesystemWatcherJob) RestartProjectsWatcher(ctx context.Context) error {
slog.InfoContext(ctx, "Restarting projects filesystem watcher")
+ return j.projectsWatcher.Restart(ctx, "restart projects filesystem watcher", j.startProjectsWatcherInternal)
+}
- j.mu.Lock()
- oldProjectsWatcher := j.projectsWatcher
- j.projectsWatcher = nil
- j.mu.Unlock()
-
- if oldProjectsWatcher != nil {
- if err := oldProjectsWatcher.Stop(); err != nil {
- slog.WarnContext(ctx, "Failed to stop projects watcher during restart", "error", err)
- }
- }
-
- // Get fresh settings to get the new projects directory
+func (j *FilesystemWatcherJob) startProjectsWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {
settings, err := j.settingsService.GetSettings(ctx)
if err != nil {
- return err
+ return nil, err
}
projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)
if err != nil {
- return err
+ return nil, err
}
- followProjectSymlinks := settings.FollowProjectSymlinks.IsTrue()
j.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)
- // Create a new watcher with the updated path
- sw, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))
+ watcher, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(settings.FollowProjectSymlinks.IsTrue()))
if err != nil {
- return err
+ return nil, err
}
-
- // Start the new watcher
- if err := sw.Start(ctx); err != nil {
- if stopErr := sw.Stop(); stopErr != nil {
- slog.ErrorContext(ctx, "Failed to stop projects watcher after restart error", "error", stopErr)
- }
- return err
+ if err := watcher.Start(j.lifecycleCtx); err != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.
+ return watcher, err
}
- j.mu.Lock()
- j.projectsWatcher = sw
- j.mu.Unlock()
-
- slog.InfoContext(ctx, "Projects filesystem watcher restarted", "path", projectsDirectory)
-
- // Perform a sync to ensure we have the latest state from the new directory
- if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {
- slog.ErrorContext(ctx, "Initial project sync after watcher restart failed", "error", err)
+ slog.InfoContext(ctx, "Projects filesystem watcher started", "path", projectsDirectory)
+ if j.projectService != nil {
+ if err := j.projectService.SyncProjectsFromFileSystem(ctx); err != nil {
+ slog.ErrorContext(ctx, "Initial project sync after watcher start failed", "error", err)
+ }
}
-
- return nil
+ return watcher, nil
}
func (j *FilesystemWatcherJob) logRecursiveProjectsWatchLimitWarningInternal(ctx context.Context, projectsDirectory string) {
@@ -276,66 +161,55 @@ func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlin
}
func (j *FilesystemWatcherJob) RestartTemplatesWatcher(ctx context.Context) error {
- if j.templateService == nil {
- return nil
- }
slog.InfoContext(ctx, "Restarting templates filesystem watcher")
+ return j.templatesWatcher.Restart(ctx, "restart templates filesystem watcher", j.startTemplatesWatcherInternal)
+}
- j.mu.Lock()
- oldTemplatesWatcher := j.templatesWatcher
- j.templatesWatcher = nil
- j.mu.Unlock()
-
- if oldTemplatesWatcher != nil {
- if err := oldTemplatesWatcher.Stop(); err != nil {
- slog.WarnContext(ctx, "Failed to stop templates watcher during restart", "error", err)
- }
+func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {
+ if j.templateService == nil {
+ return nil, nil
}
settings, err := j.settingsService.GetSettings(ctx)
if err != nil {
- return err
+ return nil, err
}
projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)
if err != nil {
- return err
+ return nil, err
}
templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)
if err != nil {
- return err
+ return nil, err
+ }
+
+ if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
+ slog.ErrorContext(ctx, "Initial template sync failed", "error", err)
}
if directoriesOverlapInternal(projectsDirectory, templatesDir) {
slog.ErrorContext(ctx,
- "Templates and projects directories overlap; templates watcher not restarted",
+ "Templates and projects directories overlap; templates watcher disabled",
"projectsDirectory", projectsDirectory,
"templatesDirectory", templatesDir)
- return nil
+ return nil, nil
}
- tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{
+ watcher, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{
Debounce: 3 * time.Second,
- OnChange: j.handleTemplatesChange,
+ OnChange: j.handleTemplatesChangeInternal,
MaxDepth: 1,
})
if err != nil {
- return err
+ return nil, err
}
- if err := tw.Start(ctx); err != nil {
- return err
+ if err := watcher.Start(j.lifecycleCtx); err != nil { //nolint:contextcheck // watcher lifetime belongs to the application, not this replacement task.
+ return watcher, err
}
- j.mu.Lock()
- j.templatesWatcher = tw
- j.mu.Unlock()
-
- slog.InfoContext(ctx, "Templates filesystem watcher restarted", "path", templatesDir)
-
- if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
- slog.ErrorContext(ctx, "Initial template sync after watcher restart failed", "error", err)
- }
+ slog.InfoContext(ctx, "Templates filesystem watcher started", "path", templatesDir)
- return nil
+ return watcher, nil
}
// directoriesOverlapInternal returns true when a or b is the same as or contained in the
diff --git a/backend/pkg/scheduler/filesystem_watcher_job_test.go b/backend/pkg/scheduler/filesystem_watcher_job_test.go
index e6a8a37cc..408c63d89 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job_test.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job_test.go
@@ -1,9 +1,16 @@
package scheduler
import (
+ "context"
+ "sync"
"testing"
+ "time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/fswatch"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/fx/fxtest"
)
func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *testing.T) {
@@ -16,3 +23,48 @@ func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *te
assert.Equal(t, 1, opts.MaxDepth)
assert.True(t, opts.FollowSymlinkDirs)
}
+
+func TestFilesystemWatcherJob_ConcurrentProjectRestartsAreSerializedInternal(t *testing.T) {
+ _, settingsService, _ := setupAnalyticsStateServicesInternal(t)
+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "projectsDirectory", t.TempDir()))
+ require.NoError(t, settingsService.SetStringSetting(t.Context(), "templatesDirectory", t.TempDir()))
+
+ lifecycle := fxtest.NewLifecycle(t)
+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ job, err := NewFilesystemWatcherJob(t.Context(), actorRuntime, nil, nil, settingsService, 2)
+ require.NoError(t, err)
+
+ restartErrors := make(chan error, 2)
+ var callers sync.WaitGroup
+ callers.Add(2)
+ for range 2 {
+ go func() {
+ defer callers.Done()
+ restartErrors <- job.RestartProjectsWatcher(t.Context())
+ }()
+ }
+ callers.Wait()
+ close(restartErrors)
+ for restartErr := range restartErrors {
+ require.NoError(t, restartErr)
+ }
+ var earlyWatcher *fswatch.Watcher
+ require.NoError(t, job.projectsWatcher.Do(t.Context(), "capture early watcher", func(_ context.Context, watcher *fswatch.Watcher) error {
+ earlyWatcher = watcher
+ return nil
+ }))
+ require.NotNil(t, earlyWatcher)
+ require.NoError(t, job.Start(t.Context()))
+ var startedWatcher *fswatch.Watcher
+ require.NoError(t, job.projectsWatcher.Do(t.Context(), "capture started watcher", func(_ context.Context, watcher *fswatch.Watcher) error {
+ startedWatcher = watcher
+ return nil
+ }))
+ require.NotSame(t, earlyWatcher, startedWatcher)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, job.Stop(stopCtx))
+ require.NoError(t, lifecycle.Stop(stopCtx))
+}
diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.go
index d677af00b..fdd564ade 100644
--- a/backend/pkg/scheduler/image_update_watcher.go
+++ b/backend/pkg/scheduler/image_update_watcher.go
@@ -9,6 +9,7 @@ import (
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/services"
@@ -22,6 +23,8 @@ const (
imageUpdateWatcherDebounce = 2 * time.Second
imageUpdateWatcherBackfillRetry = 5 * time.Second
imageUpdateWatcherDefaultSchedule = "0 0 * * * *"
+ imageUpdateWatcherStopTimeout = 30 * time.Second
+ imageUpdateWatcherActorID = "image-update-watcher"
)
type imageUpdateScannerInternal interface {
@@ -45,8 +48,23 @@ type projectImageRefsBackfillerInternal interface {
BackfillProjectImageRefs(ctx context.Context) (int, error)
}
-type imageUpdateScanRunInternal struct {
- done chan struct{}
+type imageUpdateMessageKindInternal uint8
+
+const (
+ imageUpdateTriggerMessageInternal imageUpdateMessageKindInternal = iota
+ imageUpdateScheduleRefreshMessageInternal
+ imageUpdateBackfillCompletedMessageInternal
+ imageUpdateDebounceElapsedMessageInternal
+ imageUpdateScheduledPollMessageInternal
+ imageUpdateScanCompletedMessageInternal
+ imageUpdateManualScanRequestMessageInternal
+ imageUpdateManualScanAdmissionMessageInternal
+)
+
+type imageUpdateScanCompletionInternal struct {
+ automatic bool
+ triggerGeneration uint64
+ done *actors.Promise[error]
}
// ImageUpdateWatcher continuously reconciles image update state after Docker image changes.
@@ -56,18 +74,27 @@ type ImageUpdateWatcher struct {
environmentService registryCredentialLoaderInternal
dockerService dockerEventBusProviderInternal
projectService projectImageRefsBackfillerInternal
- triggerCh chan struct{}
- scheduleRefreshCh chan struct{}
+ actorRuntime *actors.Runtime
+ actorProcess atomic.Pointer[actors.Actor]
+ stopping atomic.Bool
+ triggerIngress *actors.Ingress[imageUpdateMessageKindInternal, time.Time]
+ scheduleIngress *actors.Ingress[imageUpdateMessageKindInternal, actors.NoPayload]
location *time.Location
debounce time.Duration
backfillRetry time.Duration
metadataReady chan struct{}
metadataReadyOnce sync.Once
- activeRun atomic.Pointer[imageUpdateScanRunInternal]
+ started chan struct{}
+ startedOnce sync.Once
+ stopped chan struct{}
+ stoppedOnce sync.Once
}
// NewImageUpdateWatcher constructs the image update watcher from the existing services.
-func NewImageUpdateWatcher(cfg *config.Config, imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService, projectService *services.ProjectService) *ImageUpdateWatcher {
+func NewImageUpdateWatcher(runtime *actors.Runtime, cfg *config.Config, imageUpdateService *services.ImageUpdateService, settingsService *services.SettingsService, environmentService *services.EnvironmentService, dockerService *services.DockerClientService, projectService *services.ProjectService) (*ImageUpdateWatcher, error) {
+ if runtime == nil || imageUpdateService == nil || settingsService == nil || environmentService == nil || dockerService == nil || projectService == nil {
+ return nil, errors.New("image update watcher dependencies unavailable")
+ }
location := time.UTC
if cfg != nil {
location = cfg.GetLocation()
@@ -78,13 +105,16 @@ func NewImageUpdateWatcher(cfg *config.Config, imageUpdateService *services.Imag
environmentService: environmentService,
dockerService: dockerService,
projectService: projectService,
- triggerCh: make(chan struct{}, 1),
- scheduleRefreshCh: make(chan struct{}, 1),
+ actorRuntime: runtime,
+ triggerIngress: actors.NewIngress[imageUpdateMessageKindInternal, time.Time](imageUpdateTriggerMessageInternal),
+ scheduleIngress: actors.NewIngress[imageUpdateMessageKindInternal, actors.NoPayload](imageUpdateScheduleRefreshMessageInternal),
location: location,
debounce: imageUpdateWatcherDebounce,
backfillRetry: imageUpdateWatcherBackfillRetry,
metadataReady: make(chan struct{}),
- }
+ started: make(chan struct{}),
+ stopped: make(chan struct{}),
+ }, nil
}
// Name identifies the watcher in scheduler lifecycle logs.
@@ -92,109 +122,153 @@ func (w *ImageUpdateWatcher) Name() string {
return "image-polling"
}
-// Start subscribes to Docker image events, prepares project metadata, and runs scans until ctx is canceled.
+// Start subscribes to Docker image events and owns the watcher actor until ctx is canceled.
func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
- if w == nil || w.dockerService == nil || w.dockerService.EventBus() == nil {
- return errors.New("docker event bus unavailable")
+ if w.stopping.Load() {
+ return errors.New("image update watcher cannot be restarted after stop")
+ }
+ if w.actorProcess.Load() != nil {
+ return errors.New("image update watcher already started")
}
- if w.projectService == nil || w.metadataReady == nil {
- return errors.New("project image metadata backfiller unavailable")
+ if w.debounce <= 0 {
+ w.debounce = imageUpdateWatcherDebounce
+ }
+ if w.backfillRetry <= 0 {
+ w.backfillRetry = imageUpdateWatcherBackfillRetry
}
- eventCh, unsubscribe := w.dockerService.EventBus().Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
+ eventBus := w.dockerService.EventBus()
+ if eventBus == nil {
+ return errors.New("docker event bus unavailable")
+ }
+ eventCh, unsubscribe := eventBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
defer unsubscribe()
- var listener sync.WaitGroup
- listenerStarted := make(chan struct{})
- listener.Go(func() {
- close(listenerStarted)
- for {
- select {
- case <-ctx.Done():
- return
- case _, ok := <-eventCh:
- if !ok {
- return
- }
- w.Trigger()
+ process, err := actors.NewActor(
+ ctx,
+ w.actorRuntime,
+ "scheduler",
+ imageUpdateWatcherActorID,
+ 3,
+ func() actors.Behavior {
+ state := &imageUpdateWatcherActorInternal{watcher: w}
+ return actors.Behavior{
+ Initialize: state.initializeInternal,
+ Handle: state.receiveInternal,
+ Cleanup: state.cleanupInternal,
}
- }
- })
- listener.Go(func() {
- w.runScheduledPollsInternal(ctx)
- })
-
- select {
- case <-ctx.Done():
- listener.Wait()
- return nil
- case <-listenerStarted:
+ },
+ w.triggerIngress,
+ w.scheduleIngress,
+ )
+ if err != nil {
+ return errors.WrapIf(err, "start image update watcher actor")
}
+ w.actorProcess.Store(process)
+ w.startedOnce.Do(func() { close(w.started) })
slog.InfoContext(ctx, "image update watcher started")
defer func() {
- listener.Wait()
- slog.InfoContext(ctx, "image update watcher stopped")
+ w.actorProcess.CompareAndSwap(process, nil)
+ stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), imageUpdateWatcherStopTimeout)
+ defer cancel()
+ if err := process.Stop(stopCtx); err != nil {
+ slog.ErrorContext(context.WithoutCancel(ctx), "failed to stop image update watcher actor", "error", err)
+ }
+ slog.InfoContext(context.WithoutCancel(ctx), "image update watcher stopped")
}()
- if !w.backfillProjectImageRefsInternal(ctx) {
- return nil
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-process.Done():
+ if ctx.Err() != nil || w.stopping.Load() {
+ return nil
+ }
+ return errors.New("image update watcher actor stopped unexpectedly")
+ case _, ok := <-eventCh:
+ if !ok {
+ slog.WarnContext(ctx, "docker image event subscription closed; scheduled image polling remains active")
+ eventCh = nil
+ continue
+ }
+ w.Trigger()
+ }
}
-
- w.Trigger()
- w.runTriggeredScansInternal(ctx)
- return nil
}
-// Trigger queues an image scan without blocking the Docker event publisher.
-func (w *ImageUpdateWatcher) Trigger() {
- if w == nil {
- return
+// Stop terminates the watcher actor within the caller's lifecycle deadline.
+func (w *ImageUpdateWatcher) Stop(ctx context.Context) error {
+ if ctx == nil {
+ return errors.New("image update watcher stop context unavailable")
}
- select {
- case w.triggerCh <- struct{}{}:
- default:
+ w.stopping.Store(true)
+ w.stoppedOnce.Do(func() { close(w.stopped) })
+ process := w.actorProcess.Load()
+ if process == nil {
+ return nil
}
+ return process.Stop(ctx)
}
-// RefreshSchedule wakes the scheduled-poll loop so it re-reads pollingInterval.
+// Trigger records a trailing-edge image scan without blocking the event publisher.
+func (w *ImageUpdateWatcher) Trigger() {
+ w.triggerIngress.Send(time.Now())
+}
+
+// RefreshSchedule wakes the actor so it re-reads pollingInterval.
func (w *ImageUpdateWatcher) RefreshSchedule() {
- if w == nil || w.scheduleRefreshCh == nil {
- return
- }
- select {
- case w.scheduleRefreshCh <- struct{}{}:
- default:
- }
+ w.scheduleIngress.Send(actors.NoPayload{})
}
-// RunNow performs the same full-host image scan used by automatic watcher
-// triggers. It does not wait behind an active scan: a request during one
-// returns ImageScanInProgressError immediately instead of parking
-// (potentially forever, given the detached run-now context) on the
-// single-flight gate. The triggered-scan loop waits and retries itself.
+// RunNow requests immediate admission from the watcher actor and waits for that scan.
+// An active scan is refused immediately instead of being queued.
func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
- if w == nil || w.settingsService == nil || w.imageUpdateService == nil || w.environmentService == nil || w.metadataReady == nil {
- return errors.New("image update watcher is not initialized")
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-w.stopped:
+ return errors.New("image update watcher is not running")
+ case <-w.started:
}
select {
case <-ctx.Done():
return ctx.Err()
+ case <-w.stopped:
+ return errors.New("image update watcher is not running")
case <-w.metadataReady:
}
- run := &imageUpdateScanRunInternal{done: make(chan struct{})}
- for !w.activeRun.CompareAndSwap(nil, run) {
- if w.activeRun.Load() != nil {
- return common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress"))
- }
- // Lost the CAS to a run that has since been released; retry.
+ process := w.actorProcess.Load()
+ if process == nil {
+ return errors.New("image update watcher is not running")
}
- defer func() {
- w.activeRun.Store(nil)
- close(run.done)
- }()
+ admission, err := actors.Request[imageUpdateMessageKindInternal, context.Context, *actors.Promise[error]](
+ ctx,
+ process,
+ actors.Message[imageUpdateMessageKindInternal, context.Context]{Kind: imageUpdateManualScanRequestMessageInternal, Value: ctx},
+ )
+ if err != nil {
+ return errors.WrapIf(err, "request image scan admission")
+ }
+ if admission.Kind != imageUpdateManualScanAdmissionMessageInternal {
+ return errors.New("invalid image scan admission response")
+ }
+ if admission.Err != nil {
+ return admission.Err
+ }
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case err := <-admission.Value.Done():
+ return err
+ }
+}
+
+func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
@@ -204,7 +278,6 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
}
slog.InfoContext(ctx, "image scan run started")
-
creds, err := w.environmentService.GetEnabledRegistryCredentials(ctx)
if err != nil {
slog.WarnContext(ctx, "failed to load registry credentials for image scan", "error", err.Error())
@@ -216,7 +289,6 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
return errors.WrapIf(err, "image scan failed")
}
- total := len(results)
updates := 0
scanErrors := 0
for _, result := range results {
@@ -231,146 +303,185 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
updates++
}
}
-
- slog.InfoContext(ctx, "image scan run completed", "checked", total, "updates", updates, "errors", scanErrors)
+ slog.InfoContext(ctx, "image scan run completed", "checked", len(results), "updates", updates, "errors", scanErrors)
return nil
}
-func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context) bool {
- retryDelay := w.backfillRetry
- if retryDelay <= 0 {
- retryDelay = imageUpdateWatcherBackfillRetry
+func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context, attempt int) (actors.NoPayload, error) {
+ startedAt := time.Now()
+ count, err := w.projectService.BackfillProjectImageRefs(ctx)
+ duration := time.Since(startedAt)
+ if err == nil {
+ slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", duration, "attempt", attempt)
+ return actors.NoPayload{}, nil
}
+ if ctx.Err() == nil {
+ slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", duration, "attempt", attempt, "retryIn", w.backfillRetry, "error", err)
+ }
+ return actors.NoPayload{}, err
+}
- for attempt := 1; ; attempt++ {
- if ctx.Err() != nil {
- return false
- }
-
- startedAt := time.Now()
- count, err := w.projectService.BackfillProjectImageRefs(ctx)
- duration := time.Since(startedAt)
- if err == nil {
- slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", duration, "attempt", attempt)
- w.metadataReadyOnce.Do(func() { close(w.metadataReady) })
- return true
- }
- if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {
- return false
- }
+type imageUpdateWatcherActorInternal struct {
+ watcher *ImageUpdateWatcher
+ scanRunning bool
+ automaticPending bool
+ triggerGeneration uint64
+ metadataReady bool
+ debounceTimer actors.Timer[imageUpdateMessageKindInternal]
+ scheduleTimer actors.Timer[imageUpdateMessageKindInternal]
+}
- slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", duration, "attempt", attempt, "retryIn", retryDelay, "error", err)
- timer := time.NewTimer(retryDelay)
- select {
- case <-ctx.Done():
- timer.Stop()
- return false
- case <-timer.C:
- }
+func (a *imageUpdateWatcherActorInternal) initializeInternal(ctx *actors.Context) {
+ select {
+ case <-a.watcher.metadataReady:
+ a.metadataReady = true
+ default:
+ a.startBackfillInternal(ctx)
}
+ a.resetScheduleInternal(ctx)
}
-func (w *ImageUpdateWatcher) runTriggeredScansInternal(ctx context.Context) {
- for {
- select {
- case <-ctx.Done():
+func (a *imageUpdateWatcherActorInternal) cleanupInternal(*actors.Context) {
+ a.debounceTimer.Stop()
+ a.scheduleTimer.Stop()
+}
+
+func (a *imageUpdateWatcherActorInternal) receiveInternal(ctx *actors.Context, rawMessage any) {
+ switch message := rawMessage.(type) {
+ case actors.Message[imageUpdateMessageKindInternal, actors.NoPayload]:
+ a.receiveSignalInternal(ctx, message)
+ case actors.Message[imageUpdateMessageKindInternal, uint64]:
+ a.receiveTimerInternal(ctx, message)
+ case actors.Message[imageUpdateMessageKindInternal, context.Context]:
+ if message.Kind != imageUpdateManualScanRequestMessageInternal {
return
- case <-w.triggerCh:
}
-
- if !w.waitForDebounceInternal(ctx) {
+ if a.scanRunning {
+ ctx.Respond(actors.Message[imageUpdateMessageKindInternal, *actors.Promise[error]]{
+ Kind: imageUpdateManualScanAdmissionMessageInternal,
+ Err: common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress")),
+ })
return
}
- err := w.RunNow(ctx)
- if errors.Is(err, common.ErrImageScanInProgress) {
- // A manual scan holds the gate; wait it out and re-queue the
- // trigger so this scan attempt is not lost.
- if !w.waitForActiveScanInternal(ctx) {
- return
- }
- w.Trigger()
- continue
+ done := actors.NewPromise[error]()
+ a.startScanInternal(message.Value, ctx, false, 0, done)
+ ctx.Respond(actors.Message[imageUpdateMessageKindInternal, *actors.Promise[error]]{Kind: imageUpdateManualScanAdmissionMessageInternal, Value: done})
+ case actors.Message[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]:
+ defer message.Acknowledge()
+ if message.Kind != imageUpdateScanCompletedMessageInternal {
+ return
+ }
+ a.scanRunning = false
+ if message.Value.automatic {
+ a.watcher.triggerIngress.Acknowledge(message.Value.triggerGeneration)
+ a.automaticPending = a.watcher.triggerIngress.Pending()
+ }
+ if message.Value.automatic && message.Err != nil && !errors.Is(message.Err, context.Canceled) {
+ slog.ErrorContext(ctx.Context(), "image update watcher scan failed", "error", message.Err)
+ }
+ if a.automaticPending {
+ a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, a.watcher.debounce)
}
- if err != nil && !errors.Is(err, context.Canceled) {
- slog.ErrorContext(ctx, "image update watcher scan failed", "error", err)
+ if message.Value.done != nil {
+ message.Value.done.Resolve(message.Err)
}
}
}
-// runScheduledPollsInternal periodically queues scans based on the
-// pollingInterval cron setting so registry-side updates are detected even when
-// no local Docker image events occur. Fires funnel through Trigger, sharing
-// the same debounce and single-flight path as event-driven scans.
-func (w *ImageUpdateWatcher) runScheduledPollsInternal(ctx context.Context) {
- if w.scheduleRefreshCh == nil {
+func (a *imageUpdateWatcherActorInternal) receiveSignalInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, actors.NoPayload]) {
+ if message.Kind == imageUpdateTriggerMessageInternal {
+ triggeredAt, generation := a.watcher.triggerIngress.Begin()
+ a.automaticPending = true
+ a.triggerGeneration = generation
+ if a.metadataReady && !a.scanRunning {
+ a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, max(time.Until(triggeredAt.Add(a.watcher.debounce)), 0))
+ }
+ return
+ }
+ if message.Kind == imageUpdateScheduleRefreshMessageInternal {
+ a.watcher.scheduleIngress.Take()
+ a.resetScheduleInternal(ctx)
return
}
- location := w.location
- if location == nil {
- location = time.UTC
+ if message.Kind != imageUpdateBackfillCompletedMessageInternal {
+ return
+ }
+ defer message.Acknowledge()
+ if message.Err != nil {
+ slog.ErrorContext(ctx.Context(), "image update metadata backfill worker failed", "error", message.Err)
+ return
}
+ a.metadataReady = true
+ a.watcher.metadataReadyOnce.Do(func() { close(a.watcher.metadataReady) })
+ a.watcher.Trigger()
+}
- for {
- spec := imageUpdateWatcherDefaultSchedule
- if w.settingsService != nil {
- spec = w.settingsService.GetStringSetting(ctx, "pollingInterval", imageUpdateWatcherDefaultSchedule)
- }
- schedule, err := cronScheduleParser.Parse(spec)
- if err != nil {
- slog.WarnContext(ctx, "invalid pollingInterval cron expression; using default schedule", "pollingInterval", spec, "default", imageUpdateWatcherDefaultSchedule, "error", err)
- schedule, err = cronScheduleParser.Parse(imageUpdateWatcherDefaultSchedule)
- if err != nil {
- slog.ErrorContext(ctx, "failed to parse default image polling schedule", "error", err)
- return
- }
+func (a *imageUpdateWatcherActorInternal) receiveTimerInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, uint64]) {
+ if message.Kind == imageUpdateDebounceElapsedMessageInternal {
+ if !a.debounceTimer.Current(message.Value) || !a.automaticPending || a.scanRunning {
+ return
}
-
- timer := time.NewTimer(time.Until(schedule.Next(time.Now().In(location))))
- select {
- case <-ctx.Done():
- timer.Stop()
+ if remaining := time.Until(a.watcher.triggerIngress.Latest().Add(a.watcher.debounce)); remaining > 0 {
+ a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, remaining)
return
- case <-w.scheduleRefreshCh:
- timer.Stop()
- case <-timer.C:
- w.Trigger()
}
+ a.startScanInternal(ctx.Context(), ctx, true, a.triggerGeneration, nil)
+ return
+ }
+ if message.Kind != imageUpdateScheduledPollMessageInternal || !a.scheduleTimer.Current(message.Value) {
+ return
}
+ a.watcher.Trigger()
+ a.resetScheduleInternal(ctx)
}
-// waitForActiveScanInternal blocks until any in-flight scan releases the
-// single-flight gate. It returns false when ctx is cancelled first.
-func (w *ImageUpdateWatcher) waitForActiveScanInternal(ctx context.Context) bool {
- activeRun := w.activeRun.Load()
- if activeRun == nil {
- return true
+func (a *imageUpdateWatcherActorInternal) startBackfillInternal(ctx *actors.Context) {
+ if a.watcher.backfillRetry <= 0 {
+ a.watcher.backfillRetry = imageUpdateWatcherBackfillRetry
}
- select {
- case <-ctx.Done():
- return false
- case <-activeRun.done:
- return true
+ attempt := 0
+ worker := actors.Worker[imageUpdateMessageKindInternal, actors.NoPayload]{
+ Actor: ctx,
+ WorkContext: ctx.Context(),
+ Label: "image update metadata backfill worker",
+ CompletionKind: imageUpdateBackfillCompletedMessageInternal,
+ RetryDelay: a.watcher.backfillRetry,
}
+ worker.Start(func(workerCtx context.Context) (actors.NoPayload, error) {
+ attempt++
+ return a.watcher.backfillProjectImageRefsInternal(workerCtx, attempt)
+ })
}
-func (w *ImageUpdateWatcher) waitForDebounceInternal(ctx context.Context) bool {
- timer := time.NewTimer(w.debounce)
- defer timer.Stop()
+func (a *imageUpdateWatcherActorInternal) startScanInternal(scanCtx context.Context, ctx *actors.Context, automatic bool, triggerGeneration uint64, done *actors.Promise[error]) {
+ a.scanRunning = true
+ worker := actors.Worker[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]{
+ Actor: ctx,
+ WorkContext: scanCtx,
+ Label: "image update scan worker",
+ CompletionKind: imageUpdateScanCompletedMessageInternal,
+ PanicValue: imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done},
+ ActorStopped: func(_ imageUpdateScanCompletionInternal, err error) {
+ done.Resolve(err)
+ },
+ }
+ worker.Start(func(workerCtx context.Context) (imageUpdateScanCompletionInternal, error) {
+ return imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done}, a.watcher.executeScanInternal(workerCtx)
+ })
+}
- for {
- select {
- case <-ctx.Done():
- return false
- case <-w.triggerCh:
- if !timer.Stop() {
- select {
- case <-timer.C:
- default:
- }
- }
- timer.Reset(w.debounce)
- case <-timer.C:
- return true
+func (a *imageUpdateWatcherActorInternal) resetScheduleInternal(ctx *actors.Context) {
+ spec := a.watcher.settingsService.GetStringSetting(ctx.Context(), "pollingInterval", imageUpdateWatcherDefaultSchedule)
+ schedule, err := cronScheduleParser.Parse(spec)
+ if err != nil {
+ slog.WarnContext(ctx.Context(), "invalid pollingInterval cron expression; using default schedule", "pollingInterval", spec, "default", imageUpdateWatcherDefaultSchedule, "error", err)
+ schedule, err = cronScheduleParser.Parse(imageUpdateWatcherDefaultSchedule)
+ if err != nil {
+ slog.ErrorContext(ctx.Context(), "failed to parse default image polling schedule", "error", err)
+ return
}
}
+
+ delay := time.Until(schedule.Next(time.Now().In(a.watcher.location)))
+ a.scheduleTimer.Reset(ctx, imageUpdateScheduledPollMessageInternal, delay)
}
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index b6cba9172..368639271 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -3,19 +3,22 @@ package scheduler
import (
"bytes"
"context"
- "errors"
+ "fmt"
"log/slog"
"strings"
"sync"
"testing"
"time"
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
"github.com/moby/moby/api/types/events"
"github.com/stretchr/testify/require"
"go.getarcane.app/streams/bus"
+ "go.uber.org/fx/fxtest"
)
type imageUpdateScannerFakeInternal struct {
@@ -24,6 +27,7 @@ type imageUpdateScannerFakeInternal struct {
active int
maxActive int
errors []error
+ panics []bool
startedCh chan int
releaseCh <-chan struct{}
}
@@ -38,9 +42,15 @@ func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ i
if call <= len(s.errors) {
err = s.errors[call-1]
}
+ shouldPanic := call <= len(s.panics) && s.panics[call-1]
startedCh := s.startedCh
releaseCh := s.releaseCh
s.mu.Unlock()
+ defer func() {
+ s.mu.Lock()
+ s.active--
+ s.mu.Unlock()
+ }()
if startedCh != nil {
select {
@@ -55,10 +65,9 @@ func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ i
case <-releaseCh:
}
}
-
- s.mu.Lock()
- s.active--
- s.mu.Unlock()
+ if shouldPanic {
+ panic("deliberate image scan panic")
+ }
return map[string]*imageupdate.Response{}, err
}
@@ -166,29 +175,37 @@ func (b *lockedBufferInternal) stringInternal() string {
return b.String()
}
-func newImageUpdateWatcherForTestInternal(scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {
+func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScannerInternal, settings pollingSettingReaderInternal, eventBus *bus.DockerEventBus, backfiller projectImageRefsBackfillerInternal) *ImageUpdateWatcher {
+ t.Helper()
if backfiller == nil {
backfiller = &projectImageRefsBackfillerFakeInternal{}
}
+ lifecycle := fxtest.NewLifecycle(t)
+ actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, lifecycle.Stop(stopCtx))
+ })
return &ImageUpdateWatcher{
imageUpdateService: scanner,
settingsService: settings,
environmentService: registryCredentialLoaderFakeInternal{},
dockerService: dockerEventBusProviderFakeInternal{eventBus: eventBus},
projectService: backfiller,
- triggerCh: make(chan struct{}, 1),
- scheduleRefreshCh: make(chan struct{}, 1),
+ actorRuntime: actorRuntime,
+ triggerIngress: actors.NewIngress[imageUpdateMessageKindInternal, time.Time](imageUpdateTriggerMessageInternal),
+ scheduleIngress: actors.NewIngress[imageUpdateMessageKindInternal, actors.NoPayload](imageUpdateScheduleRefreshMessageInternal),
location: time.UTC,
debounce: 10 * time.Millisecond,
backfillRetry: 10 * time.Millisecond,
metadataReady: make(chan struct{}),
+ started: make(chan struct{}),
+ stopped: make(chan struct{}),
}
}
-func markImageUpdateWatcherMetadataReadyForTestInternal(watcher *ImageUpdateWatcher) {
- watcher.metadataReadyOnce.Do(func() { close(watcher.metadataReady) })
-}
-
func startImageUpdateWatcherForTestInternal(t *testing.T, watcher *ImageUpdateWatcher) (context.CancelFunc, <-chan error) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
@@ -207,7 +224,7 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tes
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -235,6 +252,22 @@ func TestImageUpdateWatcher_StartScansAtStartupAndCoalescesAllImageEvents(t *tes
require.Equal(t, 2, scanner.countInternal())
}
+func TestImageUpdateWatcher_TrailingEdgeDebounceExtendsWithNewTriggers(t *testing.T) {
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher.debounce = 50 * time.Millisecond
+ startImageUpdateWatcherForTestInternal(t, watcher)
+
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)
+ watcher.Trigger()
+ time.Sleep(30 * time.Millisecond)
+ watcher.Trigger()
+ time.Sleep(30 * time.Millisecond)
+ require.Equal(t, 1, scanner.countInternal())
+ require.Eventually(t, func() bool { return scanner.countInternal() == 2 }, time.Second, time.Millisecond)
+}
+
func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testing.T) {
releaseCh := make(chan struct{})
scanner := &imageUpdateScannerFakeInternal{
@@ -243,7 +276,7 @@ func TestImageUpdateWatcher_EventDuringScanQueuesOneSerializedFollowUp(t *testin
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Equal(t, 1, <-scanner.startedCh)
@@ -262,7 +295,7 @@ func TestImageUpdateWatcher_DisabledTriggersAreSkippedUntilEnabled(t *testing.T)
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: false}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
time.Sleep(30 * time.Millisecond)
@@ -281,7 +314,7 @@ func TestImageUpdateWatcher_ScanErrorDoesNotStopFutureEvents(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{errors: []error{errors.New("registry unavailable")}}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
@@ -296,22 +329,46 @@ func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *test
releaseCh: releaseCh,
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
- markImageUpdateWatcherMetadataReadyForTestInternal(watcher)
-
- firstErrCh := make(chan error, 1)
- go func() { firstErrCh <- watcher.RunNow(context.Background()) }()
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ startImageUpdateWatcherForTestInternal(t, watcher)
require.Equal(t, 1, <-scanner.startedCh)
require.ErrorIs(t, watcher.RunNow(context.Background()), common.ErrImageScanInProgress)
require.Equal(t, 1, scanner.countInternal())
close(releaseCh)
- require.NoError(t, <-firstErrCh)
+ require.Eventually(t, func() bool {
+ return watcher.RunNow(context.Background()) == nil
+ }, time.Second, time.Millisecond)
+ require.Equal(t, 2, scanner.countInternal())
require.Equal(t, 1, scanner.maxActiveInternal())
+}
+
+func TestImageUpdateWatcher_RunNowConsumesHollywoodResponseInternal(t *testing.T) {
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ startImageUpdateWatcherForTestInternal(t, watcher)
+
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)
+ require.Eventually(t, func() bool {
+ return watcher.RunNow(context.Background()) == nil
+ }, time.Second, time.Millisecond)
+ for range 10 {
+ require.NoError(t, watcher.RunNow(context.Background()))
+ }
+}
+
+func TestImageUpdateWatcher_RunNowReturnsContainedScanPanicInternal(t *testing.T) {
+ scanner := &imageUpdateScannerFakeInternal{panics: []bool{false, true}}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ startImageUpdateWatcherForTestInternal(t, watcher)
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)
- // The gate is free again after the first scan finishes.
- require.NoError(t, watcher.RunNow(context.Background()))
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.ErrorContains(t, watcher.RunNow(ctx), "image update scan worker panicked")
require.Equal(t, 2, scanner.countInternal())
}
@@ -323,18 +380,26 @@ func TestImageUpdateWatcher_TriggeredScanRetriesAfterManualScanFinishes(t *testi
}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, nil)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
startImageUpdateWatcherForTestInternal(t, watcher)
// Let the startup scan finish so the gate is free.
require.Equal(t, 1, <-scanner.startedCh)
releaseCh <- struct{}{}
- require.Eventually(t, func() bool { return watcher.activeRun.Load() == nil }, time.Second, time.Millisecond)
-
// A manual scan takes the gate, then a Docker event fires: the triggered
// loop must wait out the manual scan and still run its own scan after.
manualErrCh := make(chan error, 1)
- go func() { manualErrCh <- watcher.RunNow(context.Background()) }()
+ go func() {
+ for {
+ err := watcher.RunNow(context.Background())
+ if errors.Is(err, common.ErrImageScanInProgress) {
+ time.Sleep(time.Millisecond)
+ continue
+ }
+ manualErrCh <- err
+ return
+ }
+ }()
require.Equal(t, 2, <-scanner.startedCh)
eventBus.Publish(events.Message{Type: events.ImageEventType, Action: events.ActionPull})
@@ -375,7 +440,7 @@ func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *test
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, eventBus, backfiller)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, backfiller)
startImageUpdateWatcherForTestInternal(t, watcher)
<-backfillStarted
@@ -397,13 +462,31 @@ func TestImageUpdateWatcher_BackfillGatesFirstScanAndCoalescesEventBurst(t *test
t.Logf("coalesced %d image events into one scan after backfilling %d projects in %s", eventCount, projectCount, time.Since(burstStartedAt))
}
+func TestImageUpdateWatcher_BackfillRetriesContainedPanicInternal(t *testing.T) {
+ backfiller := &projectImageRefsBackfillerFakeInternal{
+ run: func(_ context.Context, call int) (int, error) {
+ if call == 1 {
+ panic("deliberate backfill panic")
+ }
+ return 1, nil
+ },
+ }
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
+ startImageUpdateWatcherForTestInternal(t, watcher)
+
+ require.Eventually(t, func() bool { return backfiller.countInternal() == 2 }, time.Second, time.Millisecond)
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)
+}
+
func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {
secondAttemptStarted := make(chan struct{})
releaseSecondAttempt := make(chan struct{})
backfiller := &projectImageRefsBackfillerFakeInternal{
run: func(ctx context.Context, call int) (int, error) {
if call == 1 {
- return 0, errors.New("database unavailable")
+ return 0, fmt.Errorf("database statement timeout: %w", context.DeadlineExceeded)
}
close(secondAttemptStarted)
select {
@@ -416,7 +499,7 @@ func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {
}
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
startImageUpdateWatcherForTestInternal(t, watcher)
select {
@@ -442,7 +525,7 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.
}
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), backfiller)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
ctx, cancel := context.WithCancel(context.Background())
errCh := make(chan error, 1)
go func() { errCh <- watcher.Start(ctx) }()
@@ -461,17 +544,61 @@ func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.
func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true, schedule: "* * * * * *"}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
startImageUpdateWatcherForTestInternal(t, watcher)
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, 3*time.Second, 10*time.Millisecond)
}
+func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsActorRunning(t *testing.T) {
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ eventBus := bus.NewDockerEventBus()
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, eventBus, nil)
+ startImageUpdateWatcherForTestInternal(t, watcher)
+
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
+ eventBus.Close()
+ watcher.Trigger()
+
+ require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, time.Second, 5*time.Millisecond)
+ require.NotNil(t, watcher.actorProcess.Load())
+}
+
+func TestImageUpdateWatcher_RunnerRestartsAfterUnexpectedActorExit(t *testing.T) {
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
+ runner, err := actors.NewRunner(
+ t.Context(),
+ watcher.actorRuntime,
+ "image-update-watcher-test",
+ "supervision",
+ "supervised image update watcher",
+ 3,
+ watcher.Start,
+ )
+ require.NoError(t, err)
+
+ require.Eventually(t, func() bool { return watcher.actorProcess.Load() != nil }, time.Second, time.Millisecond)
+ firstProcess := watcher.actorProcess.Load()
+ require.NoError(t, firstProcess.Stop(t.Context()))
+ require.Eventually(t, func() bool {
+ current := watcher.actorProcess.Load()
+ return current != nil && current != firstProcess
+ }, time.Second, time.Millisecond)
+
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, runner.Stop(stopCtx))
+ require.NoError(t, watcher.Stop(stopCtx))
+}
+
func TestImageUpdateWatcher_RunNowWaitsForMetadataReadiness(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(scanner, settings, bus.NewDockerEventBus(), nil)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.go
index dcea6ab5b..b4ccd5d1e 100644
--- a/backend/pkg/scheduler/job_schedule_integration_test.go
+++ b/backend/pkg/scheduler/job_schedule_integration_test.go
@@ -23,13 +23,13 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
appConfig := &config.Config{Timezone: "UTC"}
lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())
- jobScheduler := NewJobScheduler(lifecycleCtx, appConfig.GetLocation())
+ jobScheduler := newJobSchedulerForTestInternal(t, lifecycleCtx, appConfig.GetLocation())
jobService := services.NewJobService(db, settingsService, appConfig)
jobService.SetScheduler(lifecycleCtx, jobScheduler)
- jobScheduler.StartScheduler()
+ require.NoError(t, jobScheduler.StartScheduler())
- _, ok := jobScheduler.GetJobRuntimeState("image-polling").Get()
+ _, ok := jobScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
require.Empty(t, jobScheduler.cron.Entries())
@@ -45,7 +45,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
require.Equal(t, requestedSchedule, persisted.Value)
require.Equal(t, requestedSchedule, settingsService.GetStringSetting(ctx, "pollingInterval", ""))
- _, ok = jobScheduler.GetJobRuntimeState("image-polling").Get()
+ _, ok = jobScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
require.Empty(t, jobScheduler.cron.Entries())
@@ -63,17 +63,17 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
restartDB, restartSettingsService := openJobScheduleTestDatabaseInternal(t, ctx, databasePath)
restartLifecycleCtx, cancelRestartLifecycle := context.WithCancel(context.Background())
- restartScheduler := NewJobScheduler(restartLifecycleCtx, appConfig.GetLocation())
+ restartScheduler := newJobSchedulerForTestInternal(t, restartLifecycleCtx, appConfig.GetLocation())
restartJobService := services.NewJobService(restartDB, restartSettingsService, appConfig)
restartJobService.SetScheduler(restartLifecycleCtx, restartScheduler)
- restartScheduler.StartScheduler()
+ require.NoError(t, restartScheduler.StartScheduler())
t.Cleanup(func() {
cancelRestartLifecycle()
waitForSchedulerStopInternal(restartScheduler)
closeJobScheduleTestDatabaseInternal(t, restartDB)
})
- _, ok = restartScheduler.GetJobRuntimeState("image-polling").Get()
+ _, ok = restartScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
require.Empty(t, restartScheduler.cron.Entries())
require.Equal(t, requestedSchedule, restartSettingsService.GetStringSetting(ctx, "pollingInterval", ""))
@@ -87,7 +87,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
_, err = restartJobService.UpdateJobSchedules(ctx, jobschedule.Update{PollingInterval: &disabledSchedule})
require.NoError(t, err)
- _, ok = restartScheduler.GetJobRuntimeState("image-polling").Get()
+ _, ok = restartScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
require.Empty(t, restartScheduler.cron.Entries())
@@ -106,7 +106,7 @@ func openJobScheduleTestDatabaseInternal(t *testing.T, ctx context.Context, data
require.NoError(t, gormDB.AutoMigrate(&models.SettingVariable{}))
db := &database.DB{DB: gormDB}
- settingsService, err := services.NewSettingsService(ctx, db)
+ settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
require.NoError(t, settingsService.EnsureDefaultSettings(ctx))
require.NoError(t, settingsService.LoadDatabaseSettings(ctx))
@@ -122,7 +122,7 @@ func closeJobScheduleTestDatabaseInternal(t *testing.T, db *database.DB) {
require.NoError(t, sqlDB.Close())
}
-func waitForSchedulerStopInternal(jobScheduler *JobScheduler) {
+func waitForSchedulerStopInternal(jobScheduler *jobSchedulerInternal) {
<-jobScheduler.cron.Stop().Done()
}
diff --git a/backend/pkg/scheduler/scheduled_prune_job_test.go b/backend/pkg/scheduler/scheduled_prune_job_test.go
index a703c82e1..a7cc2c63d 100644
--- a/backend/pkg/scheduler/scheduled_prune_job_test.go
+++ b/backend/pkg/scheduler/scheduled_prune_job_test.go
@@ -21,7 +21,7 @@ func setupScheduledPruneSettingsServiceInternal(t *testing.T) *services.Settings
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))
- svc, err := services.NewSettingsService(context.Background(), &database.DB{DB: db})
+ svc, err := newSettingsServiceForTestInternal(t, context.Background(), &database.DB{DB: db})
require.NoError(t, err)
return svc
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index 2c7ee3e64..647574960 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -3,14 +3,15 @@ package scheduler
import (
"context"
"log/slog"
- "sync"
+ "maps"
+ "slices"
"time"
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/robfig/cron/v3"
- "github.com/samber/mo"
)
// cronScheduleParser is the shared parser for all cron settings: six fields
@@ -18,115 +19,164 @@ import (
// schedule with the same spec so Jobs-UI cron values behave identically.
var cronScheduleParser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)
-type JobScheduler struct {
- // mu guards jobs, jobsByID, entryIDs and schedules. It is held across the cron
- // add/remove calls (which are themselves quick and never block on job
- // execution) but never across job.Run — Run executes on cron's own
- // goroutine, so a running job must not call back into a locking method here.
- mu sync.Mutex
- cron *cron.Cron
- jobs []schedulertypes.Job
- jobsByID map[string]schedulertypes.Job
- watchers map[string]schedulertypes.BusWatcher
- entryIDs map[string]cron.EntryID
- schedules map[string]string
- parser cron.Parser
- context context.Context
- location *time.Location
- watcherWG sync.WaitGroup
+var errJobSchedulerStoppedInternal = errors.Sentinel("job scheduler stopped")
+
+type schedulerStateInternal struct {
+ jobs []schedulertypes.Job
+ jobsByID map[string]schedulertypes.Job
+ watchers map[string]schedulertypes.BusWatcher
+ allWatchers map[string]schedulertypes.BusWatcher
+ runners map[string]*actors.Runner
+ entryIDs map[string]cron.EntryID
+ schedules map[string]string
+ stopping bool
}
-// NewJobScheduler creates a new job scheduler with the specified timezone location.
-// The location is used for interpreting cron expressions.
-// If location is nil, UTC is used.
-func NewJobScheduler(ctx context.Context, location *time.Location) *JobScheduler {
+type watcherRegistrationInternal struct {
+ watcher schedulertypes.BusWatcher
+ runner *actors.Runner
+}
+
+type schedulerShutdownInternal struct {
+ cronDone <-chan struct{}
+ watchers []watcherRegistrationInternal
+}
+
+// jobSchedulerInternal serializes its control plane through one shared actor executor.
+// robfig/cron remains the timing engine and invokes job bodies outside the actor.
+type jobSchedulerInternal struct {
+ cron *cron.Cron
+ parser cron.Parser
+ context context.Context
+ location *time.Location
+ runtime *actors.Runtime
+ state *actors.State[schedulerStateInternal]
+}
+
+// NewJobScheduler creates an actor-owned scheduler control plane.
+func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *time.Location) (schedulertypes.JobScheduler, error) {
+ if ctx == nil {
+ return nil, errors.New("job scheduler context unavailable")
+ }
+ if runtime == nil {
+ return nil, errors.New("actor runtime unavailable")
+ }
if location == nil {
location = time.UTC
}
+ initial := schedulerStateInternal{
+ jobs: []schedulertypes.Job{},
+ jobsByID: make(map[string]schedulertypes.Job),
+ watchers: make(map[string]schedulertypes.BusWatcher),
+ allWatchers: make(map[string]schedulertypes.BusWatcher),
+ runners: make(map[string]*actors.Runner),
+ entryIDs: make(map[string]cron.EntryID),
+ schedules: make(map[string]string),
+ }
+ state, err := actors.NewState(ctx, runtime, "scheduler", "control-plane", 3, initial, func(value schedulerStateInternal) schedulerStateInternal {
+ value.jobs = slices.Clone(value.jobs)
+ value.jobsByID = maps.Clone(value.jobsByID)
+ value.watchers = maps.Clone(value.watchers)
+ value.allWatchers = maps.Clone(value.allWatchers)
+ value.runners = maps.Clone(value.runners)
+ value.entryIDs = maps.Clone(value.entryIDs)
+ value.schedules = maps.Clone(value.schedules)
+ return value
+ })
+ if err != nil {
+ return nil, err
+ }
parser := cronScheduleParser
- slog.InfoContext(ctx, "Initializing job scheduler", "timezone", location.String())
- return &JobScheduler{
- cron: cron.New(cron.WithParser(parser), cron.WithLocation(location)),
- jobs: []schedulertypes.Job{},
- jobsByID: make(map[string]schedulertypes.Job),
- watchers: make(map[string]schedulertypes.BusWatcher),
- entryIDs: make(map[string]cron.EntryID),
- schedules: make(map[string]string),
- parser: parser,
- context: ctx,
- location: location,
+ scheduler := &jobSchedulerInternal{
+ cron: cron.New(cron.WithParser(parser), cron.WithLocation(location)),
+ parser: parser,
+ context: ctx,
+ location: location,
+ runtime: runtime,
+ state: state,
}
+ slog.InfoContext(ctx, "Initializing job scheduler", "timezone", location.String())
+ return scheduler, nil
}
-// RegisterJob records a static job to be scheduled when StartScheduler runs. Use
-// AddJob for jobs added dynamically at runtime.
-func (js *JobScheduler) RegisterJob(job schedulertypes.Job) {
- js.mu.Lock()
- defer js.mu.Unlock()
- js.jobs = append(js.jobs, job)
- js.jobsByID[job.Name()] = job
+// RegisterJob records a static job to be scheduled when StartScheduler runs.
+func (js *jobSchedulerInternal) RegisterJob(job schedulertypes.Job) error {
+ if job == nil {
+ return errors.New("scheduler job unavailable")
+ }
+ return js.state.Apply(js.context, "register scheduler job", func(_ context.Context, state *schedulerStateInternal) error {
+ if state.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ state.jobs = append(state.jobs, job)
+ state.jobsByID[job.Name()] = job
+ return nil
+ })
}
-// RegisterBusWatcher starts a continuous event watcher on the scheduler lifecycle.
-func (js *JobScheduler) RegisterBusWatcher(watcher schedulertypes.BusWatcher, canRunManually bool) {
+// RegisterBusWatcher starts a continuous watcher through a shared actor runner.
+func (js *jobSchedulerInternal) RegisterBusWatcher(watcher schedulertypes.BusWatcher, canRunManually bool) error {
if watcher == nil {
- return
+ return errors.New("scheduler bus watcher unavailable")
}
- if canRunManually {
- js.mu.Lock()
- js.watchers[watcher.Name()] = watcher
- js.mu.Unlock()
- }
-
- js.watcherWG.Go(func() {
- // A panic in a watcher goroutine would crash the process; contain it the
- // same way scheduled job runs are contained.
- defer utils.RecoverToError(nil, "bus watcher", "name", watcher.Name())
- if err := watcher.Start(js.context); err != nil {
- slog.ErrorContext(js.context, "Bus watcher failed", "name", watcher.Name(), "error", err)
+ return js.state.Apply(js.context, "register scheduler bus watcher", func(_ context.Context, state *schedulerStateInternal) error {
+ if state.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ runner, err := actors.NewRunner(js.context, js.runtime, "scheduler-watchers", watcher.Name(), "bus watcher "+watcher.Name(), 3, watcher.Start) //nolint:contextcheck // watcher lifetime belongs to the scheduler, not one state-actor receiver incarnation.
+ if err != nil {
+ return err
}
+ state.runners[watcher.Name()] = runner
+ state.allWatchers[watcher.Name()] = watcher
+ if canRunManually {
+ state.watchers[watcher.Name()] = watcher
+ }
+ return nil
})
}
// RunBusWatcherNow runs a watcher through its serialized manual path.
-func (js *JobScheduler) RunBusWatcherNow(ctx context.Context, watcherID string) error {
- js.mu.Lock()
- watcher, ok := js.watchers[watcherID]
- js.mu.Unlock()
+func (js *jobSchedulerInternal) RunBusWatcherNow(ctx context.Context, watcherID string) error {
+ snapshot, ok := js.state.Load()
+ if !ok {
+ return errors.New("job scheduler state unavailable")
+ }
+ watcher, ok := snapshot.watchers[watcherID]
if !ok {
return errors.Errorf("bus watcher %s is not manually runnable", watcherID)
}
-
return watcher.RunNow(ctx)
}
-func (js *JobScheduler) GetJob(jobID string) mo.Option[schedulertypes.Job] {
- js.mu.Lock()
- defer js.mu.Unlock()
- job, ok := js.jobsByID[jobID]
- return mo.TupleToOption(job, ok)
+func (js *jobSchedulerInternal) GetJob(jobID string) (schedulertypes.Job, bool) {
+ snapshot, ok := js.state.Load()
+ if !ok {
+ return nil, false
+ }
+ job, ok := snapshot.jobsByID[jobID]
+ return job, ok
}
// GetJobRuntimeState returns the schedule currently installed for a registered job.
-func (js *JobScheduler) GetJobRuntimeState(jobID string) mo.Option[schedulertypes.JobRuntimeState] {
- js.mu.Lock()
- defer js.mu.Unlock()
-
- if _, ok := js.jobsByID[jobID]; !ok {
- return mo.None[schedulertypes.JobRuntimeState]()
+func (js *jobSchedulerInternal) GetJobRuntimeState(jobID string) (schedulertypes.JobRuntimeState, bool) {
+ snapshot, ok := js.state.Load()
+ if !ok {
+ return schedulertypes.JobRuntimeState{}, false
+ }
+ if _, ok := snapshot.jobsByID[jobID]; !ok {
+ return schedulertypes.JobRuntimeState{}, false
}
- state := schedulertypes.JobRuntimeState{Schedule: js.schedules[jobID]}
- entryID, ok := js.entryIDs[jobID]
+ state := schedulertypes.JobRuntimeState{Schedule: snapshot.schedules[jobID]}
+ entryID, ok := snapshot.entryIDs[jobID]
if !ok {
- return mo.Some(state)
+ return state, true
}
-
entry := js.cron.Entry(entryID)
if entry.ID == 0 {
- return mo.Some(state)
+ return state, true
}
-
state.Scheduled = true
nextRun := entry.Next
if nextRun.IsZero() && entry.Schedule != nil {
@@ -135,102 +185,139 @@ func (js *JobScheduler) GetJobRuntimeState(jobID string) mo.Option[schedulertype
if !nextRun.IsZero() {
state.NextRun = new(nextRun)
}
-
- return mo.Some(state)
+ return state, true
}
-// HasJob reports whether a job with the given name is currently registered.
-func (js *JobScheduler) HasJob(jobID string) bool {
- js.mu.Lock()
- defer js.mu.Unlock()
- _, ok := js.jobsByID[jobID]
+// HasJob reports whether a job with the given name is registered.
+func (js *jobSchedulerInternal) HasJob(jobID string) bool {
+ snapshot, ok := js.state.Load()
+ if !ok {
+ return false
+ }
+ _, ok = snapshot.jobsByID[jobID]
return ok
}
-func (js *JobScheduler) StartScheduler() {
- js.mu.Lock()
- for _, job := range js.jobs {
- if err := js.upsertJobInternal(js.context, job); err != nil {
- slog.ErrorContext(js.context, "Failed to schedule job", "name", job.Name(), "error", err)
+// StartScheduler installs static schedules and starts cron.
+func (js *jobSchedulerInternal) StartScheduler() error {
+ return js.state.Apply(js.context, "start job scheduler", func(actorCtx context.Context, state *schedulerStateInternal) error {
+ if state.stopping {
+ return errJobSchedulerStoppedInternal
}
- }
- js.mu.Unlock()
- js.cron.Start()
+ for _, job := range state.jobs {
+ if err := js.upsertJobInternal(actorCtx, state, job); err != nil {
+ slog.ErrorContext(actorCtx, "Failed to schedule job; continuing scheduler startup", "name", job.Name(), "error", err)
+ }
+ }
+ js.cron.Start()
+ return nil
+ })
}
-// AddJob registers and schedules a job at runtime. It is an idempotent upsert: a
-// replacement expression is validated before the existing entry is removed, then
-// the replacement is installed without leaking a second live entry.
-// Safe to call before or after StartScheduler.
-func (js *JobScheduler) AddJob(ctx context.Context, job schedulertypes.Job) error {
- js.mu.Lock()
- defer js.mu.Unlock()
- return js.upsertJobInternal(ctx, job)
+// AddJob registers and schedules a job at runtime.
+func (js *jobSchedulerInternal) AddJob(ctx context.Context, job schedulertypes.Job) error {
+ return js.state.Apply(ctx, "add scheduler job", func(_ context.Context, state *schedulerStateInternal) error {
+ if state.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ return js.upsertJobInternal(ctx, state, job)
+ })
}
-// RemoveJob unschedules and forgets a job by name. It is a no-op (not an error)
-// when no job with that name is registered.
-func (js *JobScheduler) RemoveJob(ctx context.Context, jobName string) {
- js.mu.Lock()
- defer js.mu.Unlock()
-
- if entryID, ok := js.entryIDs[jobName]; ok {
- js.cron.Remove(entryID)
- delete(js.entryIDs, jobName)
- }
- delete(js.jobsByID, jobName)
- delete(js.schedules, jobName)
- for i, j := range js.jobs {
- if j.Name() == jobName {
- js.jobs = append(js.jobs[:i], js.jobs[i+1:]...)
- break
+// RemoveJob unschedules and forgets a job. Actor failures are logged because
+// DynamicScheduler preserves the historical no-result removal contract.
+func (js *jobSchedulerInternal) RemoveJob(ctx context.Context, jobName string) {
+ err := js.state.Apply(ctx, "remove scheduler job", func(_ context.Context, state *schedulerStateInternal) error {
+ if state.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ if entryID, ok := state.entryIDs[jobName]; ok {
+ js.cron.Remove(entryID)
+ delete(state.entryIDs, jobName)
+ }
+ delete(state.jobsByID, jobName)
+ delete(state.schedules, jobName)
+ for index, job := range state.jobs {
+ if job.Name() == jobName {
+ state.jobs = append(state.jobs[:index], state.jobs[index+1:]...)
+ break
+ }
}
+ slog.DebugContext(ctx, "Job removed", "name", jobName)
+ return nil
+ })
+ if err != nil && !errors.Is(err, errJobSchedulerStoppedInternal) {
+ slog.ErrorContext(ctx, "Failed to remove scheduler job", "name", jobName, "error", err)
}
- slog.DebugContext(ctx, "Job removed", "name", jobName)
}
-func (js *JobScheduler) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {
- js.mu.Lock()
- defer js.mu.Unlock()
- return js.upsertJobInternal(ctx, job)
+func (js *jobSchedulerInternal) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {
+ return js.state.Apply(ctx, "reschedule job", func(_ context.Context, state *schedulerStateInternal) error {
+ if state.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ return js.upsertJobInternal(ctx, state, job)
+ })
}
-// GetLocation returns the timezone location used by the scheduler for cron expressions.
-func (js *JobScheduler) GetLocation() *time.Location {
+func (js *jobSchedulerInternal) GetLocation() *time.Location {
return js.location
}
-// Stop stops the scheduler and waits for running jobs and bus watchers to finish.
-// If ctx is canceled or reaches its deadline first, Stop returns ctx.Err().
-func (js *JobScheduler) Stop(ctx context.Context) error {
- cronDone := js.cron.Stop().Done()
- watchersDone := make(chan struct{})
- go func() {
- js.watcherWG.Wait()
- close(watchersDone)
- }()
-
- select {
- case <-cronDone:
- case <-ctx.Done():
- return ctx.Err()
+// Stop fences the control plane, stops cron, and joins every actor-owned watcher.
+func (js *jobSchedulerInternal) Stop(ctx context.Context) error {
+ var shutdown schedulerShutdownInternal
+ err := js.state.Apply(ctx, "stop job scheduler", func(_ context.Context, state *schedulerStateInternal) error {
+ if state.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ state.stopping = true
+ registrations := make([]watcherRegistrationInternal, 0, len(state.runners))
+ for name, runner := range state.runners {
+ registrations = append(registrations, watcherRegistrationInternal{watcher: state.allWatchers[name], runner: runner})
+ }
+ shutdown = schedulerShutdownInternal{cronDone: js.cron.Stop().Done(), watchers: registrations}
+ return nil
+ })
+ if errors.Is(err, errJobSchedulerStoppedInternal) {
+ return js.state.Stop(ctx)
+ }
+ var stopErr error
+ if err != nil {
+ if snapshot, ok := js.state.Load(); ok && snapshot.stopping {
+ return nil
+ }
+ stopErr = err
+ shutdown.cronDone = js.cron.Stop().Done()
+ if snapshot, ok := js.state.Load(); ok {
+ shutdown.watchers = make([]watcherRegistrationInternal, 0, len(snapshot.runners))
+ for name, runner := range snapshot.runners {
+ shutdown.watchers = append(shutdown.watchers, watcherRegistrationInternal{watcher: snapshot.allWatchers[name], runner: runner})
+ }
+ }
}
+ for _, registration := range shutdown.watchers {
+ stopErr = errors.Combine(stopErr, registration.runner.Stop(ctx))
+ if stopper, ok := registration.watcher.(schedulertypes.StoppableBusWatcher); ok {
+ stopErr = errors.Combine(stopErr, stopper.Stop(ctx))
+ }
+ }
select {
- case <-watchersDone:
- return nil
+ case <-shutdown.cronDone:
case <-ctx.Done():
- return ctx.Err()
+ stopErr = errors.Combine(stopErr, ctx.Err())
}
+ return errors.Combine(stopErr, js.state.Stop(ctx))
}
-// upsertJobInternal records the job and (re)schedules it. A replacement expression
-// is parsed before the previous entry is removed so invalid input leaves the last
-// valid entry untouched. Callers must hold js.mu.
-func (js *JobScheduler) upsertJobInternal(ctx context.Context, job schedulertypes.Job) error {
+func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *schedulerStateInternal, job schedulertypes.Job) error {
+ if job == nil {
+ return errors.New("scheduler job unavailable")
+ }
jobName := job.Name()
- previousSchedule := js.schedules[jobName]
- previousEntryID, hadPreviousEntry := js.entryIDs[jobName]
+ previousSchedule := state.schedules[jobName]
+ previousEntryID, hadPreviousEntry := state.entryIDs[jobName]
schedule := job.Schedule(ctx)
shouldSchedule := true
@@ -255,33 +342,26 @@ func (js *JobScheduler) upsertJobInternal(ctx context.Context, job schedulertype
if hadPreviousEntry {
js.cron.Remove(previousEntryID)
- delete(js.entryIDs, jobName)
+ delete(state.entryIDs, jobName)
}
if shouldSchedule {
entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule)
- js.entryIDs[jobName] = entryID
+ state.entryIDs[jobName] = entryID
}
-
- js.jobsByID[jobName] = job
- js.schedules[jobName] = schedule
+ state.jobsByID[jobName] = job
+ state.schedules[jobName] = schedule
if previousSchedule == "" && shouldSchedule {
slog.InfoContext(ctx, "Starting Job", "name", jobName, "schedule", schedule)
} else if previousSchedule != schedule || hadPreviousEntry != shouldSchedule {
slog.InfoContext(ctx, "Job rescheduled", "name", jobName, "previousSchedule", previousSchedule, "newSchedule", schedule, "nextRun", nextRun)
}
-
slog.DebugContext(ctx, "Job scheduled", "name", jobName, "scheduled", shouldSchedule, "contextCanceled", ctx.Err() != nil)
return nil
}
-// addCronEntryInternal adds a job closure that always runs with the scheduler's
-// lifecycle context. Callers must hold js.mu.
-func (js *JobScheduler) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {
+func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {
entryID := js.cron.Schedule(parsedSchedule, cron.FuncJob(func() {
- // robfig/cron has no default panic recovery, so a panic in any job would
- // otherwise take down the whole process. Recovering here covers every
- // scheduled job in one place.
defer utils.RecoverToError(nil, "scheduled job", "name", job.Name(), "schedule", schedule)
slog.InfoContext(js.context, "Job starting", "name", job.Name(), "schedule", schedule)
job.Run(js.context)
@@ -296,6 +376,5 @@ func (js *JobScheduler) addCronEntryInternal(job schedulertypes.Job, schedule st
if nextRun.IsZero() {
return entryID, nil
}
-
return entryID, new(nextRun)
}
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index 51dff1105..e5716e65c 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"time"
+ "emperror.dev/errors"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/stretchr/testify/require"
)
@@ -38,6 +39,36 @@ type testBusWatcherInternal struct {
ranNow chan context.Context
}
+type orderedStopBusWatcherInternal struct {
+ name string
+ started chan struct{}
+ runnerExited chan struct{}
+}
+
+func (w *orderedStopBusWatcherInternal) Name() string {
+ return w.name
+}
+
+func (w *orderedStopBusWatcherInternal) Start(ctx context.Context) error {
+ close(w.started)
+ <-ctx.Done()
+ close(w.runnerExited)
+ return nil
+}
+
+func (*orderedStopBusWatcherInternal) RunNow(context.Context) error {
+ return nil
+}
+
+func (w *orderedStopBusWatcherInternal) Stop(context.Context) error {
+ select {
+ case <-w.runnerExited:
+ return nil
+ default:
+ return errors.New("watcher stopped before its runner exited")
+ }
+}
+
func (w *testBusWatcherInternal) RunNow(ctx context.Context) error {
if w.ranNow != nil {
w.ranNow <- ctx
@@ -63,7 +94,7 @@ func (j *conditionalTestSchedulerJob) ShouldSchedule(ctx context.Context) bool {
}
func TestJobScheduler_StartScheduler_SkipsDisabledConditionalJobs(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
job := &conditionalTestSchedulerJob{
testSchedulerJob: &testSchedulerJob{
@@ -73,17 +104,37 @@ func TestJobScheduler_StartScheduler_SkipsDisabledConditionalJobs(t *testing.T)
shouldSchedule: func(context.Context) bool { return false },
}
- js.RegisterJob(job)
- js.StartScheduler()
+ require.NoError(t, js.RegisterJob(job))
+ require.NoError(t, js.StartScheduler())
defer js.cron.Stop()
- require.NotContains(t, js.entryIDs, job.Name())
+ state, ok := js.GetJobRuntimeState(job.Name())
+ require.True(t, ok)
+ require.False(t, state.Scheduled)
require.Empty(t, js.cron.Entries())
}
+func TestJobScheduler_StartScheduler_ContinuesAfterInvalidJobSchedule(t *testing.T) {
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ invalid := &testSchedulerJob{name: "invalid-startup-job", schedule: "not a cron schedule"}
+ valid := &testSchedulerJob{name: "valid-startup-job", schedule: "*/1 * * * * *"}
+
+ require.NoError(t, js.RegisterJob(invalid))
+ require.NoError(t, js.RegisterJob(valid))
+ require.NoError(t, js.StartScheduler())
+
+ invalidState, ok := js.GetJobRuntimeState(invalid.Name())
+ require.True(t, ok)
+ require.False(t, invalidState.Scheduled)
+ validState, ok := js.GetJobRuntimeState(valid.Name())
+ require.True(t, ok)
+ require.True(t, validState.Scheduled)
+ require.Len(t, js.cron.Entries(), 1)
+}
+
func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
- js := NewJobScheduler(ctx, nil)
+ js := newJobSchedulerForTestInternal(t, ctx, nil)
watcher := &testBusWatcherInternal{
name: "test-bus-watcher",
started: make(chan struct{}),
@@ -91,8 +142,8 @@ func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {
ranNow: make(chan context.Context, 1),
}
- js.RegisterBusWatcher(watcher, true)
- js.StartScheduler()
+ require.NoError(t, js.RegisterBusWatcher(watcher, true))
+ require.NoError(t, js.StartScheduler())
select {
case <-watcher.started:
@@ -117,10 +168,27 @@ func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {
}
}
+func TestJobScheduler_StopJoinsRunnerBeforeStoppingWatcher(t *testing.T) {
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ watcher := &orderedStopBusWatcherInternal{
+ name: "ordered-stop-watcher",
+ started: make(chan struct{}),
+ runnerExited: make(chan struct{}),
+ }
+
+ require.NoError(t, js.RegisterBusWatcher(watcher, false))
+ select {
+ case <-watcher.started:
+ case <-time.After(time.Second):
+ t.Fatal("timed out waiting for bus watcher to start")
+ }
+ require.NoError(t, js.Stop(t.Context()))
+}
+
func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {
ctx := t.Context()
- js := NewJobScheduler(ctx, nil)
+ js := newJobSchedulerForTestInternal(t, ctx, nil)
manualWatcher := &testBusWatcherInternal{
name: "manual-bus-watcher",
started: make(chan struct{}),
@@ -133,8 +201,8 @@ func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {
stopped: make(chan struct{}),
}
- js.RegisterBusWatcher(manualWatcher, true)
- js.RegisterBusWatcher(automaticOnlyWatcher, false)
+ require.NoError(t, js.RegisterBusWatcher(manualWatcher, true))
+ require.NoError(t, js.RegisterBusWatcher(automaticOnlyWatcher, false))
runCtx := context.Background()
require.NoError(t, js.RunBusWatcherNow(runCtx, manualWatcher.Name()))
@@ -143,7 +211,7 @@ func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {
}
func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
enabled := true
job := &conditionalTestSchedulerJob{
@@ -155,17 +223,21 @@ func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {
}
require.NoError(t, js.RescheduleJob(context.Background(), job))
- require.Contains(t, js.entryIDs, job.Name())
+ state, ok := js.GetJobRuntimeState(job.Name())
+ require.True(t, ok)
+ require.True(t, state.Scheduled)
enabled = false
require.NoError(t, js.RescheduleJob(context.Background(), job))
- require.NotContains(t, js.entryIDs, job.Name())
+ state, ok = js.GetJobRuntimeState(job.Name())
+ require.True(t, ok)
+ require.False(t, state.Scheduled)
require.Empty(t, js.cron.Entries())
}
func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
enabled := false
job := &conditionalTestSchedulerJob{
@@ -177,33 +249,39 @@ func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {
}
require.NoError(t, js.RescheduleJob(context.Background(), job))
- require.NotContains(t, js.entryIDs, job.Name())
+ state, ok := js.GetJobRuntimeState(job.Name())
+ require.True(t, ok)
+ require.False(t, state.Scheduled)
enabled = true
require.NoError(t, js.RescheduleJob(context.Background(), job))
- require.Contains(t, js.entryIDs, job.Name())
+ state, ok = js.GetJobRuntimeState(job.Name())
+ require.True(t, ok)
+ require.True(t, state.Scheduled)
require.Len(t, js.cron.Entries(), 1)
}
func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
job := &testSchedulerJob{
name: "test-non-conditional-startup",
schedule: "*/1 * * * * *",
}
- js.RegisterJob(job)
- js.StartScheduler()
+ require.NoError(t, js.RegisterJob(job))
+ require.NoError(t, js.StartScheduler())
defer js.cron.Stop()
- require.Contains(t, js.entryIDs, job.Name())
+ state, ok := js.GetJobRuntimeState(job.Name())
+ require.True(t, ok)
+ require.True(t, state.Scheduled)
require.Len(t, js.cron.Entries(), 1)
}
func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
var once sync.Once
runErrCh := make(chan error, 1)
@@ -231,7 +309,7 @@ func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {
func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T) {
lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())
- js := NewJobScheduler(lifecycleCtx, nil)
+ js := newJobSchedulerForTestInternal(t, lifecycleCtx, nil)
startedCh := make(chan struct{}, 1)
stoppedCh := make(chan struct{}, 1)
@@ -271,22 +349,22 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T
}
func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
job := &testSchedulerJob{name: "dyn-upsert", schedule: "*/5 * * * * *"}
require.NoError(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
require.Len(t, js.cron.Entries(), 1)
- firstEntry := js.entryIDs[job.Name()]
+ firstEntry := js.cron.Entries()[0].ID
// Re-adding with a changed schedule (e.g. a new sync interval) must replace the
// existing cron entry, not leak a second one that keeps firing forever.
job.schedule = "*/10 * * * * *"
require.NoError(t, js.AddJob(context.Background(), job))
require.Len(t, js.cron.Entries(), 1)
- require.NotEqual(t, firstEntry, js.entryIDs[job.Name()])
+ require.NotEqual(t, firstEntry, js.cron.Entries()[0].ID)
- state, ok := js.GetJobRuntimeState(job.Name()).Get()
+ state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
require.True(t, state.Scheduled)
require.Equal(t, "*/10 * * * * *", state.Schedule)
@@ -294,28 +372,28 @@ func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {
}
func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
job := &testSchedulerJob{name: "dyn-invalid-reschedule", schedule: "*/5 * * * * *"}
require.NoError(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
require.Len(t, js.cron.Entries(), 1)
- firstEntry := js.entryIDs[job.Name()]
+ firstEntry := js.cron.Entries()[0].ID
job.schedule = "not a cron schedule"
require.Error(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
- require.Equal(t, firstEntry, js.entryIDs[job.Name()])
+ require.Equal(t, firstEntry, js.cron.Entries()[0].ID)
require.Len(t, js.cron.Entries(), 1)
- state, ok := js.GetJobRuntimeState(job.Name()).Get()
+ state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
require.True(t, state.Scheduled)
require.Equal(t, "*/5 * * * * *", state.Schedule)
}
func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
// Removing an unknown job must be a safe no-op (e.g. deleting a sync that never
// had auto-sync enabled).
@@ -328,12 +406,11 @@ func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
js.RemoveJob(context.Background(), job.Name())
require.False(t, js.HasJob(job.Name()))
- require.NotContains(t, js.entryIDs, job.Name())
require.Empty(t, js.cron.Entries())
}
func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {
- js := NewJobScheduler(context.Background(), nil)
+ js := newJobSchedulerForTestInternal(t, context.Background(), nil)
job := &schedulertypes.GenericJob{
JobName: "generic-dyn",
@@ -353,8 +430,8 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {
jobFinished := make(chan struct{}, 1)
var runOnce sync.Once
- js := NewJobScheduler(lifecycleCtx, nil)
- js.RegisterJob(&testSchedulerJob{
+ js := newJobSchedulerForTestInternal(t, lifecycleCtx, nil)
+ require.NoError(t, js.RegisterJob(&testSchedulerJob{
name: "test-shutdown-waits",
schedule: "*/1 * * * * *",
run: func(ctx context.Context) {
@@ -366,8 +443,8 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {
jobFinished <- struct{}{}
})
},
- })
- js.StartScheduler()
+ }))
+ require.NoError(t, js.StartScheduler())
t.Cleanup(func() {
cancelLifecycle()
select {
diff --git a/backend/pkg/utils/waitgroup.go b/backend/pkg/utils/waitgroup.go
new file mode 100644
index 000000000..29ce376e9
--- /dev/null
+++ b/backend/pkg/utils/waitgroup.go
@@ -0,0 +1,27 @@
+package utils
+
+import (
+ "context"
+ "sync"
+
+ "emperror.dev/errors"
+)
+
+// WaitGroup waits for all work or returns when ctx ends. After a timeout, its
+// waiter goroutine remains until the group eventually completes.
+func WaitGroup(ctx context.Context, group *sync.WaitGroup) error {
+ if ctx == nil || group == nil {
+ return errors.New("wait group context unavailable")
+ }
+ done := make(chan struct{})
+ go func() {
+ group.Wait()
+ close(done)
+ }()
+ select {
+ case <-done:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
diff --git a/cli/go.mod b/cli/go.mod
index fb473bf07..be2c8963a 100644
--- a/cli/go.mod
+++ b/cli/go.mod
@@ -58,7 +58,7 @@ require (
github.com/mattn/go-shellwords v1.0.13 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/moby/api v1.55.0 // indirect
- github.com/moby/moby/client v0.5.0 // indirect
+ github.com/moby/moby/client v0.5.1 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/mango v0.2.0 // indirect
github.com/muesli/mango-cobra v1.3.0 // indirect
@@ -92,7 +92,7 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
- golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect
+ golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
diff --git a/cli/go.sum b/cli/go.sum
index 5d79d7ee2..582eb5aa2 100644
--- a/cli/go.sum
+++ b/cli/go.sum
@@ -109,8 +109,8 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
-github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
-github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
+github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw=
+github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM=
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/muesli/mango v0.2.0 h1:iNNc0c5VLQ6fsMgAqGQofByNUBH2Q2nEbD6TaI+5yyQ=
@@ -208,8 +208,8 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
-golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q=
-golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw=
+golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A=
+golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
diff --git a/go.work.sum b/go.work.sum
index b370c7937..0ede2f355 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -517,6 +517,8 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc=
@@ -544,6 +546,8 @@ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkH
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.9 h1:FjwMGxhQ6uY31o+leQBWtvZu8z0vfkDsuG7IpaI6goo=
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.9/go.mod h1:lTA9XMO2us+m6TK8wzbL7cOYmdPLnSV1T5aRz6Sv4rE=
+github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.5 h1:7ZFdtE1XEHH+58GZU4Mbhq6SO/UbColleDApOtlv3vo=
+github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.5/go.mod h1:M/qt7xBBXqilBwNmrO1yiu0cywZwQx5aqtKpdJN9J2A=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 h1:IdCLsiiIj5YJ3AFevsewURCPV+YWUlOW8JiPhoAy8vg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4/go.mod h1:l4bdfCD7XyyZA9BolKBo1eLqgaJxl0/x91PL4Yqe0ao=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
@@ -582,6 +586,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
@@ -595,6 +601,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNE
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po=
github.com/aws/aws-sdk-go-v2/service/kms v1.52.0 h1:QNtg+Mtj1zmepk568+UKBD5DFfqh+ESTUUqQT27JkQc=
github.com/aws/aws-sdk-go-v2/service/kms v1.52.0/go.mod h1:Y0+uxvxz6ib4KktRdK0V4X45Vcs/JyYoz8H71pO8xeI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw=
@@ -603,6 +611,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3 h1:JRseEu/vIDMaWis4bSw0QbXL+cvIGc1XnX076H5ZXLE=
github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
github.com/aws/aws-sdk-go-v2/service/signin v1.1.5/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ=
@@ -699,6 +709,8 @@ github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8 h1:jP7ki8Tzx9
github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY=
github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM=
github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo=
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -778,8 +790,6 @@ github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcK
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g=
github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
-github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
-github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
github.com/compose-spec/compose-go/v2 v2.9.1/go.mod h1:Oky9AZGTRB4E+0VbTPZTUu4Kp+oEMMuwZXZtPPVT1iE=
@@ -982,6 +992,7 @@ github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/X
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fernet/fernet-go v0.0.0-20211208181803-9f70042a33ee h1:v6Eju/FhxsACGNipFEPBZZAzGr1F/jlRQr1qiBw2nEE=
github.com/fernet/fernet-go v0.0.0-20211208181803-9f70042a33ee/go.mod h1:2H9hjfbpSMHwY503FclkV/lZTBh2YlOmLLSda12uL8c=
github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e h1:Ss/B3/5wWRh8+emnK0++g5zQzwDTi30W10pKxKc4JXI=
@@ -1138,6 +1149,8 @@ github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6x
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg=
+github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ=
github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
@@ -1258,6 +1271,8 @@ github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
+github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
+github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs=
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
@@ -1531,6 +1546,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
@@ -1550,6 +1567,8 @@ github.com/microsoft/go-mssqldb v1.9.8 h1:d4IFMvF/o+HdpXUqbBfzHvn/NlFA75YGcfHUUv
github.com/microsoft/go-mssqldb v1.9.8/go.mod h1:eGSRSGAW4hKMy5YcAenhCDjIRm2rhqIdmmwgciMzLus=
github.com/microsoft/go-mssqldb v1.10.0 h1:pHEt+Qz6YFPWqREq10mqSE524QQo+/QremwTCQht7TY=
github.com/microsoft/go-mssqldb v1.10.0/go.mod h1:mnG7lGa9iYJbzJqGCXyuQCegStKMr3kogDLD6+bmggg=
+github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
+github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
@@ -1610,6 +1629,8 @@ github.com/moby/sys/mount v0.3.4 h1:yn5jq4STPztkkzSKpZkLcmjue+bZJ0u2AuQY1iNI1Ww=
github.com/moby/sys/mount v0.3.4/go.mod h1:KcQJMbQdJHPlq5lcYT+/CjatWM4PuxKe+XLSVS4J6Os=
github.com/moby/sys/mount v0.3.5-0.20260529155943-fc52b7222d0b h1:RB/940R3dFkU1fLLk5rBIvcv/C3HfbjqVkStfi/wlb4=
github.com/moby/sys/mount v0.3.5-0.20260529155943-fc52b7222d0b/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M=
+github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00=
+github.com/moby/sys/mount v0.3.5/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M=
github.com/moby/sys/reexec v0.1.0 h1:RrBi8e0EBTLEgfruBOFcxtElzRGTEUkeIFaVXgU7wok=
github.com/moby/sys/reexec v0.1.0/go.mod h1:EqjBg8F3X7iZe5pU6nRZnYCMUTXoxsjiIfHup5wYIN8=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
@@ -2082,10 +2103,10 @@ github.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0
github.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
+github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g=
+github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
-github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
github.com/zmap/zcrypto v0.0.0-20210511125630-18f1e0152cfc h1:zkGwegkOW709y0oiAraH/3D8njopUR/pARHv4tZZ6pw=
@@ -2103,6 +2124,8 @@ go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVi
go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
+go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
+go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
go.etcd.io/etcd/api/v3 v3.6.8 h1:gqb1VN92TAI6G2FiBvWcqKtHiIjr4SU2GdXxTwyexbM=
go.etcd.io/etcd/api/v3 v3.6.8/go.mod h1:qyQj1HZPUV3B5cbAL8scG62+fyz5dSxxu0w8pn28N6Q=
go.etcd.io/etcd/client/pkg/v3 v3.5.6 h1:TXQWYceBKqLp4sa87rcPs11SXxUA/mHwH975v+BDvLU=
@@ -2189,6 +2212,8 @@ go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PD
go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk=
go.opentelemetry.io/otel/exporters/prometheus v0.65.0 h1:jOveH/b4lU9HT7y+Gfamf18BqlOuz2PWEvs8yM7Q6XE=
go.opentelemetry.io/otel/exporters/prometheus v0.65.0/go.mod h1:i1P8pcumauPtUI4YNopea1dhzEMuEqWP1xoUZDylLHo=
+go.opentelemetry.io/otel/exporters/prometheus v0.66.0 h1:vkrK8PAznv2NKt2r+kdu252ccGzkEqLc2aSXbQIALYQ=
+go.opentelemetry.io/otel/exporters/prometheus v0.66.0/go.mod h1:V/UB6D3vMF/UBOL5igAsAYnk1nG/bzYYTzvsB16cy7o=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk=
@@ -2283,8 +2308,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
-golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A=
-golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -2546,6 +2569,7 @@ golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
@@ -2835,6 +2859,8 @@ sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k=
software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=
+storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI=
+storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4=
tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=
tags.cncf.io/container-device-interface/specs-go v1.0.0 h1:8gLw29hH1ZQP9K1YtAzpvkHCjjyIxHZYzBAvlQ+0vD8=
tags.cncf.io/container-device-interface/specs-go v1.0.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=
diff --git a/types/scheduler/job.go b/types/scheduler/job.go
index 79e8ad30a..80872402c 100644
--- a/types/scheduler/job.go
+++ b/types/scheduler/job.go
@@ -32,6 +32,41 @@ type BusWatcher interface {
RunNow(ctx context.Context) error
}
+// StoppableBusWatcher lets a continuous watcher perform explicit shutdown work
+// before its actor runner is joined.
+type StoppableBusWatcher interface {
+ BusWatcher
+ Stop(ctx context.Context) error
+}
+
+// DynamicScheduler owns jobs registered and removed while the application is running.
+type DynamicScheduler interface {
+ AddJob(ctx context.Context, job Job) error
+ RemoveJob(ctx context.Context, name string)
+ HasJob(name string) bool
+}
+
+// JobController exposes scheduler operations used by job-management services.
+type JobController interface {
+ GetJob(jobID string) (Job, bool)
+ GetJobRuntimeState(jobID string) (JobRuntimeState, bool)
+ RescheduleJob(ctx context.Context, job Job) error
+ RunBusWatcherNow(ctx context.Context, watcherID string) error
+}
+
+// JobScheduler is the shared public contract for the actor-owned scheduler.
+// Concrete cron and actor state remains private to the backend implementation.
+type JobScheduler interface {
+ DynamicScheduler
+ JobController
+
+ RegisterJob(job Job) error
+ RegisterBusWatcher(watcher BusWatcher, canRunManually bool) error
+ StartScheduler() error
+ GetLocation() *time.Location
+ Stop(ctx context.Context) error
+}
+
// GenericJob is a reusable Job built from closures. It lets a service register a
// per-entity dynamic job (e.g. one per GitOps sync or one per environment) without
// importing the scheduler package: the service constructs a GenericJob and hands it