Skip to content

Modern Docker Management, Designed for Everyone

refactor: use francis for distrbuted actors (#4261)

Kyle Mendell09606d5987bf
diff --git a/Justfile b/Justfile
index ef14db841..6c589bc9c 100644
--- a/Justfile
+++ b/Justfile
@@ -66,9 +66,25 @@ _dev-agent:
[group('dev')]
_dev-all:
#!/usr/bin/env bash
- trap 'kill 0' EXIT
- (cd backend && air) &
- vp -C frontend run dev
+ set -euo pipefail
+
+ shutdown() {
+ status=$?
+ trap - EXIT
+ trap '' INT TERM
+ kill -TERM "$backend_pid" "$frontend_pid" 2>/dev/null || true
+ wait "$backend_pid" "$frontend_pid" 2>/dev/null || true
+ exit "$status"
+ }
+
+ (cd backend && exec air) &
+ backend_pid=$!
+ vp -C frontend run dev &
+ frontend_pid=$!
+ trap shutdown EXIT
+ trap 'exit 130' INT
+ trap 'exit 143' TERM
+ wait "$frontend_pid"
# Rebuild Docker dev environment
[group('dev')]
diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.go
index 8bd731a80..f7a2c6ddb 100644
--- a/backend/api/ws/container_stats.go
+++ b/backend/api/ws/container_stats.go
@@ -30,7 +30,7 @@ import (
func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {
containerID := c.Param("containerId")
if strings.TrimSpace(containerID) == "" {
- return wsErrorJSONInternal(c, http.StatusBadRequest, "Container ID is required")
+ return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})
}
conn, onRemove, ok := h.acceptWSInternal(c, systemtypes.WSKindContainerStats, containerID)
diff --git a/backend/api/ws/exec.go b/backend/api/ws/exec.go
index b1d476b8d..edb3adadd 100644
--- a/backend/api/ws/exec.go
+++ b/backend/api/ws/exec.go
@@ -33,7 +33,7 @@ import (
func (h *WebSocketHandler) ContainerExec(c *echo.Context) error {
containerID := c.Param("containerId")
if strings.TrimSpace(containerID) == "" {
- return wsErrorJSONInternal(c, http.StatusBadRequest, "Container ID is required")
+ return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})
}
shell := cmp.Or(c.QueryParam("shell"), "/bin/sh")
diff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.go
index 75bc1d1cf..fba2911ae 100644
--- a/backend/api/ws/handler.go
+++ b/backend/api/ws/handler.go
@@ -9,7 +9,6 @@ import (
"time"
"github.com/coder/websocket"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
@@ -20,7 +19,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
systemlib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/system"
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
- httputil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
"github.com/labstack/echo/v5"
"github.com/samber/hot"
@@ -29,10 +29,6 @@ import (
var defaultWebSocketMetrics = wshub.NewWebSocketMetrics()
-// ============================================================================
-// WebSocket Handler
-// ============================================================================
-
// WebSocketHandler consolidates all WebSocket and streaming endpoints.
// REST endpoints are handled by Huma handlers.
type WebSocketHandler struct {
@@ -46,29 +42,22 @@ type WebSocketHandler struct {
activeConnections sync.Map
logStreamsMu sync.Mutex
logStreams map[string]*wsLogStream
- cpuCache actors.Snapshot[float64]
+ cpuCache concurrency.Snapshot[float64]
systemStaticInfo struct {
once sync.Once
cpuCount int
hostname string
}
systemStatsSampler struct {
- latest actors.Snapshot[systemtypes.SystemStats]
- lifecycleMu sync.Mutex
- clients int
- // intervals counts subscribers per requested interval; the sampler
- // ticks at the smallest one (in effectiveInterval) rather than a
- // fixed 1Hz, so samples are never produced faster than the fastest
- // consumer reads them.
+ latest concurrency.Snapshot[systemtypes.SystemStats]
+ lifecycleMu sync.Mutex
+ clients int
intervals map[time.Duration]int
effectiveInterval atomic.Int64
cancel context.CancelFunc
ready chan struct{}
- // wake nudges the running sampler when effectiveInterval changes,
- // so a newly joined faster subscriber doesn't wait out a slower
- // tick already in flight.
- wake chan struct{}
- running bool
+ wake chan struct{}
+ running bool
}
containerStatsHubs sync.Map
cgroupCache *cgroup.Cache
@@ -104,7 +93,7 @@ func NewWebSocketHandler(
diskUsagePathCache: hot.NewHotCache[struct{}, string](hot.LRU, 1).
WithTTL(5 * time.Minute).
Build(),
- checkWSOrigin: httputil.ValidateWebSocketOrigin(cfg.GetAppURL()),
+ checkWSOrigin: httpx.ValidateWebSocketOrigin(cfg.GetAppURL()),
}
wsGroup := group.Group("/environments/:id/ws", authMiddleware.WithAdminNotRequired().Add())
for _, r := range handler.proxiedRoutes() {
@@ -138,12 +127,6 @@ func (h *WebSocketHandler) acceptWSInternal(c *echo.Context, kind, resourceID st
return conn, func() { h.wsMetrics.UnregisterConnection(connID) }, true
}
-// wsErrorJSONInternal replies with the {"success":false,"error":...} body
-// shared by every WS endpoint's pre-upgrade error path.
-func wsErrorJSONInternal(c *echo.Context, status int, msg string) error {
- return c.JSON(status, map[string]any{"success": false, "error": msg})
-}
-
// keepWSConnAliveInternal pings the peer every period. Ping round-trips (the
// pong must be serviced by the connection's concurrent reader) and is safe
// alongside a concurrent writer. A failed ping means the client is gone, so
diff --git a/backend/api/ws/log_streams.go b/backend/api/ws/log_streams.go
index e3a4a6067..57b295b44 100644
--- a/backend/api/ws/log_streams.go
+++ b/backend/api/ws/log_streams.go
@@ -211,7 +211,7 @@ func broadcastLogStreamErrorInternal(resourceLabel, errorPrefix, resourceID, for
func (h *WebSocketHandler) ProjectLogs(c *echo.Context) error {
projectID := c.Param("projectId")
if strings.TrimSpace(projectID) == "" {
- return wsErrorJSONInternal(c, http.StatusBadRequest, "Project ID is required")
+ return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Project ID is required"})
}
streamLogs := h.projectLogStreamer
@@ -416,7 +416,7 @@ func mapLogLinesInternal[T any](ctx context.Context, lines <-chan string, transf
func (h *WebSocketHandler) ContainerLogs(c *echo.Context) error {
containerID := c.Param("containerId")
if strings.TrimSpace(containerID) == "" {
- return wsErrorJSONInternal(c, http.StatusBadRequest, "Container ID is required")
+ return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Container ID is required"})
}
streamLogs := h.containerLogStreamer
@@ -461,7 +461,7 @@ func (h *WebSocketHandler) ContainerLogs(c *echo.Context) error {
func (h *WebSocketHandler) ServiceLogs(c *echo.Context) error {
serviceID := c.Param("serviceId")
if strings.TrimSpace(serviceID) == "" {
- return wsErrorJSONInternal(c, http.StatusBadRequest, "Service ID is required")
+ return c.JSON(http.StatusBadRequest, map[string]any{"success": false, "error": "Service ID is required"})
}
params := parseLogStreamParamsInternal(c)
diff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.go
index 65f1bcac2..69afaa0d9 100644
--- a/backend/api/ws/system_stats.go
+++ b/backend/api/ws/system_stats.go
@@ -426,7 +426,7 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
count, allowed := h.checkRateLimitInternal(clientIP)
if !allowed {
- return wsErrorJSONInternal(c, http.StatusTooManyRequests, "Too many concurrent stats connections from this IP")
+ return c.JSON(http.StatusTooManyRequests, map[string]any{"success": false, "error": "Too many concurrent stats connections from this IP"})
}
defer h.releaseRateLimitInternal(clientIP, count)
diff --git a/backend/cli/admin/reset_mfa.go b/backend/cli/admin/reset_mfa.go
index 7842fe6fa..e119bc59e 100644
--- a/backend/cli/admin/reset_mfa.go
+++ b/backend/cli/admin/reset_mfa.go
@@ -56,7 +56,7 @@ func runResetMFACommandInternal(cmd *cobra.Command, _ []string) error {
}
}()
- userService := user.NewUserService(db)
+ userService := user.NewUserService(db, nil)
user, err := userService.GetUserByUsername(cmd.Context(), username)
if err != nil {
if errors.Is(err, common.ErrUserNotFound) {
diff --git a/backend/cli/admin/reset_password.go b/backend/cli/admin/reset_password.go
index 82fd4d659..351e09625 100644
--- a/backend/cli/admin/reset_password.go
+++ b/backend/cli/admin/reset_password.go
@@ -157,7 +157,7 @@ func resetPasswordInternal(ctx context.Context, db *database.DB, username, passw
}
roleService := role.NewRoleService(db)
- userService := user.NewUserService(db).WithRoleService(roleService)
+ userService := user.NewUserService(db, roleService)
target, err := userService.GetUserByUsername(ctx, username)
if err != nil {
if errors.Is(err, common.ErrUserNotFound) {
diff --git a/backend/cli/admin/reset_password_test.go b/backend/cli/admin/reset_password_test.go
index c44ab2ed9..bcef74295 100644
--- a/backend/cli/admin/reset_password_test.go
+++ b/backend/cli/admin/reset_password_test.go
@@ -43,7 +43,7 @@ func createGlobalAdminInternal(t *testing.T, db *database.DB) (*common.User, *ro
ctx := context.Background()
roleService := role.NewRoleService(db)
require.NoError(t, roleService.EnsureBuiltInRoles(ctx))
- userService := user.NewUserService(db).WithRoleService(roleService)
+ userService := user.NewUserService(db, roleService)
passwordHash, err := userService.HashPassword("old-password")
require.NoError(t, err)
adminUser, err := userService.CreateUser(ctx, &common.User{
diff --git a/backend/cli/recovery/recovery.go b/backend/cli/recovery/recovery.go
index 33e6f32eb..8e370a39a 100644
--- a/backend/cli/recovery/recovery.go
+++ b/backend/cli/recovery/recovery.go
@@ -18,8 +18,10 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ dbtypes "github.com/getarcaneapp/arcane/types/v2/database"
recoverytypes "github.com/getarcaneapp/arcane/types/v2/recovery"
"github.com/libtnb/sqlite"
"github.com/moby/moby/api/types/container"
@@ -94,6 +96,17 @@ func runRestoreInternal(_ *cobra.Command, _ []string) error {
if rollbackErr := runStagesInternal(context.WithoutCancel(ctx), dockerClient, request, request.RollbackStages); rollbackErr != nil {
return errors.Combine(err, fmt.Errorf("restoring the pre-restore system backup failed; Arcane was left stopped: %w", rollbackErr))
}
+ rollbackManifest, readErr := os.ReadFile("/app/data/.arcane-recovery.json")
+ var rollback recoverytypes.Manifest
+ if readErr == nil {
+ readErr = json.Unmarshal(rollbackManifest, &rollback)
+ }
+ if readErr == nil {
+ readErr = francis.ClearRestoredHosts(context.WithoutCancel(ctx), rollback.Environment["DATABASE_URL"])
+ }
+ if readErr != nil {
+ return errors.Combine(err, fmt.Errorf("clear restored actor ownership; Arcane was left stopped: %w", readErr))
+ }
restart()
return fmt.Errorf("%w; the pre-restore system backup was restored", err)
}
@@ -119,7 +132,6 @@ func runRestoreInternal(_ *cobra.Command, _ []string) error {
// configuration must keep pointing there rather than at the backup-time path.
manifest.Environment["PROJECTS_DIRECTORY"] = cmp.Or(request.ProjectsSetting, manifest.Environment["PROJECTS_DIRECTORY"])
if err := finalizeRestoredBackupInternal(ctx, manifest.Environment["DATABASE_URL"], manifest.BackupID, manifest.ActivityID, request); err != nil {
- restart()
return fmt.Errorf("finalize restored system backup: %w", err)
}
if err := upgrade.UpgradeContainer(ctx, dockerClient, inspect.Container, request.ContainerImage, manifest.Environment); err != nil {
@@ -132,7 +144,7 @@ func finalizeRestoredBackupInternal(ctx context.Context, databaseURL, manifestBa
if !strings.HasPrefix(databaseURL, "file:") {
return errors.New("restored Arcane database is not SQLite")
}
- dsn, err := database.ParseSQLiteConnectionString(databaseURL)
+ dsn, err := database.ParseSQLiteConnectionString(databaseURL, dbtypes.SQLiteConnectionOptions{})
if err != nil {
return err
}
@@ -161,7 +173,10 @@ func finalizeRestoredBackupInternal(ctx context.Context, databaseURL, manifestBa
return err
}
}
- return finalizeRestoredActivityInternal(db, manifestActivityID)
+ if err := finalizeRestoredActivityInternal(db, manifestActivityID); err != nil {
+ return err
+ }
+ return francis.ClearRestoredHosts(ctx, databaseURL)
}
// finalizeRestoredRunInternal marks the restored run succeeded, filling only
diff --git a/backend/cli/recovery/recovery_test.go b/backend/cli/recovery/recovery_test.go
index 7a12f31d7..b1229c10a 100644
--- a/backend/cli/recovery/recovery_test.go
+++ b/backend/cli/recovery/recovery_test.go
@@ -11,6 +11,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ dbtypes "github.com/getarcaneapp/arcane/types/v2/database"
recoverytypes "github.com/getarcaneapp/arcane/types/v2/recovery"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
@@ -20,7 +21,7 @@ import (
func newRestoredDatabaseForTestInternal(t *testing.T, runIDs ...string) (string, string) {
t.Helper()
databaseURL := "file:" + filepath.Join(t.TempDir(), "arcane.db")
- dsn, err := database.ParseSQLiteConnectionString(databaseURL)
+ dsn, err := database.ParseSQLiteConnectionString(databaseURL, dbtypes.SQLiteConnectionOptions{})
require.NoError(t, err)
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
require.NoError(t, err)
diff --git a/backend/docs/francis-migration.md b/backend/docs/francis-migration.md
new file mode 100644
index 000000000..9052a0e5d
--- /dev/null
+++ b/backend/docs/francis-migration.md
@@ -0,0 +1,9 @@
+# Durable jobs migration
+
+Stop the old Arcane process before upgrading to the Francis-based release. Take a system backup before the upgrade. The first startup imports legacy persisted job records before accepting requests or executing jobs. Import resumes safely if startup is interrupted. Legacy records remain in the database but are never dispatched by the new runtime.
+
+Only one active Arcane process may use a database. Francis enforces this when the actor host registers. After an unclean shutdown, startup can wait for the previous registration to expire. Do not run the old and new releases against the same database at the same time.
+
+`ACTOR_PORT` defaults to `3551`. Francis uses QUIC over UDP bound to `127.0.0.1`. Do not publish this port from Docker. Processes sharing a network namespace must use different actor ports, even when they use different databases. No separate actor authentication secret is needed. Arcane derives it from the existing encryption key and instance ID.
+
+To roll back, stop Arcane and restore the pre-upgrade backup before starting the old release. Do not point the old release at the upgraded database. Restore accepts existing database/WAL/SHM archives and the new staged single-file database snapshot. The stopped restore helper removes copied live host ownership before restarting Arcane. Normal startup repairs schedules and pending dispatches from durable state.
diff --git a/backend/go.mod b/backend/go.mod
index d1ab49f81..5a262437d 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -1,6 +1,6 @@
module github.com/getarcaneapp/arcane/backend/v2
-go 1.27
+go 1.27.1
replace (
// Copacetic v0.14.2 only compiles against the buildx v0.32 imagetools API;
@@ -14,7 +14,6 @@ require (
emperror.dev/emperror v0.33.0
emperror.dev/errors v0.8.1
github.com/ProtonMail/go-crypto v1.5.2
- github.com/anthdm/hollywood v1.0.5
github.com/aws/aws-sdk-go-v2 v1.47.1
github.com/aws/aws-sdk-go-v2/config v1.33.6
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
@@ -40,7 +39,7 @@ require (
github.com/go-webauthn/webauthn v0.18.2
github.com/gofrs/flock v0.13.1
github.com/google/go-containerregistry v0.22.1
- github.com/google/uuid v1.6.0
+ github.com/italypaleale/francis v0.1.0
github.com/jwx-go/jwkfetch/v4 v4.0.4
github.com/klauspost/compress v1.20.1
github.com/labstack/echo/v5 v5.4.0
@@ -57,9 +56,9 @@ require (
github.com/orandin/slog-gorm v1.4.0
github.com/pressly/goose/v3 v3.28.0
github.com/project-copacetic/copacetic v0.15.0
+ github.com/quic-go/quic-go v0.62.0
github.com/robfig/cron/v3 v3.0.1
github.com/samber/hot v0.13.1
- github.com/samber/lo v1.53.0
github.com/samber/mo v1.17.0
github.com/samber/slog-echo/v2 v2.2.0
github.com/shirou/gopsutil/v4 v4.26.8
@@ -72,7 +71,7 @@ require (
go.getarcane.app/builds v0.4.2
go.getarcane.app/docker/compat v0.1.0
go.getarcane.app/docker/convert v0.3.2
- go.getarcane.app/kit v0.3.1
+ go.getarcane.app/kit v0.3.2
go.getarcane.app/streams v0.4.3
go.getarcane.app/sys/cgroup v0.2.3
go.getarcane.app/sys/crypto v0.2.2
@@ -99,13 +98,13 @@ require (
connectrpc.com/connect v1.20.0 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
- github.com/DataDog/gostackparse v0.7.0 // indirect
github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679 // indirect
github.com/DmitriyVTitov/size v1.5.0 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/semver/v3 v3.5.0 // indirect
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
+ github.com/alphadose/haxmap v1.4.1 // indirect
github.com/aquasecurity/go-pep440-version v0.0.2-0.20260224065243-f9bba28c51a4 // indirect
github.com/aquasecurity/go-version v0.0.1 // indirect
github.com/aquasecurity/trivy v0.72.0 // indirect
@@ -152,7 +151,8 @@ require (
github.com/docker/buildx v0.36.1 // indirect
github.com/docker/docker-credential-helpers v0.9.9 // indirect
github.com/docker/go-connections v0.8.1 // indirect
- github.com/dustin/go-humanize v1.0.1 // indirect
+ github.com/dunglas/httpsfv v1.1.1 // indirect
+ github.com/dustin/go-humanize v1.1.0 // indirect
github.com/ebitengine/purego v0.11.0 // indirect
github.com/eclipse/paho.golang v0.23.0 // indirect
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 // indirect
@@ -175,6 +175,7 @@ require (
github.com/google/go-tpm v0.9.8 // indirect
github.com/google/go-tpm-tools v0.4.8 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+ github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -185,9 +186,12 @@ require (
github.com/in-toto/in-toto-golang v0.11.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect
+ github.com/italypaleale/go-kit v1.2.4 // indirect
+ github.com/italypaleale/go-sql-utils v0.3.9 // indirect
+ github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
- github.com/jackc/pgx/v5 v5.10.0 // indirect
+ github.com/jackc/pgx/v5 v5.11.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jinzhu/copier v0.4.0 // indirect
@@ -248,9 +252,12 @@ require (
github.com/prometheus/client_model v0.6.3 // indirect
github.com/prometheus/common v0.71.0 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
+ github.com/quic-go/qpack v0.6.0 // indirect
+ github.com/quic-go/webtransport-go v0.13.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/samber/go-singleflightx v0.3.2 // indirect
+ github.com/samber/lo v1.53.0 // indirect
github.com/samber/oops v1.18.1 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect
github.com/secure-systems-lab/go-securesystemslib v0.11.1 // indirect
@@ -273,11 +280,12 @@ require (
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vito/midterm v0.1.4 // indirect
github.com/vito/progrock v0.10.1 // indirect
+ github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
+ github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
- github.com/zeebo/xxh3 v1.0.2 // indirect
github.com/zmb3/spotify/v2 v2.3.1 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
@@ -298,16 +306,17 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
- golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect
+ golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect
golang.org/x/tools v0.50.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
- google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20260908043556-f8649ddbbfe6 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260908043556-f8649ddbbfe6 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gotest.tools/v3 v3.5.2 // indirect
k8s.io/apimachinery v0.36.3 // indirect
+ k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 // indirect
mellium.im/reader v0.1.0 // indirect
mellium.im/sasl v0.3.2 // indirect
mellium.im/xmlstream v0.15.4 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index ef5013bd4..3a17f05de 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -46,8 +46,6 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=
-github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679 h1:qNT7R4qrN+5u5ajSbqSW1opHP4LA8lzA+ASyw5MQZjs=
github.com/DefangLabs/secret-detector v0.0.0-20250811234530-d4b4214cd679/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=
@@ -69,12 +67,12 @@ github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7l
github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
+github.com/alphadose/haxmap v1.4.1 h1:VtD6VCxUkjNIfJk/aWdYFfOzrRddDFjmvmRmILg7x8Q=
+github.com/alphadose/haxmap v1.4.1/go.mod h1:rjHw1IAqbxm0S3U5tD16GoKsiAd8FWx5BJ2IYqXwgmM=
github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30=
github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
-github.com/anthdm/hollywood v1.0.5 h1:SuCTVRRFqx0MZ4E0RijHl4+Xt56PF0C7aqYKguBU6j8=
-github.com/anthdm/hollywood v1.0.5/go.mod h1:wU4WxIRVs++E2PuiVXc8dA2An/Wlom4AhzwQ7e3tDzI=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/aquasecurity/go-pep440-version v0.0.2-0.20260224065243-f9bba28c51a4 h1:QC8nsF3LvAGHUxkr5bvcekaUQPE9Zk4jyMugyg5Fxy0=
@@ -241,8 +239,10 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/dunglas/httpsfv v1.1.1 h1:HoSs101zIE9I23DlqlmljJ/OIi7ILwrH347pXhRZdxI=
+github.com/dunglas/httpsfv v1.1.1/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg=
+github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
+github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
@@ -407,12 +407,20 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
+github.com/italypaleale/francis v0.1.0 h1:5/Z0mPZ3xrq4VeJR29oSPKPTqgr9Aygwg/uAKIdObDM=
+github.com/italypaleale/francis v0.1.0/go.mod h1:skoh94KSgrvNuKLNPk4VcTN10GiLVadG9TLTrQuJ1JM=
+github.com/italypaleale/go-kit v1.2.4 h1:gPp2HHov0qlemFPhZMHOW11dU3RGwJW51Q0l11aO/a8=
+github.com/italypaleale/go-kit v1.2.4/go.mod h1:+Jwwwm1BwqLnerW3SnAGxfj8aN1Kgrok5uUllabDE6E=
+github.com/italypaleale/go-sql-utils v0.3.9 h1:1P7VOC6rjv68PTlEjp83ziVnq8W/YjYFxGvIdWi3s4E=
+github.com/italypaleale/go-sql-utils v0.3.9/go.mod h1:NSIj8h4doAfDq/y31S4ITPDSUP9k8vnIxpAmzsxmT+g=
+github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ=
+github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
-github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
+github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jarcoal/httpmock v1.4.2 h1:dKwiP/9zITCPfBLsDn3kchbSOu16JrnxtVEmL0fPRcI=
@@ -602,6 +610,14 @@ github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
+github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
+github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.62.0 h1:ZHDjCk5OacATwGvs8PWE97CTvX7AqZiVoW7++ZOXTf8=
+github.com/quic-go/quic-go v0.62.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
+github.com/quic-go/webtransport-go v0.13.0 h1:RJLrTUHlTj8jJaQlQJUy0z0Mf7u1fVM0I6L1b9pe2M0=
+github.com/quic-go/webtransport-go v0.13.0/go.mod h1:K83X9YHbAqgSLO6ikS6BXCMdWOvqh9JTHALulvb2JVk=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -709,6 +725,10 @@ github.com/vito/midterm v0.1.4 h1:SALq5mQ+AgzeZxjL4loB6Uk5TZc9JMyX2ELA/NVH65c=
github.com/vito/midterm v0.1.4/go.mod h1:Mm3u6lrpzo2EFSJbwksKOdottTJzYePK8c1KJy4aRbk=
github.com/vito/progrock v0.10.1 h1:4M/nwUhJKO2Hmo0D89dfor0XZKx5k4mqLkEHWLINSYo=
github.com/vito/progrock v0.10.1/go.mod h1:ysw2W2gc20Snmlc0a34JbWO45HPM0oXO8IC59hyFc3k=
+github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
+github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/wneessen/go-mail v0.8.1 h1:tVcncj02/QySVFw3zr/kXOzZcuFQqBNT6K+Rbgm/pcM=
github.com/wneessen/go-mail v0.8.1/go.mod h1:dWZ61zadzCIyvB4y1/YzC5O7MrbbzBfPkARmbosdf8w=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -726,10 +746,6 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zclconf/go-cty v1.18.1 h1:yEGE8M4iIZlyKQURZNb2SnEyZlZHUcBCnx6KF81KuwM=
github.com/zclconf/go-cty v1.18.1/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
-github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
-github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
-github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zmb3/spotify/v2 v2.3.1 h1:aEyIPotROM3JJjHMCImFROgnPIUpzVo8wymYSaPSd9w=
github.com/zmb3/spotify/v2 v2.3.1/go.mod h1:+LVh9CafHu7SedyqYmEf12Rd01dIVlEL845yNhksW0E=
go.getarcane.app/acfs v0.6.1 h1:xPDfof8Tbr3ZQ6ATcyFM4EtyUh+K657CThpGdg+dz50=
@@ -740,8 +756,8 @@ go.getarcane.app/docker/compat v0.1.0 h1:GIycV+Pn/kEIXlqhmjfSGuDgw5OhECYudRdHZgm
go.getarcane.app/docker/compat v0.1.0/go.mod h1:lSGOgKO2ikdy8WzFGHLtkT05L7Amak8HCtnudV6Deow=
go.getarcane.app/docker/convert v0.3.2 h1:4y2zEpkdv6CfWqzdz3whxWSpF9cKbhihcNBwWK4cXVM=
go.getarcane.app/docker/convert v0.3.2/go.mod h1:iZ3vyiTnj7/K70aDSY/G689YnvqOYtZle9IVPFlnM24=
-go.getarcane.app/kit v0.3.1 h1:fuSl8ZMshuFPUvJl/9QwoI/ds1+zsfpixZYqk/hFLUE=
-go.getarcane.app/kit v0.3.1/go.mod h1:KTjbXTEeC3i9e1QxZowDJWjFyABlD6UF3ISVtoIZs10=
+go.getarcane.app/kit v0.3.2 h1:L7CAvGawDg70K/vVxDeM2xZBCHqwuthc9B5OwQwL0Hc=
+go.getarcane.app/kit v0.3.2/go.mod h1:KTjbXTEeC3i9e1QxZowDJWjFyABlD6UF3ISVtoIZs10=
go.getarcane.app/streams v0.4.3 h1:Rzg6xCWzYmD4IN+GiKYmAc8HqCgVIq/sQ4DC3OxiZ1o=
go.getarcane.app/streams v0.4.3/go.mod h1:SC/7A3pR8RanF5DqdX99A40qEw13TbmmbEtJK3yPuy4=
go.getarcane.app/sys/cgroup v0.2.3 h1:+eGLUC+3Ct+BNcPBQwC/g/psDOzjWN2vUr6KhKyNWgI=
@@ -826,8 +842,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
-golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
+golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
+golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1055,10 +1071,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4 h1:NCe/UiklGd/9xjT+ROBVhJ1kf6TRQaFedsR+z7u1gvo=
-google.golang.org/genproto/googleapis/api v0.0.0-20260904194346-d0f1323225a4/go.mod h1:fJ2lYaWjqNknJyQBOCd0fA3HnEElJqGplH71a2txi+g=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4 h1:5t+ZydAFj5kGVLrgCvLmpmCf9ylGRd64hpEronfRaws=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260904194346-d0f1323225a4/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
+google.golang.org/genproto/googleapis/api v0.0.0-20260908043556-f8649ddbbfe6 h1:O4Tjo2vlGeGM3+tNgwJrGj5fQHO2V71gDkFqbm328C8=
+google.golang.org/genproto/googleapis/api v0.0.0-20260908043556-f8649ddbbfe6/go.mod h1:r4KD2hOq82JBWpTWkJ9NZLf6EwmRvAPIFmk7hPNtd+0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260908043556-f8649ddbbfe6 h1:ieEbjQ6lzbvntOXUB9nMx9uH+yIU/HbgkNDjnk/mJuk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260908043556-f8649ddbbfe6/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1120,8 +1136,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM=
k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE=
-k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM=
-k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
+k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE=
+k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM=
mellium.im/reader v0.1.0 h1:UUEMev16gdvaxxZC7fC08j7IzuDKh310nB6BlwnxTww=
mellium.im/reader v0.1.0/go.mod h1:F+X5HXpkIfJ9EE1zHQG9lM/hO946iYAmU7xjg5dsQHI=
mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
diff --git a/backend/internal/activity/backup.go b/backend/internal/activity/backup.go
index 7b0faf959..508fe085a 100644
--- a/backend/internal/activity/backup.go
+++ b/backend/internal/activity/backup.go
@@ -8,9 +8,9 @@ import (
)
// FailInterruptedBackups finalizes backup activities before startup admits work.
-func (s *ActivityService) FailInterruptedBackups(ctx context.Context) error {
+func (s *ActivityService) FailInterruptedBackups(ctx context.Context, protectedIDs ...string) error {
var pending []Activity
- if err := s.db.WithContext(ctx).Where("status IN ?", []activitytypes.Status{activitytypes.StatusQueued, activitytypes.StatusRunning}).Find(&pending).Error; err != nil {
+ if err := s.db.WithContext(ctx).Where("status IN ?", []activitytypes.Status{activitytypes.StatusQueued, activitytypes.StatusRunning}).Where("id NOT IN ?", append(protectedIDs, "")).Find(&pending).Error; err != nil {
return errors.WrapIf(err, "find interrupted backup activities")
}
const message = "Backup interrupted by Arcane restart"
diff --git a/backend/internal/activity/handler.go b/backend/internal/activity/handler.go
index 847a58029..bbaa65e6a 100644
--- a/backend/internal/activity/handler.go
+++ b/backend/internal/activity/handler.go
@@ -31,9 +31,6 @@ import (
type ActivityHandler struct {
activityService *ActivityService
environment EnvironmentDependencies
-
- // remoteStreamHub shares one poller per remote environment across every
- // connected stream client instead of polling per client × environment.
remoteStreamHub *agg.Hub[activitytypes.StreamEvent]
}
diff --git a/backend/internal/activity/module.go b/backend/internal/activity/module.go
index 5a06e4d75..53bb322a4 100644
--- a/backend/internal/activity/module.go
+++ b/backend/internal/activity/module.go
@@ -4,24 +4,15 @@ package activity
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-type Dependencies struct {
- DB *database.DB
- Settings *settings.SettingsService
- Environment EnvironmentDependencies
-}
-
type Module struct {
service *ActivityService
handler *ActivityHandler
}
-func New(deps Dependencies) *Module {
- service := NewActivityService(deps.DB, deps.Settings)
- return &Module{service: service, handler: NewHandler(service, deps.Environment)}
+func New(service *ActivityService, environment EnvironmentDependencies) *Module {
+ return &Module{service: service, handler: NewHandler(service, environment)}
}
func (m *Module) Service() *ActivityService {
diff --git a/backend/internal/activity/remote.go b/backend/internal/activity/remote.go
index 280b82eb6..520bca046 100644
--- a/backend/internal/activity/remote.go
+++ b/backend/internal/activity/remote.go
@@ -7,7 +7,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
- "github.com/samber/lo"
kit "go.getarcane.app/kit/pkg"
)
@@ -109,19 +108,19 @@ func compareActivitiesInternal(a, b activitytypes.Activity, params pagination.Qu
case "status":
result = cmp.Compare(a.Status, b.Status)
case "resourceType":
- result = strings.Compare(lo.FromPtr(a.ResourceType), lo.FromPtr(b.ResourceType))
+ result = strings.Compare(kit.FromPtr(a.ResourceType), kit.FromPtr(b.ResourceType))
case "resourceName":
- result = strings.Compare(lo.FromPtr(a.ResourceName), lo.FromPtr(b.ResourceName))
+ result = strings.Compare(kit.FromPtr(a.ResourceName), kit.FromPtr(b.ResourceName))
case "startedAt":
result = a.StartedAt.Compare(b.StartedAt)
case "createdAt":
result = a.CreatedAt.Compare(b.CreatedAt)
case "updatedAt":
- result = lo.FromPtr(a.UpdatedAt).Compare(lo.FromPtr(b.UpdatedAt))
+ result = kit.FromPtr(a.UpdatedAt).Compare(kit.FromPtr(b.UpdatedAt))
case "endedAt":
- result = lo.FromPtr(a.EndedAt).Compare(lo.FromPtr(b.EndedAt))
+ result = kit.FromPtr(a.EndedAt).Compare(kit.FromPtr(b.EndedAt))
case "durationMs":
- result = cmp.Compare(lo.FromPtr(a.DurationMs), lo.FromPtr(b.DurationMs))
+ result = cmp.Compare(kit.FromPtr(a.DurationMs), kit.FromPtr(b.DurationMs))
}
if params.Order == pagination.SortDesc {
result = -result
diff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.go
index 85946dece..65b9b63a0 100644
--- a/backend/internal/activity/service.go
+++ b/backend/internal/activity/service.go
@@ -981,7 +981,7 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e
// ResolveOrphanedQueuedActivities fails any activity still queued at startup.
// Queued state is owned by a live goroutine blocked on AwaitActivitySlot, so a
// queued row after a restart can never start running.
-func (s *ActivityService) ResolveOrphanedQueuedActivities(ctx context.Context) (int64, error) {
+func (s *ActivityService) ResolveOrphanedQueuedActivities(ctx context.Context, protectedIDs ...string) (int64, error) {
if s == nil || s.db == nil {
return 0, nil
}
@@ -990,6 +990,7 @@ func (s *ActivityService) ResolveOrphanedQueuedActivities(ctx context.Context) (
if err := s.db.WithContext(ctx).
Where("status = ?", activitytypes.StatusQueued).
Where("type <> ?", activitytypes.TypeJobRun).
+ Where("id NOT IN ?", append(protectedIDs, "")).
Find(&queued).Error; err != nil {
return 0, errors.WrapIf(err, "find orphaned queued activities")
}
@@ -1045,7 +1046,7 @@ func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID
// running by a prior process lifetime. A run whose metadata marks a triggered
// self-update completed by restarting Arcane, so it is recorded as success;
// anything else still running at startup was interrupted and is failed.
-func (s *ActivityService) ResolveStaleAutoUpdateActivities(ctx context.Context) (int64, error) {
+func (s *ActivityService) ResolveStaleAutoUpdateActivities(ctx context.Context, protectedIDs ...string) (int64, error) {
if s == nil || s.db == nil {
return 0, nil
}
@@ -1053,6 +1054,7 @@ func (s *ActivityService) ResolveStaleAutoUpdateActivities(ctx context.Context)
var stale []Activity
if err := s.db.WithContext(ctx).
Where("type = ? AND status = ?", activitytypes.TypeAutoUpdate, activitytypes.StatusRunning).
+ Where("id NOT IN ?", append(protectedIDs, "")).
Find(&stale).Error; err != nil {
return 0, errors.WrapIf(err, "find stale auto-update activities")
}
diff --git a/backend/internal/activity/service_test.go b/backend/internal/activity/service_test.go
index 0874c446d..3dfddf1d9 100644
--- a/backend/internal/activity/service_test.go
+++ b/backend/internal/activity/service_test.go
@@ -5,7 +5,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
@@ -15,7 +14,6 @@ import (
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -392,21 +390,11 @@ func setupQueuedActivityServiceInternal(t *testing.T) (*ActivityService, context
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "activity-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "activity-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFreesInternal(t *testing.T) {
diff --git a/backend/internal/actors/actor.go b/backend/internal/actors/actor.go
deleted file mode 100644
index e84e03bf5..000000000
--- a/backend/internal/actors/actor.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package actors
-
-import (
- "context"
- "sync"
-
- "emperror.dev/errors"
- "github.com/anthdm/hollywood/actor"
-)
-
-var errActorRequestNoResponseInternal = errors.Sentinel("actor request completed without response")
-
-// Context exposes the operations available while handling one actor message
-// without leaking Hollywood into actor consumers.
-type Context struct {
- context context.Context
- engine *actor.Engine
- target *actor.PID
- respond func(any)
- workers *sync.WaitGroup
-}
-
-// Context returns the current receiver lifetime.
-func (c *Context) Context() context.Context {
- return c.context
-}
-
-// Respond sends exactly one response to the current request.
-func (c *Context) Respond(message any) {
- c.respond(message)
-}
-
-// Behavior defines the application behavior hosted by one shared actor.
-type Behavior struct {
- Initialize func(*Context)
- Handle func(*Context, any)
- Cleanup func(*Context)
-}
-
-// Binding connects an ingress to an actor for the duration of its lifetime.
-type Binding interface {
- bindInternal(handle *handleInternal)
- unbindInternal(handle *handleInternal)
- replayInternal()
-}
-
-// Actor owns one Hollywood process, its ingress bindings, and all workers
-// started from its behavior.
-type Actor struct {
- handle *handleInternal
- bindings []Binding
- unbindOnce sync.Once
-}
-
-type behaviorActorInternal struct {
- behavior Behavior
- bindings []Binding
- lifetime lifetimeInternal
- workers sync.WaitGroup
-}
-
-// NewActor starts one application behavior on the shared actor runtime.
-func NewActor(ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, producer func() Behavior, bindings ...Binding) (*Actor, error) {
- if ctx == nil {
- return nil, errors.New("actor context unavailable")
- }
- if runtime == nil {
- return nil, errors.New("actor runtime unavailable")
- }
- if producer == nil {
- return nil, errors.New("actor behavior unavailable")
- }
-
- handle, err := runtime.startInternal(
- func() actor.Receiver {
- return &behaviorActorInternal{behavior: producer(), bindings: bindings}
- },
- kind,
- id,
- maxRestarts,
- actor.WithContext(ctx),
- )
- if err != nil {
- return nil, err
- }
-
- applicationActor := &Actor{handle: handle, bindings: bindings}
- for _, binding := range bindings {
- binding.bindInternal(handle)
- }
- return applicationActor, nil
-}
-
-// Send queues a message for the actor.
-func (a *Actor) Send(message any) error {
- return a.handle.Send(message)
-}
-
-// Done closes after the actor and its workers stop.
-func (a *Actor) Done() <-chan struct{} {
- return a.handle.Done()
-}
-
-// Stop disconnects ingress, drains the mailbox, cancels workers, and joins
-// them before the actor terminates.
-func (a *Actor) Stop(ctx context.Context) error {
- a.unbindOnce.Do(func() {
- for _, binding := range a.bindings {
- binding.unbindInternal(a.handle)
- }
- })
- return a.handle.Stop(ctx)
-}
-
-// Request sends one typed request and consumes its response.
-func (a *Actor) Request[K comparable, Q, R any](ctx context.Context, request Message[K, Q]) (Message[K, R], error) {
- if a == nil {
- var zero Message[K, R]
- return zero, errors.New("actor unavailable")
- }
- return a.handle.request[K, Q, R](ctx, request)
-}
-
-func (a *behaviorActorInternal) Receive(ctx *actor.Context) {
- switch message := ctx.Message().(type) {
- case actor.Initialized:
- return
- case actor.Started:
- lifetimeCtx := a.lifetime.startInternal(ctx.Context())
- for _, binding := range a.bindings {
- binding.replayInternal()
- }
- if a.behavior.Initialize != nil {
- a.behavior.Initialize(a.contextInternal(lifetimeCtx, ctx))
- }
- case actor.Stopped:
- a.lifetime.stopInternal()
- if a.behavior.Cleanup != nil {
- a.behavior.Cleanup(a.contextInternal(ctx.Context(), ctx))
- }
- // Strict joining preserves actor serialization across receiver restarts:
- // a replacement must never run while an old worker can still own state.
- // Behavior workers are therefore required to honor their lifetime context.
- a.workers.Wait()
- default:
- if request, ok := message.(requestInternal); ok {
- defer request.reply.Resolve(requestResultInternal{err: errActorRequestNoResponseInternal})
- if a.behavior.Handle != nil {
- requestCtx := a.contextInternal(a.lifetime.contextInternal(), ctx)
- requestCtx.respond = func(message any) {
- request.reply.Resolve(requestResultInternal{message: message})
- }
- a.behavior.Handle(requestCtx, request.message)
- }
- return
- }
- if a.behavior.Handle != nil {
- a.behavior.Handle(a.contextInternal(a.lifetime.contextInternal(), ctx), message)
- }
- }
-}
-
-func (a *behaviorActorInternal) contextInternal(lifetimeCtx context.Context, ctx *actor.Context) *Context {
- return &Context{
- context: lifetimeCtx,
- engine: ctx.Engine(),
- target: ctx.PID().CloneVT(),
- respond: ctx.Respond,
- workers: &a.workers,
- }
-}
diff --git a/backend/internal/actors/actor_test.go b/backend/internal/actors/actor_test.go
deleted file mode 100644
index 0738b5bd5..000000000
--- a/backend/internal/actors/actor_test.go
+++ /dev/null
@@ -1,71 +0,0 @@
-package actors
-
-import (
- "context"
- "sync/atomic"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-type actorBindingFakeInternal struct {
- binds atomic.Int32
- unbinds atomic.Int32
-}
-
-func (b *actorBindingFakeInternal) bindInternal(*handleInternal) {
- b.binds.Add(1)
-}
-
-func (b *actorBindingFakeInternal) unbindInternal(*handleInternal) {
- b.unbinds.Add(1)
-}
-
-func (b *actorBindingFakeInternal) replayInternal() {}
-
-func TestActorOwnsLifecycleBindingsAndWorkerJoinInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- binding := &actorBindingFakeInternal{}
- initialized := make(chan struct{})
- workerExited := make(chan struct{})
- applicationActor, err := NewActor(
- t.Context(),
- runtime,
- "actor-test",
- "lifecycle",
- 1,
- func() Behavior {
- return Behavior{
- Initialize: func(ctx *Context) {
- close(initialized)
- Worker[uint8, NoPayload]{
- Actor: ctx,
- WorkContext: ctx.Context(),
- Label: "actor lifecycle test worker",
- CompletionKind: 1,
- }.Start(func(workerCtx context.Context) (NoPayload, error) {
- defer close(workerExited)
- <-workerCtx.Done()
- return NoPayload{}, workerCtx.Err()
- })
- },
- }
- },
- binding,
- )
- require.NoError(t, err)
- require.Equal(t, int32(1), binding.binds.Load())
- <-initialized
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, applicationActor.Stop(stopCtx))
- <-workerExited
- require.Equal(t, int32(1), binding.unbinds.Load())
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/executor.go b/backend/internal/actors/executor.go
deleted file mode 100644
index 718e7775e..000000000
--- a/backend/internal/actors/executor.go
+++ /dev/null
@@ -1,179 +0,0 @@
-package actors
-
-import (
- "context"
- "sync"
-
- "emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
-)
-
-// Executor serializes heterogeneous blocking tasks through one actor mailbox.
-// A task must not synchronously submit another task to the same Executor.
-type Executor struct {
- actor *Actor
-}
-
-// Task is one submitted executor operation whose result may be awaited with a
-// context independent from the operation's own lifetime.
-type Task[T any] struct {
- result *Promise[executorResultInternal[T]]
- actor *Actor
-}
-
-type executorTaskInternal interface {
- executeInternal(actorCtx context.Context)
-}
-
-type executorResultInternal[T any] struct {
- value T
- err error
-}
-
-type executorTaskValueInternal[T any] struct {
- ctx context.Context
- label string
- work func(context.Context) (T, error)
- after func(T, error)
- result *Promise[executorResultInternal[T]]
- started *sync.Once
-}
-
-// NewExecutor creates a serial task executor on the shared actor runtime.
-func NewExecutor(ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32) (*Executor, error) {
- if runtime == nil {
- return nil, errors.New("actor runtime unavailable")
- }
- if ctx == nil {
- return nil, errors.New("executor context unavailable")
- }
- applicationActor, err := NewActor(
- ctx,
- runtime,
- kind,
- id,
- maxRestarts,
- func() Behavior {
- return Behavior{Handle: receiveExecutorInternal}
- },
- )
- if err != nil {
- return nil, err
- }
- return &Executor{actor: applicationActor}, nil
-}
-
-// Execute queues work, waits for its typed result, and runs after only once the
-// result is available to the caller. Work and after are both panic-contained.
-func (e *Executor) Execute[T any](ctx context.Context, label string, work func(context.Context) (T, error), after func(T, error)) (T, error) {
- var zero T
- task, err := e.Submit(ctx, label, work, after)
- if err != nil {
- return zero, err
- }
- return task.Wait(ctx)
-}
-
-// Submit queues work synchronously and returns a handle that can be awaited
-// separately. This is useful for terminal cleanup that must stay queued even
-// when the shutdown wait context expires.
-func (e *Executor) Submit[T any](ctx context.Context, label string, work func(context.Context) (T, error), after func(T, error)) (*Task[T], error) {
- if e == nil || e.actor == nil {
- return nil, errors.New("actor executor unavailable")
- }
- if ctx == nil {
- return nil, errors.New("task context unavailable")
- }
- if err := ctx.Err(); err != nil {
- return nil, err
- }
- if work == nil {
- return nil, errors.New("actor task unavailable")
- }
-
- result := NewPromise[executorResultInternal[T]]()
- if err := e.actor.Send(executorTaskValueInternal[T]{
- ctx: ctx,
- label: label,
- work: work,
- after: after,
- result: result,
- started: &sync.Once{},
- }); err != nil {
- return nil, err
- }
- return &Task[T]{result: result, actor: e.actor}, nil
-}
-
-// Wait waits for the submitted task result without changing the task lifetime.
-func (t *Task[T]) Wait(ctx context.Context) (T, error) {
- var zero T
- if t == nil || t.result == nil || t.actor == nil {
- return zero, errors.New("actor task unavailable")
- }
- if ctx == nil {
- return zero, errors.New("task wait context unavailable")
- }
- completed, err := t.actor.handle.await(ctx, t.result, errors.New("actor executor stopped"))
- if err != nil {
- return zero, err
- }
- return completed.value, completed.err
-}
-
-// Stop drains the executor mailbox and waits for the actor to terminate.
-func (e *Executor) Stop(ctx context.Context) error {
- if e == nil || e.actor == nil {
- return nil
- }
- return e.actor.Stop(ctx)
-}
-
-// Done closes after the executor actor terminates.
-func (e *Executor) Done() <-chan struct{} {
- if e == nil || e.actor == nil {
- return nil
- }
- return e.actor.Done()
-}
-
-func receiveExecutorInternal(ctx *Context, rawMessage any) {
- task, ok := rawMessage.(executorTaskInternal)
- if !ok {
- return
- }
- task.executeInternal(ctx.Context())
-}
-
-func (t executorTaskValueInternal[T]) executeInternal(actorCtx context.Context) {
- if t.result == nil || t.started == nil {
- return
- }
- t.started.Do(func() {
- workCtx, cancel := context.WithCancel(actorCtx)
- stopTaskCancel := context.AfterFunc(t.ctx, cancel) //nolint:contextcheck // task execution combines caller cancellation with the actor lifetime.
- defer cancel()
- defer stopTaskCancel()
-
- var (
- value T
- err error
- )
- func() {
- defer utils.RecoverToError(&err, t.label)
- if err = t.ctx.Err(); err == nil {
- value, err = t.work(workCtx)
- }
- }()
- t.result.Resolve(executorResultInternal[T]{value: value, err: err})
-
- if t.after == nil {
- return
- }
- func() {
- var afterErr error
- defer utils.RecoverToError(&afterErr, t.label+" completion")
- t.after(value, err)
- }()
- })
-}
diff --git a/backend/internal/actors/executor_test.go b/backend/internal/actors/executor_test.go
deleted file mode 100644
index a34c72167..000000000
--- a/backend/internal/actors/executor_test.go
+++ /dev/null
@@ -1,109 +0,0 @@
-package actors
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-func TestExecutorSerializesHeterogeneousTasksInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := NewExecutor(t.Context(), runtime, "executor-test", "serial", 1)
- require.NoError(t, err)
-
- firstStarted := make(chan struct{})
- releaseFirst := make(chan struct{})
- firstResult := make(chan error, 1)
- go func() {
- _, executeErr := executor.Execute(t.Context(), "first", func(context.Context) (int, error) {
- close(firstStarted)
- <-releaseFirst
- return 1, nil
- }, nil)
- firstResult <- executeErr
- }()
- <-firstStarted
-
- secondStarted := make(chan struct{})
- secondResult := make(chan string, 1)
- go func() {
- value, executeErr := executor.Execute(t.Context(), "second", func(context.Context) (string, error) {
- close(secondStarted)
- return "two", nil
- }, nil)
- if executeErr != nil {
- secondResult <- executeErr.Error()
- return
- }
- secondResult <- value
- }()
-
- require.Never(t, func() bool { return len(secondStarted) > 0 }, 20*time.Millisecond, time.Millisecond)
- close(releaseFirst)
- require.NoError(t, <-firstResult)
- require.Equal(t, "two", <-secondResult)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestExecutorPublishesResultBeforeCompletionCallbackInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := NewExecutor(t.Context(), runtime, "executor-test", "completion-order", 1)
- require.NoError(t, err)
-
- afterStarted := make(chan struct{})
- releaseAfter := make(chan struct{})
- result := make(chan int, 1)
- go func() {
- value, _ := executor.Execute(t.Context(), "ordered", func(context.Context) (int, error) {
- return 42, nil
- }, func(int, error) {
- close(afterStarted)
- <-releaseAfter
- })
- result <- value
- }()
-
- <-afterStarted
- require.Equal(t, 42, <-result)
- close(releaseAfter)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestExecutorContainsTaskPanicAndContinuesInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := NewExecutor(t.Context(), runtime, "executor-test", "panic", 1)
- require.NoError(t, err)
-
- _, err = executor.Execute(t.Context(), "panic task", func(context.Context) (NoPayload, error) {
- panic("deliberate executor panic")
- }, nil)
- require.ErrorContains(t, err, "panic task panicked")
-
- value, err := executor.Execute(t.Context(), "following task", func(context.Context) (int, error) {
- return 7, nil
- }, nil)
- require.NoError(t, err)
- require.Equal(t, 7, value)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/gate.go b/backend/internal/actors/gate.go
deleted file mode 100644
index a67c59f4f..000000000
--- a/backend/internal/actors/gate.go
+++ /dev/null
@@ -1,161 +0,0 @@
-package actors
-
-import (
- "context"
- "sync"
-
- "emperror.dev/errors"
-)
-
-type gateOperationInternal uint8
-
-const (
- gateAcquireInternal gateOperationInternal = iota
- gateReleaseInternal
-)
-
-// AdmissionKey namespaces a string identifier shared by the application gate.
-type AdmissionKey struct {
- Scope string
- ID string
-}
-
-// Gate admits at most one active lease for each key through one actor-owned map.
-type Gate[K comparable] struct {
- actor *Actor
-}
-
-// Lease represents one admitted key. Release is idempotent and confirms that
-// the actor processed the completion before returning.
-type Lease[K comparable] struct {
- gate *Gate[K]
- key K
- once sync.Once
-}
-
-type gateCommandInternal[K comparable] struct {
- key K
- reply *Promise[bool]
-}
-
-type gateActorInternal[K comparable] struct {
- active map[K]struct{}
-}
-
-// NewGate creates a keyed admission gate on the shared actor runtime.
-func NewGate[K comparable](ctx context.Context, runtime *Runtime, kind, id string) (*Gate[K], error) {
- if ctx == nil {
- return nil, errors.New("gate context unavailable")
- }
- if runtime == nil {
- return nil, errors.New("actor runtime unavailable")
- }
-
- active := make(map[K]struct{})
- applicationActor, err := NewActor(
- ctx,
- runtime,
- kind,
- id,
- // Shared gates tolerate transient receiver failures without forgetting
- // active leases, whose map intentionally survives receiver replacement.
- 3,
- func() Behavior {
- // The map intentionally survives receiver replacement so an actor restart
- // cannot forget leases held by work that is still running.
- state := &gateActorInternal[K]{active: active}
- return Behavior{Handle: state.receiveInternal}
- },
- )
- if err != nil {
- return nil, err
- }
- return &Gate[K]{actor: applicationActor}, nil
-}
-
-// TryAcquire returns a lease when key is idle and refuses immediately when it
-// is already active. Actor failure is reported separately from contention.
-func (g *Gate[K]) TryAcquire(ctx context.Context, key K) (*Lease[K], bool, error) {
- if ctx == nil {
- return nil, false, errors.New("gate request context unavailable")
- }
- if err := ctx.Err(); err != nil {
- return nil, false, err
- }
- if g == nil || g.actor == nil {
- return nil, false, errors.New("actor gate unavailable")
- }
-
- reply := NewPromise[bool]()
- if err := g.actor.Send(Message[gateOperationInternal, gateCommandInternal[K]]{
- Kind: gateAcquireInternal,
- Value: gateCommandInternal[K]{
- key: key,
- reply: reply,
- },
- }); err != nil {
- return nil, false, err
- }
-
- // Admission must resolve after it is queued; abandoning an admitted reply on
- // caller cancellation would leak the key without a lease to release it.
- admitted, err := g.actor.handle.await(context.WithoutCancel(ctx), reply, errors.New("actor gate stopped"))
- if err != nil {
- return nil, false, err
- }
- if !admitted {
- return nil, false, nil
- }
- return &Lease[K]{gate: g, key: key}, true, nil
-}
-
-// Release returns the lease to its gate exactly once.
-func (l *Lease[K]) Release() {
- if l == nil || l.gate == nil {
- return
- }
- l.once.Do(func() {
- reply := NewPromise[bool]()
- if err := l.gate.actor.Send(Message[gateOperationInternal, gateCommandInternal[K]]{
- Kind: gateReleaseInternal,
- Value: gateCommandInternal[K]{
- key: l.key,
- reply: reply,
- },
- }); err != nil {
- return
- }
- _, _ = l.gate.actor.handle.await(context.Background(), reply, errors.New("actor gate stopped"))
- })
-}
-
-// Stop drains the gate mailbox and waits for the actor to terminate.
-func (g *Gate[K]) Stop(ctx context.Context) error {
- if g == nil || g.actor == nil {
- return nil
- }
- return g.actor.Stop(ctx)
-}
-
-func (g *gateActorInternal[K]) receiveInternal(_ *Context, rawMessage any) {
- message, ok := rawMessage.(Message[gateOperationInternal, gateCommandInternal[K]])
- if !ok || message.Value.reply == nil {
- return
- }
- // Resolve a refusal if future gate logic panics before producing a result.
- // Promise resolution is idempotent, so successful paths win first.
- defer message.Value.reply.Resolve(false)
-
- switch message.Kind {
- case gateAcquireInternal:
- if _, active := g.active[message.Value.key]; active {
- message.Value.reply.Resolve(false)
- return
- }
- g.active[message.Value.key] = struct{}{}
- message.Value.reply.Resolve(true)
- case gateReleaseInternal:
- delete(g.active, message.Value.key)
- message.Value.reply.Resolve(true)
- }
-}
diff --git a/backend/internal/actors/gate_test.go b/backend/internal/actors/gate_test.go
deleted file mode 100644
index b15f214e8..000000000
--- a/backend/internal/actors/gate_test.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package actors
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-func TestGateSerializesEachKeyAndReleasesIndependentlyInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- gate, err := NewGate[string](t.Context(), runtime, "gate-test", "keyed")
- require.NoError(t, err)
-
- first, admitted, err := gate.TryAcquire(t.Context(), "first")
- require.NoError(t, err)
- require.True(t, admitted)
-
- _, admitted, err = gate.TryAcquire(t.Context(), "first")
- require.NoError(t, err)
- require.False(t, admitted)
-
- second, admitted, err := gate.TryAcquire(t.Context(), "second")
- require.NoError(t, err)
- require.True(t, admitted)
- second.Release()
-
- first.Release()
- first.Release()
- replacement, admitted, err := gate.TryAcquire(t.Context(), "first")
- require.NoError(t, err)
- require.True(t, admitted)
- replacement.Release()
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestGateReturnsContextCancellationWithoutLeakingKeyInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- gate, err := NewGate[string](t.Context(), runtime, "gate-test", "cancellation")
- require.NoError(t, err)
-
- canceled, cancel := context.WithCancel(context.Background())
- cancel()
- _, admitted, err := gate.TryAcquire(canceled, "key")
- require.ErrorIs(t, err, context.Canceled)
- require.False(t, admitted)
-
- lease, admitted, err := gate.TryAcquire(t.Context(), "key")
- require.NoError(t, err)
- require.True(t, admitted)
- lease.Release()
-
- stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
- defer stopCancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/message.go b/backend/internal/actors/message.go
deleted file mode 100644
index eca454db0..000000000
--- a/backend/internal/actors/message.go
+++ /dev/null
@@ -1,375 +0,0 @@
-package actors
-
-import (
- "context"
- "fmt"
- "sync"
- "sync/atomic"
- "time"
-
- "emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
-)
-
-// NoPayload is used by typed actor messages that only carry a kind.
-type NoPayload struct{}
-
-// Message is the shared typed envelope for actor commands, events, and replies.
-// Worker completion handlers must call Acknowledge before returning.
-type Message[K comparable, V any] struct {
- Kind K
- Value V
- Err error
- ack chan struct{}
-}
-
-// Acknowledge confirms that an actor processed a worker completion.
-func (m Message[K, V]) Acknowledge() {
- if m.ack == nil {
- return
- }
- select {
- case m.ack <- struct{}{}:
- default:
- }
-}
-
-// Promise is an idempotent one-result completion primitive.
-type Promise[T any] struct {
- once sync.Once
- done chan T
-}
-
-// NewPromise creates a promise that accepts exactly one result.
-func NewPromise[T any]() *Promise[T] {
- return &Promise[T]{done: make(chan T, 1)}
-}
-
-// Resolve publishes value once; later calls are harmless.
-func (p *Promise[T]) Resolve(value T) {
- if p == nil {
- return
- }
- p.once.Do(func() { p.done <- value })
-}
-
-// Done returns the channel that receives the promised result.
-func (p *Promise[T]) Done() <-chan T {
- if p == nil {
- return nil
- }
- return p.done
-}
-
-// await waits for a promise while preserving a result that races with caller
-// cancellation or actor termination.
-func (h *handleInternal) await[T any](ctx context.Context, promise *Promise[T], stoppedErr error) (T, error) {
- var zero T
- if ctx == nil || h == nil || promise == nil {
- return zero, errors.New("actor wait unavailable")
- }
- select {
- case result := <-promise.Done():
- return result, nil
- case <-ctx.Done():
- select {
- case result := <-promise.Done():
- return result, nil
- default:
- return zero, ctx.Err()
- }
- case <-h.Done():
- select {
- case result := <-promise.Done():
- return result, nil
- default:
- return zero, stoppedErr
- }
- }
-}
-
-// lifetimeInternal is an actor-receiver lifecycle context.
-type lifetimeInternal struct {
- mu sync.Mutex
- ctx context.Context
- cancel context.CancelFunc
-}
-
-func (l *lifetimeInternal) startInternal(parent context.Context) context.Context {
- l.mu.Lock()
- defer l.mu.Unlock()
- if l.cancel != nil {
- l.cancel()
- }
- l.ctx, l.cancel = context.WithCancel(parent)
- return l.ctx
-}
-
-func (l *lifetimeInternal) contextInternal() context.Context {
- l.mu.Lock()
- defer l.mu.Unlock()
- return l.ctx
-}
-
-func (l *lifetimeInternal) stopInternal() {
- l.mu.Lock()
- defer l.mu.Unlock()
- if l.cancel != nil {
- l.cancel()
- l.cancel = nil
- }
-}
-
-// Ingress coalesces concurrent producers into one pending mailbox signal while
-// retaining the newest value. Bind may happen after Send, which lets services
-// accept startup triggers before their actor exists.
-type Ingress[K comparable, V any] struct {
- mu sync.Mutex
- kind K
- value V
- generation uint64
- acknowledged uint64
- queued bool
- handle *handleInternal
-}
-
-// NewIngress creates an unbound coalescing actor ingress.
-func NewIngress[K comparable, V any](kind K) *Ingress[K, V] {
- return &Ingress[K, V]{kind: kind}
-}
-
-// Bind delivers future signals to handle and flushes one signal queued before startup.
-func (i *Ingress[K, V]) bindInternal(handle *handleInternal) {
- if i == nil {
- return
- }
- i.mu.Lock()
- defer i.mu.Unlock()
- i.handle = handle
- if i.queued && handle != nil {
- _ = handle.Send(Message[K, NoPayload]{Kind: i.kind})
- }
-}
-
-// Unbind stops delivery to handle without discarding a pending signal.
-func (i *Ingress[K, V]) unbindInternal(handle *handleInternal) {
- if i == nil {
- return
- }
- i.mu.Lock()
- defer i.mu.Unlock()
- if i.handle == handle {
- i.handle = nil
- }
-}
-
-func (i *Ingress[K, V]) replayInternal() {
- if i == nil {
- return
- }
- i.mu.Lock()
- defer i.mu.Unlock()
- if i.generation == i.acknowledged || i.queued || i.handle == nil {
- return
- }
- i.queued = true
- _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})
-}
-
-// Send records value and queues at most one mailbox signal.
-func (i *Ingress[K, V]) Send(value V) {
- if i == nil {
- return
- }
- i.mu.Lock()
- defer i.mu.Unlock()
- i.value = value
- i.generation++
- if i.queued {
- return
- }
- i.queued = true
- if i.handle != nil {
- _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})
- }
-}
-
-// Begin returns the newest value and its generation while retaining the work
-// as pending until Acknowledge commits it.
-func (i *Ingress[K, V]) Begin() (V, uint64) {
- i.mu.Lock()
- defer i.mu.Unlock()
- i.queued = false
- return i.value, i.generation
-}
-
-// Acknowledge commits work through generation and replays a newer pending value.
-func (i *Ingress[K, V]) Acknowledge(generation uint64) {
- i.mu.Lock()
- defer i.mu.Unlock()
- if generation > i.acknowledged {
- i.acknowledged = generation
- }
- if i.generation == i.acknowledged || i.queued || i.handle == nil {
- return
- }
- i.queued = true
- _ = i.handle.Send(Message[K, NoPayload]{Kind: i.kind})
-}
-
-// Take returns and immediately acknowledges the newest value.
-func (i *Ingress[K, V]) Take() V {
- value, generation := i.Begin()
- i.Acknowledge(generation)
- return value
-}
-
-// Pending reports whether an unacknowledged value remains.
-func (i *Ingress[K, V]) Pending() bool {
- i.mu.Lock()
- defer i.mu.Unlock()
- return i.generation != i.acknowledged
-}
-
-// Latest returns the newest value without changing admission state.
-func (i *Ingress[K, V]) Latest() V {
- i.mu.Lock()
- defer i.mu.Unlock()
- return i.value
-}
-
-// Timer sends generation-fenced messages to an actor. It is actor-owned and
-// must be stopped when the receiver handles actor.Stopped.
-type Timer[K comparable] struct {
- timer *time.Timer
- generation uint64
-}
-
-var timerGenerationInternal atomic.Uint64
-
-// Reset replaces the active timer and returns its generation.
-func (t *Timer[K]) Reset(ctx *Context, kind K, delay time.Duration) uint64 {
- if t.timer != nil {
- t.timer.Stop()
- }
- generation := timerGenerationInternal.Add(1)
- t.generation = generation
- engine := ctx.engine
- target := ctx.target
- actorCtx := ctx.Context()
- t.timer = time.AfterFunc(delay, func() {
- if actorCtx.Err() == nil {
- engine.Send(target, Message[K, uint64]{Kind: kind, Value: generation})
- }
- })
- return generation
-}
-
-// Current reports whether generation belongs to the latest Reset call.
-func (t *Timer[K]) Current(generation uint64) bool {
- return generation == t.generation
-}
-
-// Stop cancels the timer if one is active.
-func (t *Timer[K]) Stop() {
- t.generation = timerGenerationInternal.Add(1)
- if t.timer != nil {
- t.timer.Stop()
- t.timer = nil
- }
-}
-
-// requestInternal sends a typed request through Arcane's promise envelope.
-type requestInternal struct {
- message any
- reply *Promise[requestResultInternal]
-}
-
-type requestResultInternal struct {
- message any
- err error
-}
-
-func (h *handleInternal) request[K comparable, Q, R any](ctx context.Context, request Message[K, Q]) (Message[K, R], error) {
- var zero Message[K, R]
- if ctx == nil || h == nil {
- return zero, errors.New("actor request target unavailable")
- }
- reply := NewPromise[requestResultInternal]()
- if err := h.Send(requestInternal{message: request, reply: reply}); err != nil {
- return zero, err
- }
- result, err := h.await(ctx, reply, errors.New("actor stopped"))
- if err != nil {
- return zero, err
- }
- if result.err != nil {
- return zero, result.err
- }
- typed, ok := result.message.(Message[K, R])
- if !ok {
- return zero, fmt.Errorf("unexpected actor response type %T", result.message)
- }
- return typed, nil
-}
-
-// Worker describes blocking work owned by an actor but executed outside Receive.
-type Worker[K comparable, V any] struct {
- Actor *Context
- WorkContext context.Context
- Label string
- CompletionKind K
- PanicValue V
- RetryDelay time.Duration
- ActorStopped func(V, error)
-}
-
-// Start contains panics, joins the work to WaitGroup, and posts one typed
-// completion back to the actor. Its completion handler must call Acknowledge.
-func (w Worker[K, V]) Start(work func(context.Context) (V, error)) {
- actorCtx := w.Actor.Context()
- engine := w.Actor.engine
- target := w.Actor.target
- workerCtx, cancel := context.WithCancel(w.WorkContext)
- stopActorCancel := context.AfterFunc(actorCtx, cancel)
- w.Actor.workers.Go(func() {
- var (
- value = w.PanicValue
- err error
- )
- defer cancel()
- defer stopActorCancel()
- defer func() {
- if actorCtx.Err() == nil {
- ack := make(chan struct{}, 1)
- engine.Send(target, Message[K, V]{Kind: w.CompletionKind, Value: value, Err: err, ack: ack})
- select {
- case <-ack:
- return
- case <-actorCtx.Done():
- }
- }
- if w.ActorStopped != nil {
- w.ActorStopped(value, err)
- }
- }()
- for {
- value = w.PanicValue
- err = nil
- func() {
- defer utils.RecoverToError(&err, w.Label)
- value, err = work(workerCtx)
- }()
- if err == nil || w.RetryDelay <= 0 || workerCtx.Err() != nil {
- return
- }
- timer := time.NewTimer(w.RetryDelay)
- select {
- case <-workerCtx.Done():
- timer.Stop()
- return
- case <-timer.C:
- }
- }
- })
-}
diff --git a/backend/internal/actors/message_test.go b/backend/internal/actors/message_test.go
deleted file mode 100644
index 3b667bdb0..000000000
--- a/backend/internal/actors/message_test.go
+++ /dev/null
@@ -1,415 +0,0 @@
-package actors
-
-import (
- "context"
- "sync"
- "sync/atomic"
- "testing"
- "time"
-
- "github.com/anthdm/hollywood/actor"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-type messageKindInternal uint8
-
-const (
- ingressMessageInternal messageKindInternal = iota
- timerStartMessageInternal
- timerElapsedMessageInternal
- requestMessageInternal
- responseMessageInternal
- workerCompletedMessageInternal
-)
-
-type ingressReceiverInternal struct {
- ingress *Ingress[messageKindInternal, int]
- values chan<- int
-}
-
-func (r *ingressReceiverInternal) Receive(ctx *actor.Context) {
- if message, ok := ctx.Message().(Message[messageKindInternal, NoPayload]); ok && message.Kind == ingressMessageInternal {
- r.values <- r.ingress.Take()
- }
-}
-
-func TestIngressCoalescesBeforeBindingAndRetainsLatestValueInternal(t *testing.T) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- require.NoError(t, err)
- ingress := NewIngress[messageKindInternal, int](ingressMessageInternal)
- values := make(chan int, 2)
- pid := engine.Spawn(
- func() actor.Receiver { return &ingressReceiverInternal{ingress: ingress, values: values} },
- "ingress-test",
- actor.WithID("coalesced"),
- actor.WithMaxRestarts(1),
- )
-
- ingress.Send(1)
- ingress.Send(2)
- handle := &handleInternal{engine: engine, pid: pid, done: make(chan struct{})}
- ingress.bindInternal(handle)
- require.Equal(t, 2, <-values)
-
- ingress.Send(3)
- require.Equal(t, 3, <-values)
- ingress.unbindInternal(handle)
- <-engine.Poison(pid).Done()
-}
-
-type timerReceiverInternal struct {
- timer Timer[messageKindInternal]
- elapsed chan<- uint64
-}
-
-func (r *timerReceiverInternal) Receive(ctx *actor.Context) {
- switch message := ctx.Message().(type) {
- case Message[messageKindInternal, NoPayload]:
- if message.Kind == timerStartMessageInternal {
- actorCtx := &Context{
- context: ctx.Context(),
- engine: ctx.Engine(),
- target: ctx.PID().CloneVT(),
- respond: ctx.Respond,
- workers: &sync.WaitGroup{},
- }
- r.timer.Reset(actorCtx, timerElapsedMessageInternal, time.Hour)
- r.timer.Reset(actorCtx, timerElapsedMessageInternal, time.Millisecond)
- }
- case Message[messageKindInternal, uint64]:
- if message.Kind == timerElapsedMessageInternal && r.timer.Current(message.Value) {
- r.elapsed <- message.Value
- }
- case actor.Stopped:
- r.timer.Stop()
- }
-}
-
-func TestTimerOnlyDeliversLatestGenerationInternal(t *testing.T) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- require.NoError(t, err)
- elapsed := make(chan uint64, 2)
- pid := engine.Spawn(
- func() actor.Receiver { return &timerReceiverInternal{elapsed: elapsed} },
- "timer-test",
- actor.WithID("latest"),
- actor.WithMaxRestarts(1),
- )
- engine.Send(pid, Message[messageKindInternal, NoPayload]{Kind: timerStartMessageInternal})
- require.Equal(t, uint64(2), <-elapsed)
- require.Never(t, func() bool { return len(elapsed) > 0 }, 20*time.Millisecond, time.Millisecond)
- <-engine.Poison(pid).Done()
-}
-
-func TestRequestReturnsTypedResponseInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- applicationActor, err := NewActor(
- t.Context(),
- runtime,
- "request-test",
- "typed",
- 1,
- func() Behavior {
- return Behavior{Handle: func(ctx *Context, rawMessage any) {
- if message, ok := rawMessage.(Message[messageKindInternal, int]); ok && message.Kind == requestMessageInternal {
- ctx.Respond(Message[messageKindInternal, string]{Kind: responseMessageInternal, Value: "ok"})
- }
- }}
- },
- )
- require.NoError(t, err)
- for range 10 {
- response, requestErr := applicationActor.Request[messageKindInternal, int, string](t.Context(), Message[messageKindInternal, int]{Kind: requestMessageInternal, Value: 1})
- require.NoError(t, requestErr)
- require.Equal(t, responseMessageInternal, response.Kind)
- require.Equal(t, "ok", response.Value)
- }
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, applicationActor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestRequestFailsWhenHandlerReturnsWithoutResponseInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- applicationActor, err := NewActor(
- t.Context(),
- runtime,
- "request-test",
- "missing-response",
- 1,
- func() Behavior {
- return Behavior{Handle: func(*Context, any) {}}
- },
- )
- require.NoError(t, err)
-
- _, err = applicationActor.Request[messageKindInternal, int, string](context.Background(), Message[messageKindInternal, int]{Kind: requestMessageInternal})
- require.ErrorContains(t, err, "without response")
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, applicationActor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-type workerReceiverInternal struct {
- errors chan<- error
- values chan<- string
-}
-
-func (r workerReceiverInternal) Receive(ctx *actor.Context) {
- if message, ok := ctx.Message().(Message[messageKindInternal, string]); ok && message.Kind == workerCompletedMessageInternal {
- defer message.Acknowledge()
- if r.errors != nil {
- r.errors <- message.Err
- }
- if r.values != nil {
- r.values <- message.Value
- }
- }
-}
-
-func TestWorkerContainsPanicAndDeliversCompletionInternal(t *testing.T) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- require.NoError(t, err)
- workerErrors := make(chan error, 1)
- workerValues := make(chan string, 1)
- pid := engine.Spawn(
- func() actor.Receiver { return workerReceiverInternal{errors: workerErrors, values: workerValues} },
- "worker-test",
- actor.WithID("panic"),
- actor.WithMaxRestarts(1),
- )
-
- var wg sync.WaitGroup
- Worker[messageKindInternal, string]{
- Actor: &Context{
- context: context.Background(),
- engine: engine,
- target: pid,
- workers: &wg,
- },
- WorkContext: context.Background(),
- Label: "deliberate worker test",
- CompletionKind: workerCompletedMessageInternal,
- PanicValue: "panic fallback",
- }.Start(func(context.Context) (string, error) {
- panic("deliberate worker panic")
- })
-
- require.ErrorContains(t, <-workerErrors, "deliberate worker test panicked")
- require.Equal(t, "panic fallback", <-workerValues)
- wg.Wait()
- <-engine.Poison(pid).Done()
-}
-
-func TestWorkerRetriesContainedPanicInternal(t *testing.T) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- require.NoError(t, err)
- workerErrors := make(chan error, 1)
- workerValues := make(chan string, 1)
- pid := engine.Spawn(
- func() actor.Receiver { return workerReceiverInternal{errors: workerErrors, values: workerValues} },
- "worker-test",
- actor.WithID("retry"),
- actor.WithMaxRestarts(1),
- )
-
- var (
- wg sync.WaitGroup
- attempts int
- )
- Worker[messageKindInternal, string]{
- Actor: &Context{
- context: context.Background(),
- engine: engine,
- target: pid,
- workers: &wg,
- },
- WorkContext: context.Background(),
- Label: "retrying worker test",
- CompletionKind: workerCompletedMessageInternal,
- PanicValue: "panic fallback",
- RetryDelay: time.Millisecond,
- }.Start(func(context.Context) (string, error) {
- attempts++
- if attempts == 1 {
- panic("deliberate retryable panic")
- }
- return "completed", nil
- })
-
- require.NoError(t, <-workerErrors)
- require.Equal(t, "completed", <-workerValues)
- require.Equal(t, 2, attempts)
- wg.Wait()
- <-engine.Poison(pid).Done()
-}
-
-type lifetimeReceiverInternal struct {
- lifetime lifetimeInternal
- started chan<- context.Context
-}
-
-func (r *lifetimeReceiverInternal) Receive(ctx *actor.Context) {
- switch ctx.Message().(type) {
- case actor.Started:
- r.started <- r.lifetime.startInternal(ctx.Context())
- case panicActorMessageInternal:
- panic("deliberate lifetime restart")
- case actor.Stopped:
- r.lifetime.stopInternal()
- }
-}
-
-func TestLifetimeCancelsOldReceiverBeforeHollywoodRestartInternal(t *testing.T) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- require.NoError(t, err)
- started := make(chan context.Context, 2)
- pid := engine.Spawn(
- func() actor.Receiver { return &lifetimeReceiverInternal{started: started} },
- "lifetime-test",
- actor.WithID("restart"),
- actor.WithMaxRestarts(1),
- actor.WithRestartDelay(0),
- )
-
- first := <-started
- engine.Send(pid, panicActorMessageInternal{})
- require.Eventually(t, func() bool { return first.Err() != nil }, time.Second, time.Millisecond)
- second := <-started
- require.NoError(t, second.Err())
- <-engine.Poison(pid).Done()
-}
-
-func TestWorkerResolvesFallbackWhenActorStopsBeforeCompletionInternal(t *testing.T) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- require.NoError(t, err)
- pid := engine.Spawn(
- func() actor.Receiver { return workerReceiverInternal{errors: make(chan error, 1)} },
- "worker-test",
- actor.WithID("stopped"),
- actor.WithMaxRestarts(1),
- )
-
- actorCtx, stopActor := context.WithCancel(context.Background())
- result := NewPromise[error]()
- var wg sync.WaitGroup
- Worker[messageKindInternal, string]{
- Actor: &Context{
- context: actorCtx,
- engine: engine,
- target: pid,
- workers: &wg,
- },
- WorkContext: context.Background(),
- Label: "stopped worker test",
- CompletionKind: workerCompletedMessageInternal,
- ActorStopped: func(_ string, err error) {
- result.Resolve(err)
- },
- }.Start(func(ctx context.Context) (string, error) {
- <-ctx.Done()
- return "", ctx.Err()
- })
-
- stopActor()
- require.ErrorIs(t, <-result.Done(), context.Canceled)
- wg.Wait()
- <-engine.Poison(pid).Done()
-}
-
-type noAcknowledgementReceiverInternal struct {
- received chan<- struct{}
-}
-
-func (r noAcknowledgementReceiverInternal) Receive(ctx *actor.Context) {
- if message, ok := ctx.Message().(Message[messageKindInternal, string]); ok && message.Kind == workerCompletedMessageInternal {
- r.received <- struct{}{}
- }
-}
-
-func TestWorkerWithoutAcknowledgementUnblocksWhenActorLifetimeEndsInternal(t *testing.T) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- require.NoError(t, err)
- received := make(chan struct{}, 1)
- pid := engine.Spawn(
- func() actor.Receiver { return noAcknowledgementReceiverInternal{received: received} },
- "worker-test",
- actor.WithID("missing-ack"),
- )
- actorCtx, stopActor := context.WithCancel(context.Background())
- fallback := make(chan error, 1)
- var workers sync.WaitGroup
- Worker[messageKindInternal, string]{
- Actor: &Context{
- context: actorCtx,
- engine: engine,
- target: pid,
- workers: &workers,
- },
- WorkContext: actorCtx,
- Label: "missing acknowledgement worker",
- CompletionKind: workerCompletedMessageInternal,
- ActorStopped: func(_ string, err error) {
- fallback <- err
- },
- }.Start(func(context.Context) (string, error) {
- return "done", nil
- })
-
- <-received
- stopActor()
- require.NoError(t, <-fallback)
- require.NoError(t, utils.WaitGroup(t.Context(), &workers))
- <-engine.Poison(pid).Done()
-}
-
-func TestIngressReplaysUnacknowledgedValueAfterActorRestartInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- ingress := NewIngress[messageKindInternal, int](ingressMessageInternal)
- processed := make(chan int, 1)
- var attempts atomic.Int32
- applicationActor, err := NewActor(
- t.Context(),
- runtime,
- "ingress-test",
- "restart-replay",
- 2,
- func() Behavior {
- return Behavior{Handle: func(_ *Context, rawMessage any) {
- message, ok := rawMessage.(Message[messageKindInternal, NoPayload])
- if !ok || message.Kind != ingressMessageInternal {
- return
- }
- value, generation := ingress.Begin()
- if attempts.Add(1) == 1 {
- panic("restart after ingress begin")
- }
- ingress.Acknowledge(generation)
- processed <- value
- }}
- },
- ingress,
- )
- require.NoError(t, err)
-
- ingress.Send(42)
- require.Equal(t, 42, <-processed)
- require.False(t, ingress.Pending())
- require.Equal(t, int32(2), attempts.Load())
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, applicationActor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/resource.go b/backend/internal/actors/resource.go
deleted file mode 100644
index 62d39280f..000000000
--- a/backend/internal/actors/resource.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package actors
-
-import (
- "context"
- "log/slog"
- "sync/atomic"
-
- "emperror.dev/errors"
-)
-
-// ErrResourceStopped reports work submitted after terminal resource shutdown.
-var ErrResourceStopped = errors.New("actor resource stopped")
-
-// Resource serializes the lifecycle and use of one replaceable value through
-// an actor executor. Its value is never accessed outside that mailbox, and its
-// zero value represents the absence of a resource.
-type Resource[T comparable] struct {
- executor *Executor
- value T
- terminal atomic.Bool
- stop func(T) error
-}
-
-// NewResource creates an actor-owned replaceable resource.
-func NewResource[T comparable](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, stop func(T) error) (*Resource[T], error) {
- if stop == nil {
- return nil, errors.New("resource stop function unavailable")
- }
- executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)
- if err != nil {
- return nil, err
- }
- return &Resource[T]{executor: executor, stop: stop}, nil
-}
-
-// Restart stops the current value before building and publishing its replacement.
-// A non-zero failed build is cleaned up.
-func (r *Resource[T]) Restart(ctx context.Context, label string, build func(context.Context) (T, error)) error {
- if build == nil {
- return errors.New("resource build function unavailable")
- }
- if r.terminal.Load() {
- return ErrResourceStopped
- }
- _, err := r.executor.Execute(ctx, label, func(workCtx context.Context) (NoPayload, error) {
- if r.terminal.Load() {
- return NoPayload{}, ErrResourceStopped
- }
- stopErr := r.clearInternal()
- next, buildErr := build(workCtx)
- if buildErr != nil {
- var zero T
- if next != zero {
- buildErr = errors.Combine(buildErr, r.stop(next))
- }
- return NoPayload{}, errors.Combine(stopErr, buildErr)
- }
- r.value = next
- if stopErr != nil {
- slog.WarnContext(workCtx, "failed to stop replaced actor resource", "label", label, "error", stopErr)
- }
- return NoPayload{}, nil
- }, nil)
- return err
-}
-
-// Do serializes work with Restart and Stop. Work is skipped after terminal stop.
-func (r *Resource[T]) Do(ctx context.Context, label string, work func(context.Context, T) error) error {
- if work == nil {
- return errors.New("resource work function unavailable")
- }
- if r.terminal.Load() {
- return ErrResourceStopped
- }
- _, err := r.executor.Execute(ctx, label, func(workCtx context.Context) (NoPayload, error) {
- if r.terminal.Load() {
- return NoPayload{}, ErrResourceStopped
- }
- return NoPayload{}, work(workCtx, r.value)
- }, nil)
- return err
-}
-
-// Clear stops and forgets the current value while keeping the resource reusable.
-func (r *Resource[T]) Clear(ctx context.Context, label string) error {
- if r.terminal.Load() {
- return ErrResourceStopped
- }
- _, err := r.executor.Execute(ctx, label, func(context.Context) (NoPayload, error) {
- if r.terminal.Load() {
- return NoPayload{}, ErrResourceStopped
- }
- return NoPayload{}, r.clearInternal()
- }, nil)
- return err
-}
-
-// Stop permanently fences replacement, stops the current value, and joins the actor.
-func (r *Resource[T]) Stop(ctx context.Context) error {
- if ctx == nil {
- return errors.New("resource stop context unavailable")
- }
- r.terminal.Store(true)
- task, submitErr := r.executor.Submit(context.WithoutCancel(ctx), "stop actor resource", func(context.Context) (NoPayload, error) {
- return NoPayload{}, r.clearInternal()
- }, nil)
- var clearErr error
- if submitErr == nil {
- _, clearErr = task.Wait(ctx)
- }
- actorErr := r.executor.Stop(ctx)
- if submitErr != nil {
- // Submit only fails after the executor can no longer own the value, so
- // terminal cleanup must stop it directly instead of leaking it.
- clearErr = r.clearInternal()
- }
- return errors.Combine(submitErr, clearErr, actorErr)
-}
-
-func (r *Resource[T]) clearInternal() error {
- var zero T
- if r.value == zero {
- return nil
- }
- current := r.value
- r.value = zero
- return r.stop(current)
-}
diff --git a/backend/internal/actors/resource_test.go b/backend/internal/actors/resource_test.go
deleted file mode 100644
index c86f84db5..000000000
--- a/backend/internal/actors/resource_test.go
+++ /dev/null
@@ -1,99 +0,0 @@
-package actors
-
-import (
- "context"
- "errors"
- "sync"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-func TestResourceSerializesRestartWorkAndStopInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- var (
- mu sync.Mutex
- stopped []int
- )
- resource, err := NewResource(t.Context(), runtime, "resource-test", "lifecycle", 1, func(value int) error {
- mu.Lock()
- stopped = append(stopped, value)
- mu.Unlock()
- return nil
- })
- require.NoError(t, err)
- require.NoError(t, resource.Restart(t.Context(), "start first", func(context.Context) (int, error) { return 1, nil }))
- require.NoError(t, resource.Restart(t.Context(), "start second", func(context.Context) (int, error) { return 2, nil }))
-
- var current int
- require.NoError(t, resource.Do(t.Context(), "read current", func(_ context.Context, value int) error {
- current = value
- return nil
- }))
- require.Equal(t, 2, current)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, resource.Stop(stopCtx))
- mu.Lock()
- require.Equal(t, []int{1, 2}, stopped)
- mu.Unlock()
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestResourceFailedReplacementCleansPartialValueInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- var stopped []int
- resource, err := NewResource(t.Context(), runtime, "resource-test", "failed-replacement", 1, func(value int) error {
- stopped = append(stopped, value)
- return nil
- })
- require.NoError(t, err)
- require.NoError(t, resource.Restart(t.Context(), "start", func(context.Context) (int, error) { return 1, nil }))
- require.Error(t, resource.Restart(t.Context(), "replace", func(context.Context) (int, error) {
- return 2, errors.New("build failed")
- }))
- require.Equal(t, []int{1, 2}, stopped)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, resource.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestResourceCanceledStopStillCleansAndFencesInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- stopped := make(chan struct{})
- resource, err := NewResource(t.Context(), runtime, "resource-test", "canceled-stop", 1, func(int) error {
- close(stopped)
- return nil
- })
- require.NoError(t, err)
- require.NoError(t, resource.Restart(t.Context(), "start", func(context.Context) (int, error) { return 1, nil }))
-
- canceled, cancel := context.WithCancel(context.Background())
- cancel()
- require.ErrorIs(t, resource.Stop(canceled), context.Canceled)
- select {
- case <-stopped:
- case <-time.After(time.Second):
- require.FailNow(t, "resource cleanup was abandoned after stop wait cancellation")
- }
- require.ErrorIs(t, resource.Restart(t.Context(), "restart", func(context.Context) (int, error) { return 2, nil }), ErrResourceStopped)
-
- stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
- defer stopCancel()
- require.NoError(t, resource.executor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/runner.go b/backend/internal/actors/runner.go
deleted file mode 100644
index c454145ce..000000000
--- a/backend/internal/actors/runner.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package actors
-
-import (
- "context"
-
- "emperror.dev/errors"
-)
-
-// Runner owns one long-lived background function and joins it before stopping.
-type Runner struct {
- actor *Actor
-}
-
-type runnerStateInternal struct {
- run func(context.Context) error
- label string
-}
-
-type runnerEventInternal uint8
-
-const runnerExitedInternal runnerEventInternal = iota
-
-// NewRunner starts a long-lived function on the shared actor runtime.
-func NewRunner(ctx context.Context, runtime *Runtime, kind, id, label string, maxRestarts int32, run func(context.Context) error) (*Runner, error) {
- if ctx == nil {
- return nil, errors.New("runner context unavailable")
- }
- if runtime == nil {
- return nil, errors.New("actor runtime unavailable")
- }
- if run == nil {
- return nil, errors.New("runner function unavailable")
- }
-
- applicationActor, err := NewActor(
- ctx,
- runtime,
- kind,
- id,
- maxRestarts,
- func() Behavior {
- state := &runnerStateInternal{run: run, label: label}
- return Behavior{
- Initialize: state.initializeInternal,
- Handle: state.receiveInternal,
- }
- },
- )
- if err != nil {
- return nil, err
- }
- return &Runner{actor: applicationActor}, nil
-}
-
-// Stop cancels the background function and waits for it and its actor to exit.
-func (r *Runner) Stop(ctx context.Context) error {
- if r == nil || r.actor == nil {
- return nil
- }
- return r.actor.Stop(ctx)
-}
-
-func (r *runnerStateInternal) initializeInternal(ctx *Context) {
- Worker[runnerEventInternal, NoPayload]{
- Actor: ctx,
- WorkContext: ctx.Context(),
- Label: r.label,
- CompletionKind: runnerExitedInternal,
- }.Start(func(workerCtx context.Context) (NoPayload, error) {
- return NoPayload{}, r.run(workerCtx)
- })
-}
-
-func (r *runnerStateInternal) receiveInternal(_ *Context, rawMessage any) {
- message, ok := rawMessage.(Message[runnerEventInternal, NoPayload])
- if !ok {
- return
- }
- defer message.Acknowledge()
- if message.Err != nil {
- panic(message.Err)
- }
- panic(errors.New(r.label + " exited unexpectedly"))
-}
diff --git a/backend/internal/actors/runner_test.go b/backend/internal/actors/runner_test.go
deleted file mode 100644
index 28624b154..000000000
--- a/backend/internal/actors/runner_test.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package actors
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-func TestRunnerCancelsAndJoinsBackgroundFunctionInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- started := make(chan struct{})
- exited := make(chan struct{})
- runner, err := NewRunner(t.Context(), runtime, "runner-test", "lifecycle", "test runner", 1, func(ctx context.Context) error {
- close(started)
- <-ctx.Done()
- close(exited)
- return nil
- })
- require.NoError(t, err)
- select {
- case <-started:
- case <-time.After(time.Second):
- require.FailNow(t, "runner did not start")
- }
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, runner.Stop(stopCtx))
- select {
- case <-exited:
- default:
- require.FailNow(t, "runner stop returned before background function exited")
- }
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/runtime.go b/backend/internal/actors/runtime.go
deleted file mode 100644
index f8c9c5714..000000000
--- a/backend/internal/actors/runtime.go
+++ /dev/null
@@ -1,290 +0,0 @@
-// Package actors owns Arcane's in-process actor runtime and its shared safety rails.
-package actors
-
-import (
- "context"
- "fmt"
- "log/slog"
- "math"
- "sync"
- "sync/atomic"
-
- "emperror.dev/errors"
- "github.com/anthdm/hollywood/actor"
- "go.uber.org/fx"
-)
-
-const actorMonitorID = "arcane-monitor"
-
-type monitorReadyInternal struct{}
-
-// Runtime owns the shared Hollywood engine and observes actor failures.
-type Runtime struct {
- engine *actor.Engine
- monitorPID *actor.PID
- deadLetters atomic.Uint64
- restarts atomic.Uint64
- maxRestartsExceeded atomic.Uint64
- duplicateIDs atomic.Uint64
- policyMu sync.Mutex
- policies map[string]actorPolicyInternal
-}
-
-type actorPolicyInternal struct {
- maxRestarts int32
- handle *handleInternal
-}
-
-// handleInternal identifies a spawned actor and reports when its process terminates.
-type handleInternal struct {
- engine *actor.Engine
- pid *actor.PID
- done chan struct{}
- doneOnce sync.Once
- stopOnce sync.Once
-}
-
-// Done closes when the actor process permanently stops.
-func (h *handleInternal) Done() <-chan struct{} {
- if h == nil {
- return nil
- }
- return h.done
-}
-
-// Send queues a message for the actor while hiding Hollywood's engine and PID
-// from services that only need to communicate with their owned process.
-func (h *handleInternal) Send(message any) error {
- if h == nil || h.engine == nil || h.pid == nil {
- return errors.New("actor handle unavailable")
- }
- select {
- case <-h.done:
- return errors.New("actor stopped")
- default:
- h.engine.Send(h.pid, message)
- return nil
- }
-}
-
-// Stop drains the actor mailbox, initiates termination exactly once, and waits
-// for the process to exit. A canceled wait does not cancel the queued stop.
-func (h *handleInternal) Stop(ctx context.Context) error {
- if h == nil || h.engine == nil || h.pid == nil {
- return nil
- }
- if ctx == nil {
- return errors.New("actor stop context unavailable")
- }
- h.stopOnce.Do(func() { h.engine.Poison(h.pid) })
- select {
- case <-h.done:
- return nil
- case <-ctx.Done():
- return ctx.Err()
- }
-}
-
-func (h *handleInternal) stopInternal() {
- if h != nil {
- h.doneOnce.Do(func() { close(h.done) })
- }
-}
-
-// NewRuntime constructs the shared actor engine and registers its lifecycle monitor.
-func NewRuntime(appCtx context.Context, lc fx.Lifecycle) (*Runtime, error) {
- engine, err := actor.NewEngine(actor.NewEngineConfig())
- if err != nil {
- return nil, err
- }
-
- runtime := &Runtime{
- engine: engine,
- policies: make(map[string]actorPolicyInternal),
- }
- ready := make(chan struct{})
- var readyOnce sync.Once
- runtime.monitorPID = engine.Spawn(
- func() actor.Receiver {
- return &runtimeMonitorInternal{
- runtime: runtime,
- ready: ready,
- readyOnce: &readyOnce,
- }
- },
- "runtime-monitor",
- actor.WithID(actorMonitorID),
- actor.WithContext(appCtx),
- actor.WithMaxRestarts(math.MaxInt32),
- )
- engine.Subscribe(runtime.monitorPID)
- engine.BroadcastEvent(monitorReadyInternal{})
-
- select {
- case <-ready:
- case <-appCtx.Done():
- <-engine.Poison(runtime.monitorPID).Done()
- return nil, appCtx.Err()
- }
-
- lc.Append(fx.Hook{
- OnStop: func(ctx context.Context) error {
- engine.Unsubscribe(runtime.monitorPID)
- stopped := engine.PoisonCtx(ctx, runtime.monitorPID)
- select {
- case <-stopped.Done():
- return nil
- case <-ctx.Done():
- return ctx.Err()
- }
- },
- })
-
- return runtime, nil
-}
-
-// startInternal starts a Hollywood receiver with a deterministic ID and an
-// Arcane-enforced restart limit.
-// Hollywood v1.0.5 panics while cleaning up its native max-restart path, so the
-// shared monitor enforces the same ceiling and stops the actor safely instead.
-func (r *Runtime) startInternal(producer actor.Producer, kind, id string, maxRestarts int32, opts ...actor.OptFunc) (*handleInternal, error) {
- if r == nil || r.engine == nil {
- return nil, errors.New("actor runtime unavailable")
- }
- if producer == nil {
- return nil, errors.New("actor producer unavailable")
- }
-
- policyKey := actor.NewPID(r.engine.Address(), kind+"/"+id).String()
- r.policyMu.Lock()
- if _, exists := r.policies[policyKey]; exists || r.engine.Registry.GetPID(kind, id) != nil {
- r.policyMu.Unlock()
- return nil, fmt.Errorf("actor ID already in use: %s/%s", kind, id)
- }
- handle := &handleInternal{
- engine: r.engine,
- pid: actor.NewPID(r.engine.Address(), kind+"/"+id),
- done: make(chan struct{}),
- }
- r.policies[policyKey] = actorPolicyInternal{maxRestarts: maxRestarts, handle: handle}
- r.policyMu.Unlock()
-
- options := append([]actor.OptFunc{}, opts...)
- options = append(options, actor.WithID(id), actor.WithMaxRestarts(math.MaxInt32))
- handle.pid = r.engine.Spawn(producer, kind, options...)
- if handle.pid.String() != policyKey {
- r.policyMu.Lock()
- delete(r.policies, policyKey)
- r.policyMu.Unlock()
- r.engine.Stop(handle.pid)
- return nil, fmt.Errorf("actor PID mismatch: got %s, expected %s", handle.pid.String(), policyKey)
- }
- return handle, nil
-}
-
-// DeadLetterCount returns the number of undeliverable actor messages observed.
-func (r *Runtime) DeadLetterCount() uint64 {
- if r == nil {
- return 0
- }
- return r.deadLetters.Load()
-}
-
-// RestartCount returns the number of actor restarts observed.
-func (r *Runtime) RestartCount() uint64 {
- if r == nil {
- return 0
- }
- return r.restarts.Load()
-}
-
-// MaxRestartsExceededCount returns the number of permanently stopped actors observed.
-func (r *Runtime) MaxRestartsExceededCount() uint64 {
- if r == nil {
- return 0
- }
- return r.maxRestartsExceeded.Load()
-}
-
-// DuplicateIDCount returns the number of duplicate actor registrations observed.
-func (r *Runtime) DuplicateIDCount() uint64 {
- if r == nil {
- return 0
- }
- return r.duplicateIDs.Load()
-}
-
-type runtimeMonitorInternal struct {
- runtime *Runtime
- ready chan struct{}
- readyOnce *sync.Once
-}
-
-func (m *runtimeMonitorInternal) Receive(ctx *actor.Context) {
- switch event := ctx.Message().(type) {
- case actor.Started:
- ctx.Engine().Subscribe(ctx.PID())
-
- case monitorReadyInternal:
- m.readyOnce.Do(func() { close(m.ready) })
-
- case actor.ActorRestartedEvent:
- if event.PID == nil {
- slog.ErrorContext(ctx.Context(), "actor restarted event omitted PID")
- return
- }
- m.runtime.policyMu.Lock()
- policy, tracked := m.runtime.policies[event.PID.String()]
- m.runtime.policyMu.Unlock()
- if tracked && event.Restarts > policy.maxRestarts {
- m.runtime.maxRestartsExceeded.Add(1)
- slog.ErrorContext(ctx.Context(), "actor permanently stopped after exceeding restart limit", "pid", event.PID.String(), "reason", event.Reason, "restarts", event.Restarts, "maxRestarts", policy.maxRestarts, "stack", string(event.Stacktrace))
- ctx.Engine().Stop(event.PID)
- return
- }
- m.runtime.restarts.Add(1)
- slog.ErrorContext(ctx.Context(), "actor crashed and restarted", "pid", event.PID.String(), "reason", event.Reason, "restarts", event.Restarts, "stack", string(event.Stacktrace))
-
- case actor.ActorMaxRestartsExceededEvent:
- if event.PID == nil {
- slog.ErrorContext(ctx.Context(), "actor max-restarts event omitted PID")
- return
- }
- m.runtime.maxRestartsExceeded.Add(1)
- slog.ErrorContext(ctx.Context(), "actor permanently stopped after exceeding restart limit", "pid", event.PID.String())
-
- case actor.ActorStoppedEvent:
- if event.PID == nil {
- slog.ErrorContext(ctx.Context(), "actor stopped event omitted PID")
- return
- }
- m.runtime.policyMu.Lock()
- policy, tracked := m.runtime.policies[event.PID.String()]
- if tracked {
- delete(m.runtime.policies, event.PID.String())
- }
- m.runtime.policyMu.Unlock()
- if tracked {
- policy.handle.stopInternal()
- }
-
- case actor.ActorDuplicateIdEvent:
- m.runtime.duplicateIDs.Add(1)
- pid := "<nil>"
- if event.PID != nil {
- pid = event.PID.String()
- }
- slog.ErrorContext(ctx.Context(), "duplicate actor ID rejected", "pid", pid)
-
- case actor.DeadLetterEvent:
- m.runtime.deadLetters.Add(1)
- target := "<nil>"
- if event.Target != nil {
- target = event.Target.String()
- }
- slog.ErrorContext(ctx.Context(), "actor message sent to dead letter", "target", target, "messageType", fmt.Sprintf("%T", event.Message))
-
- case actor.Stopped:
- ctx.Engine().Unsubscribe(ctx.PID())
- }
-}
diff --git a/backend/internal/actors/runtime_test.go b/backend/internal/actors/runtime_test.go
deleted file mode 100644
index ec01dccae..000000000
--- a/backend/internal/actors/runtime_test.go
+++ /dev/null
@@ -1,154 +0,0 @@
-package actors
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/anthdm/hollywood/actor"
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-type panicActorInternal struct{}
-
-func (panicActorInternal) Receive(ctx *actor.Context) {
- if _, ok := ctx.Message().(panicActorMessageInternal); ok {
- panic("deliberate actor test panic")
- }
-}
-
-type panicActorMessageInternal struct{}
-
-type stopBlockingActorInternal struct {
- release <-chan struct{}
-}
-
-func (a stopBlockingActorInternal) Receive(ctx *actor.Context) {
- if _, ok := ctx.Message().(actor.Stopped); ok {
- <-a.release
- }
-}
-
-func TestRuntimeMonitorObservesRestartsMaxRestartsAndDeadLettersInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- handle, err := runtime.startInternal(
- func() actor.Receiver { return panicActorInternal{} },
- "panic-test",
- "restart-limit",
- 3,
- actor.WithContext(t.Context()),
- actor.WithRestartDelay(0),
- )
- require.NoError(t, err)
- pid := handle.pid
-
- for expectedRestarts := uint64(1); expectedRestarts <= 3; expectedRestarts++ {
- require.NoError(t, handle.Send(panicActorMessageInternal{}))
- require.Eventually(t, func() bool {
- return runtime.RestartCount() == expectedRestarts
- }, time.Second, time.Millisecond)
- }
-
- require.NoError(t, handle.Send(panicActorMessageInternal{}))
- require.Eventually(t, func() bool {
- return runtime.MaxRestartsExceededCount() == 1
- }, time.Second, time.Millisecond)
- require.Eventually(t, func() bool {
- select {
- case <-handle.Done():
- return true
- default:
- return false
- }
- }, time.Second, time.Millisecond)
-
- runtime.engine.Send(pid, panicActorMessageInternal{})
- require.Eventually(t, func() bool {
- return runtime.DeadLetterCount() == 1
- }, time.Second, time.Millisecond)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestHandleCanceledStopStillTerminatesAndCanBeJoinedInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- release := make(chan struct{})
- handle, err := runtime.startInternal(
- func() actor.Receiver { return stopBlockingActorInternal{release: release} },
- "handle-test",
- "stop",
- 1,
- actor.WithContext(t.Context()),
- )
- require.NoError(t, err)
- require.NotNil(t, handle)
-
- canceled, cancel := context.WithCancel(context.Background())
- cancel()
- require.ErrorIs(t, handle.Stop(canceled), context.Canceled)
- close(release)
-
- stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
- defer stopCancel()
- require.NoError(t, handle.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestRuntimeRejectsDuplicateActorIDWithoutOverwritingOwnerInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- first, err := runtime.startInternal(
- func() actor.Receiver { return panicActorInternal{} },
- "duplicate-test",
- "same-id",
- 1,
- actor.WithContext(t.Context()),
- )
- require.NoError(t, err)
-
- second, err := runtime.startInternal(
- func() actor.Receiver { return panicActorInternal{} },
- "duplicate-test",
- "same-id",
- 1,
- actor.WithContext(t.Context()),
- )
- require.Nil(t, second)
- require.ErrorContains(t, err, "actor ID already in use")
- require.Equal(t, actor.NewPID(runtime.engine.Address(), "duplicate-test/same-id").String(), first.pid.String())
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, first.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestRuntimeMonitorResubscribesOnStartedInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
-
- runtime.engine.Unsubscribe(runtime.monitorPID)
- time.Sleep(10 * time.Millisecond)
- runtime.engine.Send(runtime.monitorPID, actor.Started{})
- time.Sleep(10 * time.Millisecond)
- runtime.engine.Send(actor.NewPID(runtime.engine.Address(), "missing/actor"), "dead letter")
- require.Eventually(t, func() bool {
- return runtime.DeadLetterCount() == 1
- }, time.Second, time.Millisecond)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/state.go b/backend/internal/actors/state.go
deleted file mode 100644
index cb1abdaaa..000000000
--- a/backend/internal/actors/state.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package actors
-
-import (
- "context"
-
- "emperror.dev/errors"
-)
-
-// State serializes mutations through an actor and publishes immutable snapshots.
-type State[T any] struct {
- executor *Executor
- value T
- clone func(T) T
- snapshot Snapshot[T]
-}
-
-// NewState creates actor-owned state with a caller-defined snapshot clone.
-func NewState[T any](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32, initial T, clone func(T) T) (*State[T], error) {
- if clone == nil {
- return nil, errors.New("actor state clone unavailable")
- }
- executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)
- if err != nil {
- return nil, err
- }
- state := &State[T]{
- executor: executor,
- value: initial,
- clone: clone,
- }
- state.snapshot.Store(clone(initial))
- return state, nil
-}
-
-// Apply serializes one mutation and publishes the resulting state.
-func (s *State[T]) Apply(ctx context.Context, label string, mutate func(context.Context, *T) error) error {
- if s == nil || s.executor == nil {
- return errors.New("actor state unavailable")
- }
- if mutate == nil {
- return errors.New("actor state mutation unavailable")
- }
- _, err := s.executor.Execute(ctx, label, func(actorCtx context.Context) (NoPayload, error) {
- defer func() {
- s.snapshot.Store(s.clone(s.value))
- }()
- return NoPayload{}, mutate(actorCtx, &s.value)
- }, nil)
- return err
-}
-
-// Load returns the latest immutable state snapshot.
-func (s *State[T]) Load() (T, bool) {
- if s == nil {
- var zero T
- return zero, false
- }
- return s.snapshot.Load()
-}
-
-// Stop drains pending mutations and joins the state actor.
-func (s *State[T]) Stop(ctx context.Context) error {
- if s == nil || s.executor == nil {
- return nil
- }
- return s.executor.Stop(ctx)
-}
-
-// Done closes after the state actor terminates.
-func (s *State[T]) Done() <-chan struct{} {
- if s == nil || s.executor == nil {
- return nil
- }
- return s.executor.Done()
-}
diff --git a/backend/internal/actors/state_map_test.go b/backend/internal/actors/state_map_test.go
deleted file mode 100644
index fb792bffc..000000000
--- a/backend/internal/actors/state_map_test.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package actors
-
-import (
- "context"
- "sync"
- "testing"
- "time"
-
- "emperror.dev/errors"
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-func TestActorStateMapSerializesMutationsAndPublishesSnapshotsInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- state, err := NewActorStateMap[string, int](t.Context(), runtime, "state-map-test", "values", 1)
- require.NoError(t, err)
-
- require.NoError(t, state.Apply(t.Context(), "store value", func(values map[string]int) (bool, error) {
- values["answer"] = 42
- return true, nil
- }))
- value, ok := state.Get("answer")
- require.True(t, ok)
- require.Equal(t, 42, value)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, state.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
-
-func TestActorStateMapPublishesPartialMutationBeforeReturningErrorInternal(t *testing.T) {
- state := NewStateMap[string, int]()
- require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
- values["removed"] = 1
- values["retained"] = 2
- return true, nil
- }))
-
- expectedErr := errors.New("mutation failed after change")
- err := state.Apply(t.Context(), "partial mutation", func(values map[string]int) (bool, error) {
- delete(values, "removed")
- return true, expectedErr
- })
- require.ErrorIs(t, err, expectedErr)
- _, found := state.Get("removed")
- require.False(t, found)
- retained, found := state.Get("retained")
- require.True(t, found)
- require.Equal(t, 2, retained)
-}
-
-func TestStateMapPublishesPartialMutationBeforePropagatingPanicInternal(t *testing.T) {
- state := NewStateMap[string, int]()
- require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
- values["removed"] = 1
- return true, nil
- }))
-
- require.Panics(t, func() {
- _ = state.Apply(t.Context(), "panic after mutation", func(values map[string]int) (bool, error) {
- delete(values, "removed")
- panic("mutation panic")
- })
- })
- _, found := state.Get("removed")
- require.False(t, found)
-}
-
-func TestActorStateMapCanceledMutationDoesNotShareResultStorageInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- state, err := NewActorStateMap[string, int](t.Context(), runtime, "state-map-test", "canceled-result", 1)
- require.NoError(t, err)
- require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
- values["first"] = 1
- values["second"] = 2
- return true, nil
- }))
-
- mutationStarted := make(chan struct{})
- releaseMutation := make(chan struct{})
- var started sync.Once
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
- defer cancel()
- result := make(chan []int, 1)
- resultErr := make(chan error, 1)
- go func() {
- removed, removeErr := state.RemoveWhere(ctx, "blocked removal", func(string, int) bool {
- started.Do(func() { close(mutationStarted) })
- <-releaseMutation
- return true
- })
- result <- removed
- resultErr <- removeErr
- }()
-
- <-mutationStarted
- require.ErrorIs(t, <-resultErr, context.DeadlineExceeded)
- require.Empty(t, <-result)
- close(releaseMutation)
- require.Eventually(t, func() bool {
- return len(state.Values()) == 0
- }, time.Second, time.Millisecond)
-
- stopCtx, stopCancel := context.WithTimeout(context.Background(), time.Second)
- defer stopCancel()
- require.NoError(t, state.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/actors/state_test.go b/backend/internal/actors/state_test.go
deleted file mode 100644
index f57629ec0..000000000
--- a/backend/internal/actors/state_test.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package actors
-
-import (
- "context"
- "maps"
- "testing"
- "time"
-
- "emperror.dev/errors"
- "github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
-)
-
-type stateTestValueInternal struct {
- values map[string]int
-}
-
-func TestStateSerializesMutationsAndPublishesClonedSnapshotsInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- state, err := NewState(
- t.Context(),
- runtime,
- "state-test",
- "values",
- 1,
- stateTestValueInternal{values: make(map[string]int)},
- func(value stateTestValueInternal) stateTestValueInternal {
- value.values = maps.Clone(value.values)
- return value
- },
- )
- require.NoError(t, err)
-
- expectedErr := errors.New("mutation failed")
- err = state.Apply(t.Context(), "mutate state", func(_ context.Context, value *stateTestValueInternal) error {
- value.values["answer"] = 42
- return expectedErr
- })
- require.ErrorIs(t, err, expectedErr)
- snapshot, ok := state.Load()
- require.True(t, ok)
- require.Equal(t, 42, snapshot.values["answer"])
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, state.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
-}
diff --git a/backend/internal/apikey/module.go b/backend/internal/apikey/module.go
index 554bc627a..ac9e6e089 100644
--- a/backend/internal/apikey/module.go
+++ b/backend/internal/apikey/module.go
@@ -4,23 +4,14 @@ package apikey
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/role"
- "github.com/getarcaneapp/arcane/backend/v2/internal/user"
)
-type Dependencies struct {
- DB *database.DB
- User *user.UserService
- Role *role.RoleService
-}
-
type Module struct {
service *ApiKeyService
}
-func New(deps Dependencies) *Module {
- return &Module{service: NewApiKeyService(deps.DB, deps.User).WithRoleService(deps.Role)}
+func New(service *ApiKeyService) *Module {
+ return &Module{service: service}
}
func (m *Module) Service() *ApiKeyService {
diff --git a/backend/internal/apikey/service.go b/backend/internal/apikey/service.go
index 43e8df739..de2c7cdd9 100644
--- a/backend/internal/apikey/service.go
+++ b/backend/internal/apikey/service.go
@@ -87,7 +87,7 @@ type ApiKeyService struct {
lastUsedWrites map[string]time.Time
}
-func NewApiKeyService(db *database.DB, userService *user.UserService) *ApiKeyService {
+func NewApiKeyService(db *database.DB, userService *user.UserService, roleService *role.RoleService) *ApiKeyService {
pepper := make([]byte, 32)
if _, err := rand.Read(pepper); err != nil {
panic(errors.WrapIf(err, "failed to generate API key cache pepper"))
@@ -96,6 +96,7 @@ func NewApiKeyService(db *database.DB, userService *user.UserService) *ApiKeySer
cacheKeyPepper: pepper,
db: db,
userService: userService,
+ roleService: roleService,
argon2Params: user.DefaultArgon2Params(),
validatedKeyCache: hot.NewHotCache[string, ApiKey](hot.LRU, 4096).
WithTTL(15 * time.Second).
@@ -105,16 +106,6 @@ func NewApiKeyService(db *database.DB, userService *user.UserService) *ApiKeySer
}
}
-// WithRoleService wires the role.RoleService dependency. Separated from the
-// constructor to break the bootstrap-ordering cycle between ApiKeyService and
-// role.RoleService (role.RoleService.BackfillApiKeyPermissions needs ApiKeyService to
-// exist when it runs, while permission-validated CreateApiKey needs the
-// role.RoleService).
-func (s *ApiKeyService) WithRoleService(roleService *role.RoleService) *ApiKeyService {
- s.roleService = roleService
- return s
-}
-
// BackfillApiKeyPermissions ensures every ownerless (bootstrap) API key has
// its expected permission grants. Called once per boot.
//
diff --git a/backend/internal/apikey/service_test.go b/backend/internal/apikey/service_test.go
index b5f49d4e4..3f350286a 100644
--- a/backend/internal/apikey/service_test.go
+++ b/backend/internal/apikey/service_test.go
@@ -53,8 +53,8 @@ func setupAPIKeyService(t *testing.T) (*ApiKeyService, *database.DB, *user.UserS
t.Helper()
db := setupAPIKeyServiceTestDB(t)
- userService := user.NewUserService(db)
- return NewApiKeyService(db, userService), db, userService
+ userService := user.NewUserService(db, nil)
+ return NewApiKeyService(db, userService, nil), db, userService
}
func createTestAPIKeyUser(t *testing.T, ctx context.Context, userService *user.UserService, id string, usernames ...string) *common.User {
@@ -124,7 +124,7 @@ func TestListApiKeysPermissionQueryCountIsConstant(t *testing.T) {
for _, keyCount := range []int{1, 5} {
t.Run(fmt.Sprintf("%d_keys", keyCount), func(t *testing.T) {
db := setupAPIKeyServiceTestDB(t)
- service := NewApiKeyService(db, user.NewUserService(db)).WithRoleService(role.NewRoleService(db))
+ service := NewApiKeyService(db, user.NewUserService(db, nil), role.NewRoleService(db))
userID := "query-count-user"
apiKeys := make([]ApiKey, keyCount)
@@ -268,8 +268,8 @@ func TestUpdateApiKeyRollsBackMetadataWhenPermissionUpdateFails(t *testing.T) {
roleSvc := role.NewRoleService(db)
require.NoError(t, roleSvc.EnsureBuiltInRoles(ctx))
- userSvc := user.NewUserService(db).WithRoleService(roleSvc)
- service := NewApiKeyService(db, userSvc).WithRoleService(roleSvc)
+ userSvc := user.NewUserService(db, roleSvc)
+ service := NewApiKeyService(db, userSvc, roleSvc)
admin := createTestAPIKeyUser(t, ctx, userSvc, "admin-update-rollback", "admin-update-rollback")
require.NoError(t, roleSvc.SetUserAssignments(ctx, admin.ID, []role.UserRoleAssignment{
{RoleID: authz.BuiltInRoleAdmin, EnvironmentID: nil},
@@ -322,8 +322,8 @@ func TestApiKeyGrantsAreCappedByOwnerRoles(t *testing.T) {
roleSvc := role.NewRoleService(db)
require.NoError(t, roleSvc.EnsureBuiltInRoles(ctx))
- userSvc := user.NewUserService(db).WithRoleService(roleSvc)
- service := NewApiKeyService(db, userSvc).WithRoleService(roleSvc)
+ userSvc := user.NewUserService(db, roleSvc)
+ service := NewApiKeyService(db, userSvc, roleSvc)
// Owner has no roles at all — their permission ceiling is empty.
owner := createTestAPIKeyUser(t, ctx, userSvc, "roleless-owner", "roleless-owner")
@@ -837,8 +837,8 @@ func TestCreateEnvironmentApiKeySeedsAllPermissionsScopedToEnv(t *testing.T) {
roleSvc := role.NewRoleService(db)
require.NoError(t, roleSvc.EnsureBuiltInRoles(ctx))
- userSvc := user.NewUserService(db).WithRoleService(roleSvc)
- service := NewApiKeyService(db, userSvc).WithRoleService(roleSvc)
+ userSvc := user.NewUserService(db, roleSvc)
+ service := NewApiKeyService(db, userSvc, roleSvc)
admin := createTestAPIKeyUser(t, ctx, userSvc, "admin-env-bootstrap", "admin-env-bootstrap")
require.NoError(t, roleSvc.SetUserAssignments(ctx, admin.ID, []role.UserRoleAssignment{
{RoleID: authz.BuiltInRoleAdmin, EnvironmentID: nil},
@@ -868,8 +868,8 @@ func TestBackfillApiKeyPermissionsRepairsExistingBootstrapKey(t *testing.T) {
roleSvc := role.NewRoleService(db)
require.NoError(t, roleSvc.EnsureBuiltInRoles(ctx))
- userSvc := user.NewUserService(db).WithRoleService(roleSvc)
- service := NewApiKeyService(db, userSvc).WithRoleService(roleSvc)
+ userSvc := user.NewUserService(db, roleSvc)
+ service := NewApiKeyService(db, userSvc, roleSvc)
// Simulate a pre-existing env-bootstrap key with NO permission grants
// (e.g., created on a deployment where the per-key seed step failed).
@@ -921,8 +921,8 @@ func TestBackfillPermsForKeyDeduplicatesGlobalAndEnvironmentPermissions(t *testi
roleSvc := role.NewRoleService(db)
require.NoError(t, roleSvc.EnsureBuiltInRoles(ctx))
- userSvc := user.NewUserService(db).WithRoleService(roleSvc)
- service := NewApiKeyService(db, userSvc).WithRoleService(roleSvc)
+ userSvc := user.NewUserService(db, roleSvc)
+ service := NewApiKeyService(db, userSvc, roleSvc)
admin := createTestAPIKeyUser(t, ctx, userSvc, "admin", "admin")
require.NoError(t, roleSvc.SetUserAssignments(ctx, admin.ID, []role.UserRoleAssignment{
diff --git a/backend/internal/auth/huma_middleware_test.go b/backend/internal/auth/huma_middleware_test.go
index cb2b12326..d08609977 100644
--- a/backend/internal/auth/huma_middleware_test.go
+++ b/backend/internal/auth/huma_middleware_test.go
@@ -100,7 +100,7 @@ func TestNewHumaMiddleware_AcceptsEnvironmentAccessTokenViaAPIKey(t *testing.T)
func TestNewHumaMiddleware_UsesBearerWhenLoopbackProxySendsEnvironmentAccessToken(t *testing.T) {
db := setupAuthMiddlewareTestDBInternal(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
sessionSvc := session.NewSessionService(db)
signingKey := newTestSigningKeyInternal()
@@ -263,7 +263,7 @@ func TestParseSecurityRequirements(t *testing.T) {
func TestNewHumaMiddleware_OpportunisticAuthOnPublicRoute(t *testing.T) {
db := setupAuthMiddlewareTestDBInternal(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
sessionSvc := session.NewSessionService(db)
signingKey := newTestSigningKeyInternal()
@@ -350,7 +350,7 @@ func TestNewHumaMiddleware_OpportunisticAuthOnPublicRoute(t *testing.T) {
// the user is logged out on every update.
func TestNewHumaMiddleware_VersionMismatchIsRecoverable(t *testing.T) {
db := setupAuthMiddlewareTestDBInternal(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
sessionSvc := session.NewSessionService(db)
signingKey := newTestSigningKeyInternal()
diff --git a/backend/internal/auth/middleware_test.go b/backend/internal/auth/middleware_test.go
index 1faf83ffa..01f86d586 100644
--- a/backend/internal/auth/middleware_test.go
+++ b/backend/internal/auth/middleware_test.go
@@ -107,7 +107,7 @@ func TestAuthMiddleware_ManagerAuthResolvesPermissionsByKeyKind(t *testing.T) {
t.Run("browser cookie uses owner role permissions", func(t *testing.T) {
ctx := context.Background()
db := setupAuthMiddlewareTestDBInternal(t)
- userSvc := usersvc.NewUserService(db)
+ userSvc := usersvc.NewUserService(db, nil)
sessionSvc := session.NewSessionService(db)
settingsSvc, err := newSettingsServiceForAuthTestInternal(t, ctx, db)
require.NoError(t, err)
diff --git a/backend/internal/auth/module.go b/backend/internal/auth/module.go
index 5647d335d..36c98715e 100644
--- a/backend/internal/auth/module.go
+++ b/backend/internal/auth/module.go
@@ -11,26 +11,22 @@ import (
authtypes "github.com/getarcaneapp/arcane/types/v2/auth"
)
-type Dependencies struct {
- Service *AuthService
- User *user.UserService
- Settings *settings.SettingsService
- BeginMFAAuthentication func(context.Context, string, authtypes.SessionMeta, string) (*authtypes.MFAChallenge, error)
-}
-
type Module struct {
- deps Dependencies
+ service *AuthService
+ user *user.UserService
+ settings *settings.SettingsService
+ beginMFAAuthentication func(context.Context, string, authtypes.SessionMeta, string) (*authtypes.MFAChallenge, error)
}
-func New(deps Dependencies) *Module {
- return &Module{deps: deps}
+func New(service *AuthService, userService *user.UserService, settingsService *settings.SettingsService, beginMFAAuthentication func(context.Context, string, authtypes.SessionMeta, string) (*authtypes.MFAChallenge, error)) *Module {
+ return &Module{service: service, user: userService, settings: settingsService, beginMFAAuthentication: beginMFAAuthentication}
}
func (m *Module) Service() *AuthService {
if m == nil {
return nil
}
- return m.deps.Service
+ return m.service
}
func (m *Module) RegisterRoutes(api huma.API) {
@@ -38,5 +34,5 @@ func (m *Module) RegisterRoutes(api huma.API) {
RegisterAuth(api, nil, nil, nil, nil)
return
}
- RegisterAuth(api, m.deps.User, m.deps.Service, m.deps.Settings, m.deps.BeginMFAAuthentication)
+ RegisterAuth(api, m.user, m.service, m.settings, m.beginMFAAuthentication)
}
diff --git a/backend/internal/auth/service_test.go b/backend/internal/auth/service_test.go
index 62debe958..18d354bd3 100644
--- a/backend/internal/auth/service_test.go
+++ b/backend/internal/auth/service_test.go
@@ -9,7 +9,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -26,7 +25,6 @@ import (
"github.com/samber/hot"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -55,21 +53,11 @@ func setupAuthServiceTestDB(t *testing.T) *database.DB {
func newSettingsServiceForAuthTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- writes, err := actors.NewExecutor(t.Context(), runtime, "auth-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "auth-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, writes.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, writes, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func newTestSigningKeyInternal() *mldsa.PrivateKey {
@@ -188,7 +176,7 @@ func makeUnsignedToken(t *testing.T, claims map[string]any) string {
func TestVerifyToken_ValidClaims(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -251,7 +239,7 @@ func TestVerifyToken_RejectsNonMLDSAAlg(t *testing.T) {
func TestVerifyToken_Expired(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -381,7 +369,7 @@ func TestVerifyToken_VersionMismatch(t *testing.T) {
func TestRefreshToken_Valid(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
settingsSvc, err := newSettingsServiceForAuthTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService()
@@ -424,7 +412,7 @@ func TestRefreshToken_Valid(t *testing.T) {
// users logged in across backend releases — see plans/how-can-we-make-compiled-quilt.md.
func TestRefreshToken_VersionMismatchRotates(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
settingsSvc, err := newSettingsServiceForAuthTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService()
@@ -484,7 +472,7 @@ func TestRefreshToken_VersionMismatchRotates(t *testing.T) {
func TestVerifyToken_RejectsRevokedSession(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -520,7 +508,7 @@ func TestVerifyToken_RejectsRevokedSession(t *testing.T) {
func TestVerifyToken_RejectsMissingSessionID(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -540,7 +528,7 @@ func TestVerifyToken_RejectsMissingSessionID(t *testing.T) {
func TestRevokeSessionThenVerifyTokenFails(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -563,7 +551,7 @@ func TestRevokeSessionThenVerifyTokenFails(t *testing.T) {
func TestVerifyToken_RejectsRevokedCachedSession(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -589,7 +577,7 @@ func TestVerifyToken_RejectsRevokedCachedSession(t *testing.T) {
func TestRefreshToken_RotatesJTI(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
settingsSvc, err := newSettingsServiceForAuthTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService()
@@ -618,7 +606,7 @@ func TestRefreshToken_RotatesJTI(t *testing.T) {
func TestRefreshToken_RejectsRevokedSession(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
settingsSvc, err := newSettingsServiceForAuthTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService()
@@ -644,7 +632,7 @@ func TestRefreshToken_RejectsRevokedSession(t *testing.T) {
func TestChangePassword_RevokesAllSessions(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -674,7 +662,7 @@ func TestChangePassword_RevokesAllSessions(t *testing.T) {
func TestChangePassword_KeepsCurrentSessionAlive(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
s := newTestAuthService()
s.userService = userSvc
s.sessionService = session.NewSessionService(db)
@@ -789,7 +777,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailNotVerified_NoExistingUser_Creat
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
authSvc := newTestAuthService()
authSvc.userService = userSvc
authSvc.settingsService = settingsSvc
@@ -843,7 +831,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailNotVerified_WithExistingUser_Ret
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
// Seed an existing local user with matching email
email := "[email protected]"
existing := &common.User{
@@ -885,7 +873,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailVerificationMissing_WithExisting
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "oidcMergeAccounts", true))
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
// Seed an existing local user with matching email
email := "[email protected]"
existing := &common.User{
@@ -922,7 +910,7 @@ func TestFindOrCreateOidcUser_MergeEnabled_EmailVerificationMissing_WithExisting
func TestAuthenticateLocalPrimary_EmailFallback(t *testing.T) {
db := setupAuthServiceTestDB(t)
- userSvc := user.NewUserService(db)
+ userSvc := user.NewUserService(db, nil)
settingsSvc, err := newSettingsServiceForAuthTestInternal(t, context.Background(), db)
require.NoError(t, err)
s := newTestAuthService()
diff --git a/backend/internal/backup/engine.go b/backend/internal/backup/engine.go
index 07b604ec4..c205c2b6d 100644
--- a/backend/internal/backup/engine.go
+++ b/backend/internal/backup/engine.go
@@ -1,6 +1,6 @@
// Package backup owns the shared Rustic backup engine used by volume and
// system backups: typed repository operations, per-repository serialization
-// through the actor runtime, and run admission.
+// and durable run admission.
package backup
import (
@@ -18,17 +18,21 @@ import (
"regexp"
"slices"
"strings"
+ "sync"
"time"
+ "uuid"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
- "github.com/google/uuid"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
@@ -38,25 +42,17 @@ import (
)
const (
- // Admission scopes shared by the backup engine and the per-policy job
- // registries so scheduled and manual runs contend on the same leases.
-
VolumeAdmissionScope = "volume-backup"
SystemAdmissionScope = "system-backup"
-
- // RecoveryKeyConfigID is the singleton row holding the instance-wide backup recovery key.
- RecoveryKeyConfigID = "system-recovery"
+ RecoveryKeyConfigID = "system-recovery"
)
-// Repository addresses one Rustic repository. ID is the stable serialization
-// key: operations against the same ID run serially, different IDs concurrently.
type Repository struct {
ID string
Environment []string
Mounts []mount.Mount
}
-// Snapshot is the result of a completed Rustic backup operation.
type Snapshot struct {
ID string `json:"id"`
Size int64 `json:"-"`
@@ -89,11 +85,13 @@ type CreateSnapshotInput struct {
Mounts []mount.Mount
Sources []string
AsPath string
+ Tags []string
+ Globs []string
}
// RootSnapshotInput captures one mount with its contents at the snapshot root.
-func RootSnapshotInput(source mount.Mount) CreateSnapshotInput {
- return CreateSnapshotInput{Mounts: []mount.Mount{source}, Sources: []string{source.Target}, AsPath: "/"}
+func RootSnapshotInput(source mount.Mount, tags ...string) CreateSnapshotInput {
+ return CreateSnapshotInput{Mounts: []mount.Mount{source}, Sources: []string{source.Target}, AsPath: "/", Tags: tags}
}
func snapshotCommandInternal(label string, input CreateSnapshotInput) ([]string, error) {
@@ -104,6 +102,12 @@ func snapshotCommandInternal(label string, input CreateSnapshotInput) ([]string,
return nil, errors.New("a snapshot path rewrite requires a single source")
}
command := []string{"backup", "--init", "--json", "--host", "arcane", "--label", label}
+ for _, tag := range input.Tags {
+ command = append(command, "--tag", tag)
+ }
+ for _, glob := range input.Globs {
+ command = append(command, "--glob", glob)
+ }
if input.AsPath != "" {
command = append(command, "--as-path", input.AsPath)
}
@@ -111,58 +115,62 @@ func snapshotCommandInternal(label string, input CreateSnapshotInput) ([]string,
return append(command, input.Sources...), nil
}
-// Engine executes typed Rustic operations through the official Rustic image.
-// Concurrency is actor-owned: one executor per repository ID plus the shared
-// application admission gate for run exclusivity.
+// Engine owns repository serialization and application-owned backup workers.
type Engine struct {
- imageService *image.ImageService
- runtime *actors.Runtime
- admission *actors.Gate[actors.AdmissionKey]
- lifecycleCtx context.Context
- executors *actors.StateMap[string, *actors.Executor]
- runs *actors.StateMap[string, *backupRunInternal]
- stopping bool // Accessed inside runs.Apply.
-}
-
-// NewEngine creates the shared backup engine on the actor runtime. ctx is the
-// application lifecycle context repository executors are spawned on.
-func NewEngine(ctx context.Context, runtime *actors.Runtime, admission *actors.Gate[actors.AdmissionKey], imageService *image.ImageService) *Engine {
- return &Engine{
- imageService: imageService,
- runtime: runtime,
- admission: admission,
- lifecycleCtx: ctx,
- executors: actors.NewStateMap[string, *actors.Executor](),
- runs: actors.NewStateMap[string, *backupRunInternal](),
- }
-}
-
-// TryAcquireRun admits at most one in-flight backup run per (scope, id),
-// shared between scheduled jobs and manual API triggers.
-func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*actors.Lease[actors.AdmissionKey], bool, error) {
- if e == nil {
+ imageService *image.ImageService
+ admission *runs.Admission
+ lifecycleCtx context.Context
+ cancel context.CancelFunc
+ workers sync.WaitGroup
+ stopping bool
+ mu sync.Mutex
+ repositories map[string]*sync.Mutex
+ service *actor.Service
+ handlers map[string]func(context.Context, string, []byte, bool) error
+ failures map[string]func(context.Context, string, []byte, error) error
+ leases map[string]*runs.Lease
+ executionReady func() bool
+ authorize func(context.Context, backuptypes.DurableRunCommand) error
+}
+
+func NewEngine(ctx context.Context, admission *runs.Admission, imageService *image.ImageService) *Engine {
+ runCtx, cancel := context.WithCancel(ctx)
+ return &Engine{cancel: cancel, imageService: imageService, admission: admission, lifecycleCtx: runCtx, repositories: make(map[string]*sync.Mutex), handlers: make(map[string]func(context.Context, string, []byte, bool) error), failures: make(map[string]func(context.Context, string, []byte, error) error), leases: make(map[string]*runs.Lease)}
+}
+
+func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Lease, bool, error) {
+ if e == nil || e.admission == nil {
return nil, false, errors.New("backup engine is unavailable")
}
- return e.admission.TryAcquire(ctx, actors.AdmissionKey{Scope: scope, ID: id})
+ return e.admission.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: scope, ID: id})
}
-// Stop cancels and joins backup runs before stopping their repository executors.
func (e *Engine) Stop(ctx context.Context) error {
if e == nil {
return nil
}
- stopErr := e.stopRunsInternal(ctx)
- if stopErr != nil {
- return stopErr
- }
- executors, err := e.executors.Drain(ctx, "stop backup repository executors")
- if err != nil {
- return err
- }
- for _, executor := range executors {
- stopErr = errors.Combine(stopErr, executor.Stop(ctx))
+ e.mu.Lock()
+ e.stopping = true
+ e.cancel()
+ e.mu.Unlock()
+ done := make(chan struct{})
+ go func() {
+ e.workers.Wait()
+ e.mu.Lock()
+ leases := e.leases
+ e.leases = make(map[string]*runs.Lease)
+ e.mu.Unlock()
+ for _, lease := range leases {
+ lease.Release(ctx)
+ }
+ close(done)
+ }()
+ select {
+ case <-done:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
}
- return stopErr
}
// CreateSnapshot backs the input's sources up into the repository as one
@@ -296,11 +304,61 @@ type DiscoveredSnapshot struct {
ID string `json:"id"`
Time time.Time `json:"time"`
Label string `json:"label"`
+ Tags []string `json:"tags"`
Summary struct {
TotalBytesProcessed int64 `json:"total_bytes_processed"`
} `json:"summary"`
}
+// RunSnapshotTag links committed snapshots to their durable backup record.
+func RunSnapshotTag(runID string) string { return "arcane-run:" + runID }
+
+// ConfirmRunSnapshot makes a validated snapshot discoverable by run ID.
+func (e *Engine) ConfirmRunSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, runID, snapshotID string) (Snapshot, error) {
+ if !fullSnapshotIDInternal(snapshotID) {
+ return Snapshot{}, errors.New("a full snapshot ID is required")
+ }
+ command := []string{"tag", "--add", RunSnapshotTag(runID), "--", snapshotID}
+ if _, err := e.runInternal(ctx, dockerClient, repository, password, command); err != nil {
+ return Snapshot{}, err
+ }
+ // Tagging rewrites the snapshot and changes its ID.
+ snapshot, found, err := e.FindRunSnapshot(ctx, dockerClient, repository, password, runID, "")
+ if err != nil {
+ return Snapshot{}, err
+ }
+ if !found {
+ return Snapshot{}, errors.New("confirmed backup snapshot is unavailable")
+ }
+ return snapshot, nil
+}
+
+// FindRunSnapshot verifies a known snapshot or recovers its committed response by run tag.
+func (e *Engine) FindRunSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, runID, snapshotID string) (Snapshot, bool, error) {
+ snapshots, err := e.ListSnapshots(ctx, dockerClient, repository, password)
+ if err != nil {
+ return Snapshot{}, false, err
+ }
+ var found Snapshot
+ for _, snapshot := range snapshots {
+ match := snapshotID != "" && snapshot.ID == snapshotID
+ if snapshotID == "" {
+ match = slices.Contains(snapshot.Tags, RunSnapshotTag(runID))
+ }
+ if !match {
+ continue
+ }
+ if found.ID != "" {
+ return Snapshot{}, false, errors.New("multiple snapshots match the backup run")
+ }
+ found = Snapshot{ID: snapshot.ID, Size: snapshot.Summary.TotalBytesProcessed}
+ }
+ if snapshotID != "" && found.ID == "" {
+ return Snapshot{}, false, errors.New("recorded backup snapshot is unavailable")
+ }
+ return found, found.ID != "", nil
+}
+
// ListSnapshots enumerates every snapshot in the repository.
func (e *Engine) ListSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string) ([]DiscoveredSnapshot, error) {
output, err := e.runInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
@@ -414,41 +472,35 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clien
if strings.TrimSpace(repository.ID) == "" {
return errors.New("backup repository ID is required")
}
- executor, err := e.executorForInternal(repository.ID) //nolint:contextcheck // Repository executors outlive requests so shutdown cleanup can finish.
+ defer e.lockRepositoryInternal(repository.ID)()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ output, err := e.runContainerInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
if err != nil {
return err
}
- task, err := executor.Submit(ctx, "rustic forget", func(workCtx context.Context) (string, error) {
- output, err := e.runContainerInternal(workCtx, dockerClient, repository, password, []string{"snapshots", "--json"})
- if err != nil {
- return "", err
- }
- snapshots, err := decodeSnapshotsInternal(output)
- if err != nil {
- return "", err
- }
- existing := make(map[string]string, len(snapshots))
- for _, snapshot := range snapshots {
- existing[strings.ToLower(snapshot.ID)] = snapshot.ID
- }
- command := []string{"forget", "--prune"}
- for _, id := range requested {
- if listedID, found := existing[id]; found {
- if len(command) == 2 {
- command = append(command, "--")
- }
- command = append(command, listedID)
- }
- }
- if len(command) == 2 {
- command = []string{"prune"}
- }
- return e.runContainerInternal(workCtx, dockerClient, repository, password, command)
- }, nil)
+ snapshots, err := decodeSnapshotsInternal(output)
if err != nil {
return err
}
- _, err = task.Wait(context.WithoutCancel(ctx))
+ existing := make(map[string]string, len(snapshots))
+ for _, snapshot := range snapshots {
+ existing[strings.ToLower(snapshot.ID)] = snapshot.ID
+ }
+ command := []string{"forget", "--prune"}
+ for _, id := range requested {
+ if listedID, found := existing[id]; found {
+ if len(command) == 2 {
+ command = append(command, "--")
+ }
+ command = append(command, listedID)
+ }
+ }
+ if len(command) == 2 {
+ command = []string{"prune"}
+ }
+ _, err = e.runContainerInternal(ctx, dockerClient, repository, password, command)
return err
}
@@ -467,8 +519,8 @@ func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *cli
// native `copy` would move only missing packs, but it addresses the target via
// a TOML config profile, which the env-only Repository cannot express yet —
// the materialize-and-rebackup here trades disk and I/O for that simplicity.
-func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string) (Snapshot, error) {
- temporaryVolume := "arcane-rustic-copy-" + uuid.NewString()
+func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string, tags ...string) (Snapshot, error) {
+ temporaryVolume := "arcane-rustic-copy-" + uuid.New().String()
if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: temporaryVolume, Labels: volumehelper.Labels()}); err != nil {
return Snapshot{}, fmt.Errorf("failed to create temporary Rustic copy volume: %w", err)
}
@@ -480,30 +532,13 @@ func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, fro
return Snapshot{}, fmt.Errorf("failed to load Rustic snapshot for replication: %w", err)
}
copyMount.ReadOnly = true
- snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, RootSnapshotInput(copyMount))
+ snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, RootSnapshotInput(copyMount, tags...))
if err != nil {
return Snapshot{}, fmt.Errorf("failed to replicate Rustic snapshot: %w", err)
}
return snapshot, nil
}
-func (e *Engine) executorForInternal(repositoryID string) (*actors.Executor, error) {
- if executor, ok := e.executors.Get(repositoryID); ok {
- return executor, nil
- }
- return e.executors.ApplyTyped(e.lifecycleCtx, "backup repository executor", func(values map[string]*actors.Executor) (*actors.Executor, bool, error) {
- if executor, ok := values[repositoryID]; ok {
- return executor, false, nil
- }
- executor, err := actors.NewExecutor(context.WithoutCancel(e.lifecycleCtx), e.runtime, "backup-repository", repositoryID, 3)
- if err != nil {
- return nil, false, err
- }
- values[repositoryID] = executor
- return executor, true, nil
- })
-}
-
func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
if e == nil {
return "", errors.New("backup engine is unavailable")
@@ -511,18 +546,11 @@ func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, r
if strings.TrimSpace(repository.ID) == "" {
return "", errors.New("backup repository ID is required")
}
- executor, err := e.executorForInternal(repository.ID) //nolint:contextcheck // Repository executors outlive requests so shutdown cleanup can finish.
- if err != nil {
- return "", err
- }
- task, err := executor.Submit(ctx, "rustic "+command[0], func(workCtx context.Context) (string, error) {
- return e.runContainerInternal(workCtx, dockerClient, repository, password, command, extraMounts...)
- }, nil)
- if err != nil {
+ defer e.lockRepositoryInternal(repository.ID)()
+ if err := ctx.Err(); err != nil {
return "", err
}
- // Cancellation must finish helper cleanup before callers release source locks.
- return task.Wait(context.WithoutCancel(ctx))
+ return e.runContainerInternal(ctx, dockerClient, repository, password, command, extraMounts...)
}
func (e *Engine) ensureImageInternal(ctx context.Context, dockerClient *client.Client) error {
@@ -642,3 +670,15 @@ func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {
}
return nil
}
+
+func (e *Engine) lockRepositoryInternal(id string) func() {
+ e.mu.Lock()
+ lock := e.repositories[id]
+ if lock == nil {
+ lock = &sync.Mutex{}
+ e.repositories[id] = lock
+ }
+ e.mu.Unlock()
+ lock.Lock()
+ return lock.Unlock
+}
diff --git a/backend/internal/backup/engine_test.go b/backend/internal/backup/engine_test.go
index dbb3fa59a..b14e306d8 100644
--- a/backend/internal/backup/engine_test.go
+++ b/backend/internal/backup/engine_test.go
@@ -115,9 +115,9 @@ func TestSnapshotCommandInternal(t *testing.T) {
require.NoError(t, err)
require.Equal(t, []string{"backup", "--init", "--json", "--host", "arcane", "--label", "volume", "--as-path", "/", "--", "/volume"}, single)
- multi, err := snapshotCommandInternal("arcane-system-recovery", CreateSnapshotInput{Sources: []string{"/data", "/projects"}})
+ multi, err := snapshotCommandInternal("arcane-system-recovery", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, Globs: []string{"!/data/arcane.db-wal"}})
require.NoError(t, err)
- require.Equal(t, []string{"backup", "--init", "--json", "--host", "arcane", "--label", "arcane-system-recovery", "--", "/data", "/projects"}, multi)
+ require.Equal(t, []string{"backup", "--init", "--json", "--host", "arcane", "--label", "arcane-system-recovery", "--glob", "!/data/arcane.db-wal", "--", "/data", "/projects"}, multi)
_, err = snapshotCommandInternal("x", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, AsPath: "/"})
require.ErrorContains(t, err, "single source")
diff --git a/backend/internal/backup/runs.go b/backend/internal/backup/runs.go
index 62ee554b1..9aab8997a 100644
--- a/backend/internal/backup/runs.go
+++ b/backend/internal/backup/runs.go
@@ -2,83 +2,308 @@ package backup
import (
"context"
- "log/slog"
+ "encoding/json/v2"
+ "sync"
+ "time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/host/local"
)
-type backupRunInternal struct {
- executor *actors.Executor
- cancel context.CancelFunc
+const backupRunTypeInternal = "backup-run"
+
+type backupRunActorInternal struct {
+ id string
+ engine *Engine
+ service *actor.Service
+}
+
+// RegisterRunKind binds a domain handler before the actor host starts.
+func (e *Engine) RegisterRunKind(kind string, execute func(context.Context, string, []byte, bool) error, failures ...func(context.Context, string, []byte, error) error) {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ e.handlers[kind] = execute
+ if len(failures) > 0 {
+ e.failures[kind] = failures[0]
+ }
+}
+
+func (e *Engine) SetAuthorize(authorize func(context.Context, backuptypes.DurableRunCommand) error) {
+ e.authorize = authorize
+}
+
+func (e *Engine) SetExecutionReady(ready func() bool) { e.executionReady = ready }
+
+func (e *Engine) Register(runtime *francis.Runtime) error {
+ e.service = runtime.Service()
+ return runtime.RegisterActor(backupRunTypeInternal, func(id string, service *actor.Service) actor.Actor {
+ return &backupRunActorInternal{id: id, engine: e, service: service}
+ }, local.WithCapacityGroup("jobs", 4), local.WithCompletedJobRetention(7*24*time.Hour))
+}
+
+// SubmitDurableRun persists the accepted command before dispatching its work.
+func (e *Engine) SubmitDurableRun(ctx context.Context, command backuptypes.DurableRunCommand, lease *runs.Lease) error {
+ if e == nil || e.service == nil {
+ return errors.New("backup actor host is unavailable")
+ }
+ if e.authorize != nil {
+ if err := e.authorize(ctx, command); err != nil {
+ return err
+ }
+ }
+ e.mu.Lock()
+ if e.stopping {
+ e.mu.Unlock()
+ return errors.New("backup engine is stopping")
+ }
+ e.leases[command.RunID] = lease
+ e.mu.Unlock()
+ _, err := e.service.Invoke(context.WithoutCancel(ctx), backupRunTypeInternal, command.RunID, "submit", command)
+ if err != nil {
+ e.mu.Lock()
+ delete(e.leases, command.RunID)
+ e.mu.Unlock()
+ }
+ return err
+}
+
+// AcquireDurableRun keeps the original admission, or reacquires it after a restart.
+func (e *Engine) AcquireDurableRun(ctx context.Context, runID, scope, resourceID string) (*runs.Lease, bool, error) {
+ e.mu.Lock()
+ lease := e.leases[runID]
+ delete(e.leases, runID)
+ e.mu.Unlock()
+ if lease != nil {
+ return lease, true, nil
+ }
+ return e.TryAcquireRun(ctx, scope, resourceID)
}
-// SubmitRun accepts one backup on an executor independent of repository work.
-// after owns finalization only when submission succeeds.
-func (e *Engine) SubmitRun(ctx context.Context, id string, work func(context.Context) error, after func(error)) error {
- if e == nil || e.runs == nil {
- return errors.New("backup engine is unavailable")
+func (a *backupRunActorInternal) Invoke(ctx context.Context, _ string, data actor.Envelope) (any, error) {
+ var command backuptypes.DurableRunCommand
+ if err := data.Decode(&command); err != nil {
+ return nil, err
}
- if err := ctx.Err(); err != nil {
+ var state backuptypes.DurableRunState
+ err := a.service.GetState(ctx, backupRunTypeInternal, a.id, &state)
+ if errors.Is(err, actor.ErrStateNotFound) {
+ state = backuptypes.DurableRunState{Command: command, Status: st.Queued}
+ if err := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); err != nil {
+ return nil, err
+ }
+ } else if err != nil {
+ return nil, err
+ }
+ if state.Status == st.Succeeded || state.Status == st.Failed || state.Status == st.NeedsAttention {
+ return nil, nil
+ }
+ _, _, err = a.service.Dispatch(ctx, backupRunTypeInternal, a.id, "execute", nil, actor.WithIdempotencyKey(a.id))
+ return nil, err
+}
+
+func (a *backupRunActorInternal) Job(ctx context.Context, _ string, _ actor.Envelope) error {
+ if a.engine.executionReady != nil && !a.engine.executionReady() {
+ return actor.ErrJobRejected
+ }
+ a.engine.mu.Lock()
+ if a.engine.stopping {
+ a.engine.mu.Unlock()
+ return actor.ErrJobRejected
+ }
+ a.engine.workers.Add(1)
+ a.engine.mu.Unlock()
+ defer a.engine.workers.Done()
+ jobCtx, cancel := context.WithCancel(ctx)
+ stop := context.AfterFunc(a.engine.lifecycleCtx, cancel) //nolint:contextcheck // Cancels the inherited job context when the engine stops.
+ defer stop()
+ defer cancel()
+ ctx = jobCtx
+
+ var state backuptypes.DurableRunState
+ if err := a.service.GetState(ctx, backupRunTypeInternal, a.id, &state); err != nil {
return err
}
- return e.runs.Apply(ctx, "submit backup run", func(runs map[string]*backupRunInternal) (bool, error) {
- if e.stopping {
- return false, errors.New("backup engine is stopping")
+ if state.Status == st.Succeeded || state.Status == st.Failed || state.Status == st.NeedsAttention {
+ return nil
+ }
+ a.engine.mu.Lock()
+ pendingLease := a.engine.leases[a.id]
+ a.engine.mu.Unlock()
+ defer pendingLease.Release(ctx)
+ interrupted := state.Started && len(state.Targets) > 0
+ state.Started = true
+ state.Status = st.Running
+ if err := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); err != nil {
+ return err
+ }
+ var progressMu sync.Mutex
+ ctx = jobcontext.WithExecution(ctx, st.Run{ID: a.id, EnvironmentID: "0", Outcome: st.Outcome{Targets: state.Targets}}, func(target st.TargetOutcome) error {
+ progressMu.Lock()
+ defer progressMu.Unlock()
+ updated := false
+ for i := range state.Targets {
+ if state.Targets[i].ID == target.ID {
+ if len(target.RecoveryData) == 0 {
+ target.RecoveryData = state.Targets[i].RecoveryData
+ }
+ state.Targets[i] = target
+ updated = true
+ break
+ }
}
- if _, exists := runs[id]; exists {
- return false, errors.New("backup run already submitted")
+ if !updated {
+ state.Targets = append(state.Targets, target)
}
- runCtx, cancel := context.WithCancel(ctx)
- // Keep the mailbox alive long enough to finalize cancelled work.
- executor, err := actors.NewExecutor(context.WithoutCancel(runCtx), e.runtime, "backup-runs", id, 0)
- if err != nil {
- cancel()
- return false, err
+ return a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil)
+ })
+ runErr := a.executeInternal(ctx, state.Command, interrupted)
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ state.Status = st.Succeeded
+ if runErr != nil {
+ state.Status = st.NeedsAttention
+ state.Error = runErr.Error()
+ }
+ if err := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); err != nil {
+ return err
+ }
+ if runErr != nil {
+ return errors.WrapIf(actor.ErrJobPermanentFailure, runErr.Error())
+ }
+ return nil
+}
+
+func (a *backupRunActorInternal) executeInternal(ctx context.Context, command backuptypes.DurableRunCommand, interrupted bool) (err error) {
+ defer utils.RecoverToError(&err, "durable backup")
+ a.engine.mu.Lock()
+ handler := a.engine.handlers[command.Kind]
+ failure := a.engine.failures[command.Kind]
+ a.engine.mu.Unlock()
+ if handler == nil {
+ return errors.New("backup run kind is unavailable")
+ }
+ if a.engine.authorize != nil {
+ if err := a.engine.authorize(ctx, command); err != nil {
+ a.engine.mu.Lock()
+ lease := a.engine.leases[a.id]
+ delete(a.engine.leases, a.id)
+ a.engine.mu.Unlock()
+ defer lease.Release(ctx)
+ if failure != nil {
+ err = errors.Combine(err, failure(ctx, a.id, command.Payload, err))
+ }
+ return err
}
- _, err = executor.Submit(runCtx, "run backup", func(workerCtx context.Context) (actors.NoPayload, error) {
- return actors.NoPayload{}, work(workerCtx)
- }, func(_ actors.NoPayload, runErr error) {
- defer func() {
- cancel()
- // Queue termination without waiting on our own mailbox.
- if stopErr := executor.Stop(runCtx); stopErr != nil && !errors.Is(stopErr, context.Canceled) {
- slog.ErrorContext(context.WithoutCancel(ctx), "Failed to stop backup executor", "runId", id, "error", stopErr)
- }
- if _, _, removeErr := e.runs.Remove(context.WithoutCancel(ctx), "complete backup run", id); removeErr != nil {
- slog.ErrorContext(context.WithoutCancel(ctx), "Failed to remove completed backup run", "runId", id, "error", removeErr)
- }
- }()
- after(runErr)
- })
+ }
+ return handler(ctx, a.id, command.Payload, interrupted)
+}
+
+func (e *Engine) activeRunsInternal(ctx context.Context) ([]backuptypes.DurableRunState, error) {
+ result := []backuptypes.DurableRunState{}
+ for cursor := ""; ; {
+ page, err := e.service.ListStates(ctx, backupRunTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
if err != nil {
- cancel()
- stopErr := executor.Stop(runCtx)
- if errors.Is(stopErr, context.Canceled) {
- stopErr = nil // The stop was queued; only its wait was cancelled.
+ return nil, err
+ }
+ for _, item := range page.States {
+ if item.Data == nil {
+ continue
+ }
+ var state backuptypes.DurableRunState
+ if err := item.Data.Decode(&state); err != nil {
+ return nil, err
+ }
+ if state.Status == st.Queued || state.Status == st.Running || state.Status == st.NeedsAttention {
+ result = append(result, state)
}
- return false, errors.Combine(err, stopErr)
}
- runs[id] = &backupRunInternal{executor: executor, cancel: cancel}
- return true, nil
- })
+ cursor = page.AfterID()
+ if cursor == "" {
+ break
+ }
+ }
+ return result, nil
}
-func (e *Engine) stopRunsInternal(ctx context.Context) error {
- var pending []*backupRunInternal
- if err := e.runs.Apply(ctx, "stop backup runs", func(runs map[string]*backupRunInternal) (bool, error) {
- e.stopping = true
- for _, run := range runs {
- pending = append(pending, run)
- run.cancel()
+func (e *Engine) ActiveRunIDs(ctx context.Context) ([]string, error) {
+ states, err := e.activeRunsInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ ids := make([]string, 0, len(states))
+ for _, state := range states {
+ ids = append(ids, state.Command.RunID)
+ for _, target := range state.Targets {
+ var checkpoint struct {
+ BackupID string `json:"backupId"`
+ }
+ if len(target.RecoveryData) > 0 && json.Unmarshal(target.RecoveryData, &checkpoint) == nil && checkpoint.BackupID != "" {
+ ids = append(ids, checkpoint.BackupID)
+ }
}
- return false, nil
- }); err != nil {
+ }
+ return ids, nil
+}
+
+func (e *Engine) ActiveActivityIDs(ctx context.Context) ([]string, error) {
+ states, err := e.activeRunsInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ ids := make([]string, 0, len(states))
+ for _, state := range states {
+ if state.Command.ActivityID != "" {
+ ids = append(ids, state.Command.ActivityID)
+ }
+ }
+ return ids, nil
+}
+
+// ReconcileDispatches repairs accepted commands whose dispatch was interrupted.
+func (e *Engine) ReconcileDispatches(ctx context.Context) error {
+ states, err := e.activeRunsInternal(ctx)
+ if err != nil {
return err
}
- var stopErr error
- for _, run := range pending {
- stopErr = errors.Combine(stopErr, run.executor.Stop(ctx))
+ for _, state := range states {
+ if state.Status == st.NeedsAttention {
+ continue
+ }
+ jobs, err := e.service.ListJobs(ctx, backupRunTypeInternal, state.Command.RunID)
+ if err != nil {
+ return err
+ }
+ live := false
+ for _, job := range jobs {
+ if !job.Status.IsTerminal() {
+ live = true
+ break
+ }
+ if job.Status == actor.JobStatusDeadLettered {
+ if _, err := e.service.RetryJob(ctx, job.JobID); err != nil {
+ return err
+ }
+ live = true
+ break
+ }
+ if err := e.service.DeleteJob(ctx, backupRunTypeInternal, state.Command.RunID, job.JobID); err != nil && !errors.Is(err, actor.ErrJobNotFound) {
+ return err
+ }
+ }
+ if live {
+ continue
+ }
+ if _, _, err := e.service.Dispatch(ctx, backupRunTypeInternal, state.Command.RunID, "execute", nil, actor.WithIdempotencyKey(state.Command.RunID)); err != nil {
+ return err
+ }
}
- return stopErr
+ return nil
}
diff --git a/backend/internal/backup/runs_test.go b/backend/internal/backup/runs_test.go
new file mode 100644
index 000000000..f9e0ced1f
--- /dev/null
+++ b/backend/internal/backup/runs_test.go
@@ -0,0 +1,119 @@
+package backup
+
+import (
+ "context"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
+ bt "github.com/getarcaneapp/arcane/types/v2/backup"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/stretchr/testify/require"
+)
+
+func TestDurableBackupRestartKeepsCompletionInternal(t *testing.T) {
+ url := "file:" + filepath.Join(t.TempDir(), "actors.db")
+ var calls atomic.Int32
+ runtime := francistest.New(t, url)
+ engine := NewEngine(t.Context(), nil, nil)
+ engine.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, interrupted bool) error {
+ calls.Add(1)
+ require.False(t, interrupted)
+ return jobcontext.Progress(ctx, st.TargetOutcome{ID: "snapshot", Status: st.Succeeded})
+ })
+ require.NoError(t, engine.Register(runtime))
+ francistest.Start(t, runtime)
+ command := bt.DurableRunCommand{Kind: "test", RunID: "run", ActivityID: "activity"}
+ require.NoError(t, engine.SubmitDurableRun(t.Context(), command, nil))
+ require.Eventually(t, func() bool {
+ states, err := engine.activeRunsInternal(t.Context())
+ return err == nil && len(states) == 0 && calls.Load() == 1
+ }, 10*time.Second, 20*time.Millisecond)
+ stopCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+ defer cancel()
+ require.NoError(t, runtime.Stop(stopCtx))
+ restarted := francistest.New(t, url)
+ next := NewEngine(t.Context(), nil, nil)
+ next.RegisterRunKind("test", func(context.Context, string, []byte, bool) error { calls.Add(1); return nil })
+ require.NoError(t, next.Register(restarted))
+ francistest.Start(t, restarted)
+ require.NoError(t, next.SubmitDurableRun(t.Context(), command, nil))
+ require.NoError(t, next.ReconcileDispatches(t.Context()))
+ require.EqualValues(t, 1, calls.Load())
+}
+
+func TestDurableBackupShutdownKeepsInterruptedEvidenceInternal(t *testing.T) {
+ url := "file:" + filepath.Join(t.TempDir(), "actors.db")
+ runtime := francistest.New(t, url)
+ admission := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, admission.Register(runtime))
+ engine := NewEngine(t.Context(), admission, nil)
+ entered := make(chan struct{})
+ drain := make(chan struct{})
+ t.Cleanup(func() {
+ select {
+ case <-drain:
+ default:
+ close(drain)
+ }
+ })
+ engine.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, _ bool) error {
+ if err := jobcontext.Progress(ctx, st.TargetOutcome{ID: "snapshot", Status: st.Running, RecoveryData: []byte(`{"backupId":"frozen"}`)}); err != nil {
+ return err
+ }
+ close(entered)
+ <-ctx.Done()
+ <-drain
+ return ctx.Err()
+ })
+ require.NoError(t, engine.Register(runtime))
+ francistest.Start(t, runtime)
+ lease, acquired, err := engine.TryAcquireRun(t.Context(), "backup", "data")
+ require.NoError(t, err)
+ require.True(t, acquired)
+ require.NoError(t, engine.SubmitDurableRun(t.Context(), bt.DurableRunCommand{Kind: "test", RunID: "interrupted"}, lease))
+ select {
+ case <-entered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("backup did not start")
+ }
+ stopCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+ defer cancel()
+ stopped := make(chan error, 1)
+ go func() { stopped <- engine.Stop(stopCtx) }()
+ require.Eventually(t, func() bool { return engine.lifecycleCtx.Err() != nil }, time.Second, time.Millisecond)
+ _, acquired, err = engine.TryAcquireRun(t.Context(), "backup", "data")
+ require.NoError(t, err)
+ require.False(t, acquired)
+ close(drain)
+ require.NoError(t, <-stopped)
+ lease, acquired, err = engine.TryAcquireRun(t.Context(), "backup", "data")
+ require.NoError(t, err)
+ require.True(t, acquired)
+ lease.Release(t.Context())
+ require.NoError(t, runtime.Stop(stopCtx))
+ restarted := francistest.New(t, url)
+ next := NewEngine(t.Context(), nil, nil)
+ resumed := make(chan struct{})
+ next.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, interrupted bool) error {
+ require.True(t, interrupted)
+ previous, ok := jobcontext.Run(ctx)
+ require.True(t, ok)
+ require.Len(t, previous.Outcome.Targets, 1)
+ require.Equal(t, `{"backupId":"frozen"}`, string(previous.Outcome.Targets[0].RecoveryData))
+ close(resumed)
+ return nil
+ })
+ require.NoError(t, next.Register(restarted))
+ francistest.Start(t, restarted)
+ require.NoError(t, next.ReconcileDispatches(t.Context()))
+ select {
+ case <-resumed:
+ case <-time.After(15 * time.Second):
+ t.Fatal("durable backup did not resume")
+ }
+}
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index fb6de4aba..00477c9b2 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -12,7 +12,6 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/api/ws"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -37,7 +36,7 @@ import (
"github.com/moby/moby/client"
"github.com/subosito/gotenv"
"go.getarcane.app/streams/logs"
- libcrypto "go.getarcane.app/sys/crypto"
+ "go.getarcane.app/sys/crypto"
"go.uber.org/fx"
"go.uber.org/fx/fxevent"
)
@@ -61,8 +60,7 @@ func Bootstrap(ctx context.Context) error {
cfg.DockerConfig = runtimeIdentityCfg.DockerConfig
SetupSlogLogger(cfg)
- // Tee all slog output into the in-memory ring buffer that powers the
- // diagnostics live log tail.
+ // Tee all slog output into the in-memory ring buffer that powers the diagnostics live log tail.
slog.SetDefault(slog.New(logs.NewSlogHandler(slog.Default().Handler(), ws.LogBroadcaster())))
database.SetGormLogger(BuildGormLogger(cfg))
slog.InfoContext(ctx, "Arcane is starting...", "version", config.Version)
@@ -137,7 +135,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
// isWeakProductionEncryptionKeyInternal reports whether an explicit
// ENCRYPTION_KEY is an unprefixed passphrase shorter than 32 characters in
-// production. libcrypto derives a key from any non-empty passphrase, so this
+// production. crypto derives a key from any non-empty passphrase, so this
// preserves the historical fail-fast rejection of low-entropy production keys.
func isWeakProductionEncryptionKeyInternal(encryptionKey, environment string, agentMode bool) bool {
if environment != "production" || agentMode {
@@ -175,7 +173,7 @@ type initializeStartupStateParams struct {
Config *config.Config
HTTPClient *http.Client
- Volume *volume.Module
+ Volume *volume.VolumeService
Settings *settings.SettingsService
Environment *environment.EnvironmentService
GitOpsSync *gitops.GitOpsSyncService
@@ -194,8 +192,8 @@ func initializeStartupState(p initializeStartupStateParams) {
cfg := p.Config
httpClient := p.HTTPClient
- if p.Volume.Service() != nil {
- startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.Service().CleanupOrphanedVolumeHelpers)
+ if p.Volume != nil {
+ startup.CleanupOrphanedVolumeHelpers(appCtx, p.Volume.CleanupOrphanedVolumeHelpers)
}
runtimeCfg := &startup.RuntimeConfig{
@@ -218,7 +216,7 @@ func initializeStartupState(p initializeStartupStateParams) {
panic("ENCRYPTION_KEY passphrase must be at least 32 characters in production (or use a hex:/base64: encoded 32-byte key)")
}
- libcrypto.InitEncryption(&libcrypto.Config{
+ crypto.InitEncryption(&crypto.Config{
EncryptionKey: cfg.EncryptionKey,
Environment: string(cfg.Environment),
AgentMode: cfg.AgentMode,
@@ -325,11 +323,7 @@ func initializeGitOpsStartupStateInternal(appCtx context.Context, gitOpsSync *gi
if err := gitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)
}
- if err := gitOpsSync.ReconcileInterruptedBackupsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to reconcile interrupted Git backups on startup", "error", err)
- }
- // Sweep leaked gitops scratch dirs before the filesystem watcher can import
- // them as phantom projects and before the directory-sync reconcile runs.
+
if err := gitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)
}
@@ -373,12 +367,12 @@ func runRoleStartupTasks(ctx context.Context, roleService *role.RoleService, api
}
}
-func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, cfg *config.Config, router *echo.Echo, _ *http.Server) {
+func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, router *echo.Echo, _ *http.Server) {
var stop func(context.Context) error
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
var startErr error
- stop, startErr = startEdgeTunnelClientIfConfigured(appCtx, actorRuntime, cfg, router)
+ stop, startErr = startEdgeTunnelClientIfConfigured(appCtx, cfg, router)
if startErr != nil {
slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", startErr)
}
@@ -402,7 +396,7 @@ func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelFunc) {
})
}
-func startEdgeTunnelClientIfConfigured(appCtx context.Context, actorRuntime *actors.Runtime, cfg *config.Config, router http.Handler) (func(context.Context) error, error) {
+func startEdgeTunnelClientIfConfigured(appCtx context.Context, cfg *config.Config, router http.Handler) (func(context.Context) error, error) {
managerEndpointConfigured := cfg.ManagerApiUrl != ""
if !cfg.EdgeAgent || !managerEndpointConfigured || cfg.AgentToken == "" {
return nil, nil
@@ -426,7 +420,7 @@ func startEdgeTunnelClientIfConfigured(appCtx context.Context, actorRuntime *act
}
slog.InfoContext(appCtx, "Starting edge agent session client", edge.StartupLogAttrs(edgeCfg)...)
- stop, err := edge.StartTunnelClient(appCtx, actorRuntime, edgeCfg, router)
+ stop, err := edge.StartTunnelClient(appCtx, edgeCfg, router)
if err != nil {
return nil, errors.WrapIf(err, "failed to start edge tunnel client")
}
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 6ab121e22..77c553557 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -12,7 +12,6 @@ import (
"time"
"github.com/getarcaneapp/arcane/backend/v2/api"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
@@ -23,6 +22,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/labstack/echo/v5"
@@ -193,10 +194,10 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {
Context: context.Background(),
Config: cfg,
HandlerDeps: api.HandlerDeps{
- Project: &project.Module{Service: &project.ProjectService{}},
- Container: &container.Module{Service: &container.ContainerService{}},
- Swarm: &swarm.Module{Service: &swarm.SwarmService{}},
- System: &system.Module{Service: &system.SystemService{}},
+ Project: project.New(&project.ProjectService{}, nil),
+ Container: container.New(&container.ContainerService{}, nil, nil, nil),
+ Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),
+ System: system.New(&system.SystemService{}, nil, nil, nil, nil, nil),
},
AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),
TunnelRegistry: edge.NewTunnelRegistry(),
@@ -392,8 +393,7 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
appCtx := t.Context()
lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(appCtx, lifecycle)
- require.NoError(t, err)
+ runtime := francistest.New(t)
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))
@@ -401,8 +401,11 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
t.Cleanup(func() { _ = sqlDB.Close() })
- jobService := job.NewJobService(&database.DB{DB: db}, nil, &config.Config{})
- jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, runtime, nil, nil, nil, nil, jobService)
+ coordinator := runs.New(kv.NewKVService(&database.DB{DB: db}), runtime.Service(), time.UTC)
+ require.NoError(t, coordinator.Register(runtime))
+ francistest.Start(t, runtime)
+ jobService := job.NewJobService(&database.DB{DB: db}, nil, &config.Config{}, coordinator, nil, nil, nil)
+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, nil, nil, nil, nil, jobService, nil, nil, nil, nil, nil)
require.NoError(t, err)
watcher := &blockingBusWatcherInternal{
started: make(chan struct{}),
diff --git a/backend/internal/bootstrap/edge_bootstrap.go b/backend/internal/bootstrap/edge_bootstrap.go
index e3fd15207..41155971f 100644
--- a/backend/internal/bootstrap/edge_bootstrap.go
+++ b/backend/internal/bootstrap/edge_bootstrap.go
@@ -7,7 +7,6 @@ import (
"log/slog"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
@@ -15,6 +14,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
notificationdto "github.com/getarcaneapp/arcane/types/v2/notification"
"github.com/labstack/echo/v5"
kit "go.getarcane.app/kit/pkg"
@@ -27,7 +27,6 @@ import (
func registerEdgeTunnelRoutes(
ctx context.Context,
lifecycle fx.Lifecycle,
- actorRuntime *actors.Runtime,
cfg *config.Config,
apiGroup *echo.Group,
environmentService *environment.EnvironmentService,
@@ -135,18 +134,21 @@ func registerEdgeTunnelRoutes(
})
createEdgeMTLSIssueEventsInternal(ctx, eventService, envIDCopy, envNameCopy, remoteAddr, certIssued, caGenerated, reenrolled)
})
- var cleanupRunner *actors.Runner
+ var stopCleanup func(context.Context) error
lifecycle.Append(fx.Hook{
OnStart: func(context.Context) error {
var err error
- cleanupRunner, err = actors.NewRunner(ctx, actorRuntime, "edge-tunnel", "cleanup", "edge tunnel cleanup", 3, func(runCtx context.Context) error {
+ stopCleanup, err = concurrency.StartSupervised(ctx, "edge tunnel cleanup", func(runCtx context.Context) error {
server.StartCleanupLoop(runCtx)
return nil
})
return err
},
OnStop: func(stopCtx context.Context) error {
- return cleanupRunner.Stop(stopCtx)
+ if stopCleanup == nil {
+ return nil
+ }
+ return stopCleanup(stopCtx)
},
})
apiGroup.POST("/tunnel/poll", server.HandlePoll)
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 5993064fc..b1be64dc0 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -6,11 +6,13 @@ import (
"log/slog"
"net/http"
"slices"
+ "sync"
+ "time"
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
@@ -18,27 +20,37 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
"github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/updater"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/types/v2/features"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"go.uber.org/fx"
)
-func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *actors.Runtime, _ *actors.Gate[actors.AdmissionKey], imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *system.SystemUpgradeService, jobService *job.JobService) (schedulertypes.JobScheduler, error) {
+func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *runs.Coordinator, _ *runs.Admission, imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *system.SystemUpgradeService, jobService *job.JobService, backupEngine *backup.Engine, updaterService *updater.UpdaterService, volumes *volume.VolumeService, systemBackups *systembackup.SystemBackupService, gitopsSync *gitops.GitOpsSyncService) (schedulertypes.JobScheduler, error) {
schedulerCtx, cancelScheduler := context.WithCancel(appCtx)
- jobScheduler, err := scheduler.NewJobScheduler(schedulerCtx, runtime, cfg.GetLocation())
+ jobScheduler, err := scheduler.NewJobScheduler(schedulerCtx, jobService.Coordinator(), cfg.GetLocation())
if err != nil {
cancelScheduler()
return nil, err
}
jobService.SetScheduler(schedulerCtx, jobScheduler)
+ var repairDone chan struct{}
+ var resumeDone chan struct{}
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
slog.InfoContext(appCtx, "Starting scheduler")
+ if err := jobService.Coordinator().Start(schedulerCtx); err != nil {
+ return err
+ }
+ if err := reconcileCoordinatorStartupInternal(schedulerCtx, jobService, backupEngine, updaterService, volumes, systemBackups, gitopsSync); err != nil {
+ return err
+ }
if imageUpdateWatcher != nil {
- imageUpdateWatcher.SetDispatcher(jobService.Queue)
+ imageUpdateWatcher.SetCoordinator(jobService.Coordinator())
if err := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); err != nil {
return err
}
@@ -46,41 +58,159 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
if err := jobScheduler.StartScheduler(); err != nil {
return err
}
- if err := jobService.Queue.Start(schedulerCtx); err != nil {
- return err
+ jobService.Coordinator().Activate()
+ if backupEngine != nil {
+ repairDone = make(chan struct{})
+ go func() {
+ defer close(repairDone)
+ repairBackupDispatchesInternal(schedulerCtx, backupEngine)
+ }()
}
if analytics != nil {
- if _, err := jobService.Queue.Submit(schedulerCtx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {
+ if _, err := jobService.Coordinator().Submit(schedulerCtx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {
return err
}
}
if !cfg.AgentMode && systemUpgrade != nil {
- go systemUpgrade.ResumeUpdateAllOnStartup(schedulerCtx)
+ resumeDone = make(chan struct{})
+ go func() { defer close(resumeDone); systemUpgrade.ResumeUpdateAllOnStartup(schedulerCtx) }()
}
return nil
},
OnStop: func(ctx context.Context) error {
- cancelScheduler()
- err := errors.Combine(jobService.Queue.Stop(ctx), jobScheduler.Stop(ctx))
- if err != nil {
- slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)
- return err
- }
- slog.InfoContext(ctx, "Scheduler stopped")
- return nil
+ return stopJobSchedulerInternal(ctx, cancelScheduler, []chan struct{}{repairDone, resumeDone}, jobService.Coordinator(), jobScheduler)
},
})
return jobScheduler, nil
}
+func stopJobSchedulerInternal(ctx context.Context, cancelScheduler context.CancelFunc, workers []chan struct{}, coordinator *runs.Coordinator, jobScheduler schedulertypes.JobScheduler) error {
+ cancelScheduler()
+ for _, done := range workers {
+ if done != nil {
+ select {
+ case <-done:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ }
+ err := errors.Combine(coordinator.Stop(ctx), jobScheduler.Stop(ctx))
+ if err != nil {
+ slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)
+ return err
+ }
+ slog.InfoContext(ctx, "Scheduler stopped")
+ return nil
+}
+
+func reconcileCoordinatorStartupInternal(ctx context.Context, jobs *job.JobService, engine *backup.Engine, updates *updater.UpdaterService, volumes *volume.VolumeService, systemBackups *systembackup.SystemBackupService, syncService *gitops.GitOpsSyncService) error {
+ activities, backups, err := startupProtectionInternal(ctx, jobs, engine, updates)
+ if err != nil {
+ return err
+ }
+ if volumes != nil {
+ if err := volumes.ReconcileInterruptedBackups(ctx, backups...); err != nil {
+ return err
+ }
+ }
+ if systemBackups != nil {
+ if err := systemBackups.ReconcileInterruptedBackups(ctx, backups...); err != nil {
+ return err
+ }
+ }
+ if syncService != nil {
+ if err := syncService.ReconcileInterruptedBackupsOnStartup(ctx, backups...); err != nil {
+ return err
+ }
+ }
+ return jobs.ReconcileStartupActivities(ctx, activities...)
+}
+
+func startupProtectionInternal(ctx context.Context, jobs *job.JobService, engine *backup.Engine, updates *updater.UpdaterService) ([]string, []string, error) {
+ records, err := jobs.Coordinator().Records(ctx)
+ if err != nil {
+ return nil, nil, err
+ }
+ scheduled, err := scheduledBackupProtectionInternal(records)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ activities := []string{}
+ backups := make([]string, 0, len(scheduled))
+ if engine != nil {
+ var err error
+ activities, err = engine.ActiveActivityIDs(ctx)
+ if err != nil {
+ return nil, nil, err
+ }
+ backups, err = engine.ActiveRunIDs(ctx)
+ if err != nil {
+ return nil, nil, err
+ }
+ if err := engine.ReconcileDispatches(ctx); err != nil {
+ return nil, nil, err
+ }
+ }
+ if updates != nil {
+ ids, err := updates.ActiveUpdateActivityIDs(ctx)
+ if err != nil {
+ return nil, nil, err
+ }
+ activities = append(activities, ids...)
+ }
+ return activities, append(backups, scheduled...), nil
+}
+
+func scheduledBackupProtectionInternal(records []schedulertypes.QueueRecord) ([]string, error) {
+ ids := []string{}
+ for _, record := range records {
+ for _, run := range record.Runs {
+ if run.Status.Terminal() {
+ continue
+ }
+ for _, target := range run.Outcome.Targets {
+ if len(target.RecoveryData) == 0 {
+ continue
+ }
+ var checkpoint struct {
+ BackupID string `json:"backupId"`
+ }
+ if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
+ return nil, errors.WrapIf(err, "decode startup recovery evidence")
+ }
+ if checkpoint.BackupID != "" {
+ ids = append(ids, checkpoint.BackupID)
+ }
+ }
+ }
+ }
+ return ids, nil
+}
+
+func repairBackupDispatchesInternal(ctx context.Context, engine *backup.Engine) {
+ timer := time.NewTicker(15 * time.Second)
+ defer timer.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-timer.C:
+ if err := engine.ReconcileDispatches(ctx); err != nil && ctx.Err() == nil {
+ slog.ErrorContext(ctx, "Backup dispatch reconciliation failed", "error", err)
+ }
+ }
+ }
+}
+
type registerJobsParams struct {
fx.In
- Lifecycle fx.Lifecycle
- AppCtx context.Context
- Config *config.Config
- Scheduler schedulertypes.JobScheduler
- ActorRuntime *actors.Runtime
+ Lifecycle fx.Lifecycle
+ AppCtx context.Context
+ Config *config.Config
+ Scheduler schedulertypes.JobScheduler
Activity *activity.ActivityService
GitOpsSync *gitops.GitOpsSyncService
@@ -89,7 +219,7 @@ type registerJobsParams struct {
Settings *settings.SettingsService
Volume *volume.VolumeService
SystemBackup *systembackup.SystemBackupService
- Admission *actors.Gate[actors.AdmissionKey]
+ Admission *runs.Admission
Apns *apns.ApnsService
AutoUpdate *scheduler.AutoUpdateJob
@@ -115,33 +245,6 @@ type registerJobsParams struct {
func registerJobs(params registerJobsParams) error {
params.JobSchedule.SetScheduler(params.AppCtx, params.Scheduler)
- // Bootstrap owns registration, agent-mode gating, and settings callbacks.
- if params.Activity != nil {
- if err := params.Activity.FailInterruptedBackups(params.AppCtx); err != nil {
- return err
- }
- failed, err := params.Activity.FailStaleImageUpdateChecks(params.AppCtx)
- if err != nil {
- slog.WarnContext(params.AppCtx, "Failed to mark stale image update checks as failed", "count", failed, "error", err)
- } else if failed > 0 {
- slog.InfoContext(params.AppCtx, "Marked stale image update checks as failed", "count", failed)
- }
-
- resolved, err := params.Activity.ResolveStaleAutoUpdateActivities(params.AppCtx)
- if err != nil {
- slog.WarnContext(params.AppCtx, "Failed to resolve stale auto-update activities", "count", resolved, "error", err)
- } else if resolved > 0 {
- slog.InfoContext(params.AppCtx, "Resolved stale auto-update activities", "count", resolved)
- }
-
- orphaned, err := params.Activity.ResolveOrphanedQueuedActivities(params.AppCtx)
- if err != nil {
- slog.WarnContext(params.AppCtx, "Failed to resolve orphaned queued activities", "count", orphaned, "error", err)
- } else if orphaned > 0 {
- slog.InfoContext(params.AppCtx, "Resolved orphaned queued activities", "count", orphaned)
- }
- }
-
for _, job := range []schedulertypes.Job{
params.AutoUpdate,
params.DockerClientRefresh,
@@ -190,7 +293,6 @@ func registerJobs(params registerJobsParams) error {
LifecycleCtx: params.AppCtx,
Config: params.Config,
Scheduler: params.Scheduler,
- ActorRuntime: params.ActorRuntime,
Settings: params.Settings,
Environment: params.Environment,
AutoUpdate: params.AutoUpdate,
@@ -218,7 +320,7 @@ type dynamicJobsParams struct {
JobSchedule *job.JobService
Volume *volume.VolumeService
SystemBackup *systembackup.SystemBackupService
- Admission *actors.Gate[actors.AdmissionKey]
+ Admission *runs.Admission
}
// registerDynamicJobs injects the scheduler into the services that own per-entity
@@ -229,18 +331,12 @@ func registerDynamicJobs(params dynamicJobsParams) error {
// registered on managers and agents; environment proxying persists each policy
// in the correct Arcane database.
if params.Volume != nil {
- if err := params.Volume.ReconcileInterruptedBackups(params.AppCtx); err != nil {
- return err
- }
if err := params.Volume.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
return err
}
params.Volume.RegisterBackupJobsOnStartup(params.AppCtx)
}
if !params.Config.AgentMode && params.SystemBackup != nil {
- if err := params.SystemBackup.ReconcileInterruptedBackups(params.AppCtx); err != nil {
- return err
- }
if err := params.SystemBackup.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
return err
}
@@ -278,7 +374,6 @@ type settingsSubscriptionsParams struct {
LifecycleCtx context.Context
Config *config.Config
Scheduler settingsEffectsSchedulerInternal
- ActorRuntime *actors.Runtime
Settings settingsChangeSubscriberInternal
Environment timeoutSettingsEnvironmentInternal
@@ -319,10 +414,7 @@ func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) erro
}
})
}
- timeoutSyncExecutor, cancelTimeoutSync, err := setupTimeoutSettingsSubscriptionInternal(params, subscribe)
- if err != nil {
- return err
- }
+ timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscriptionInternal(params, subscribe)
subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {
if params.ImageUpdateWatcher != nil {
@@ -381,26 +473,72 @@ func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) erro
return nil
}
-func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*actors.Executor, context.CancelFunc, error) {
- if params.Config.AgentMode {
- return nil, nil, nil
+type timeoutSyncWorkerInternal struct {
+ mu sync.Mutex
+ updates [][]libarcane.SettingUpdate
+ wake chan struct{}
+ done chan struct{}
+ ctx context.Context
+ cancel context.CancelFunc
+ closed bool
+}
+
+func (w *timeoutSyncWorkerInternal) Stop(ctx context.Context) error {
+ if w == nil {
+ return nil
+ }
+ w.mu.Lock()
+ w.closed = true
+ w.cancel()
+ w.mu.Unlock()
+ select {
+ case <-w.done:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
}
+}
- timeoutSyncExecutor, err := actors.NewExecutor(params.LifecycleCtx, params.ActorRuntime, "services", "settings-timeout-sync", 3)
- if err != nil {
- return nil, nil, err
+func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*timeoutSyncWorkerInternal, context.CancelFunc) {
+ if params.Config.AgentMode {
+ return nil, nil
}
- timeoutSyncContext, cancelTimeoutSync := context.WithCancel(params.LifecycleCtx)
+ ctx, cancel := context.WithCancel(params.LifecycleCtx)
+ worker := &timeoutSyncWorkerInternal{wake: make(chan struct{}, 1), done: make(chan struct{}), ctx: ctx, cancel: cancel}
+ go func() {
+ defer close(worker.done)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-worker.wake:
+ }
+ for {
+ worker.mu.Lock()
+ if len(worker.updates) == 0 {
+ worker.mu.Unlock()
+ break
+ }
+ updates := worker.updates[0]
+ worker.updates = worker.updates[1:]
+ worker.mu.Unlock()
+ syncTimeoutSettingsToAgentsInternal(ctx, params.Environment, updates)
+ }
+ }
+ }()
subscribe(libarcane.TimeoutSettingKeys(), func(updates []libarcane.SettingUpdate) {
- _, err := timeoutSyncExecutor.Submit(timeoutSyncContext, "sync timeout settings to remote environments", func(ctx context.Context) (actors.NoPayload, error) {
- syncTimeoutSettingsToAgentsInternal(ctx, params.Environment, updates)
- return actors.NoPayload{}, nil
- }, nil)
- if err != nil && timeoutSyncContext.Err() == nil {
- slog.ErrorContext(timeoutSyncContext, "Failed to queue timeout settings sync", "error", err)
+ worker.mu.Lock()
+ defer worker.mu.Unlock()
+ if worker.closed {
+ return
+ }
+ worker.updates = append(worker.updates, slices.Clone(updates))
+ select {
+ case worker.wake <- struct{}{}:
+ default:
}
})
- return timeoutSyncExecutor, cancelTimeoutSync, nil
+ return worker, cancel
}
// syncTimeoutSettingsToAgentsInternal syncs timeout settings to all connected remote environments
diff --git a/backend/internal/bootstrap/jobs_bootstrap_test.go b/backend/internal/bootstrap/jobs_bootstrap_test.go
index 669f3b116..a4ed99212 100644
--- a/backend/internal/bootstrap/jobs_bootstrap_test.go
+++ b/backend/internal/bootstrap/jobs_bootstrap_test.go
@@ -6,7 +6,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
@@ -76,8 +75,6 @@ func (s *timeoutSyncEnvironmentStubInternal) ProxyRequest(ctx context.Context, _
func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {
lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(context.Background(), lifecycle)
- require.NoError(t, err)
settings := &settingsSubscriptionStubInternal{}
scheduler := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}
@@ -87,7 +84,6 @@ func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {
LifecycleCtx: context.Background(),
Config: &config.Config{},
Scheduler: scheduler,
- ActorRuntime: runtime,
Settings: settings,
Environment: environment,
}))
@@ -123,8 +119,6 @@ func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {
func TestFeatureChangeReschedulesScanAndPatchJobsInternal(t *testing.T) {
lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
settings := &settingsSubscriptionStubInternal{}
schedulerStub := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}
require.NoError(t, setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{
@@ -132,7 +126,6 @@ func TestFeatureChangeReschedulesScanAndPatchJobsInternal(t *testing.T) {
LifecycleCtx: t.Context(),
Config: &config.Config{},
Scheduler: schedulerStub,
- ActorRuntime: runtime,
Settings: settings,
VulnerabilityScan: scheduler.NewVulnerabilityScanJob(nil, nil),
VulnerabilityRisk: scheduler.NewVulnerabilityRiskJob(nil, nil),
diff --git a/backend/internal/bootstrap/playwright_router_bootstrap.go b/backend/internal/bootstrap/playwright_router_bootstrap.go
index fdf70e90b..57bb76eb8 100644
--- a/backend/internal/bootstrap/playwright_router_bootstrap.go
+++ b/backend/internal/bootstrap/playwright_router_bootstrap.go
@@ -20,7 +20,7 @@ func init() {
return
}
- playwrightService := playwright.NewPlaywrightService(apiKeyService, userService, deps.GitRepository.Service(), deps.GitOpsSync.Service(), deps.Project.Service)
+ playwrightService := playwright.NewPlaywrightService(apiKeyService, userService, deps.GitRepository.Service(), deps.GitOpsSync.Service(), deps.Project.Service())
playwright.SetupRoutes(apiGroup, playwrightService, deps.Federated)
slog.Info("Playwright routes registered for E2E testing")
},
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 7a25634d6..b58727b2b 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -12,7 +12,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/api"
"github.com/getarcaneapp/arcane/backend/v2/api/ws"
"github.com/getarcaneapp/arcane/backend/v2/frontend"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -161,7 +160,6 @@ type RouterParams struct {
Context context.Context
Lifecycle fx.Lifecycle
- ActorRuntime *actors.Runtime
Config *config.Config
HandlerDeps api.HandlerDeps
AuthMiddleware *auth.AuthMiddleware
@@ -291,13 +289,13 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
}
// Remaining echo handlers (WebSocket/streaming)
- ws.NewWebSocketHandler(apiGroup, deps.Project.Service, deps.Container.Service, deps.Swarm.Service, deps.System.Service, deps.Diagnostics, authMiddleware, cfg)
+ ws.NewWebSocketHandler(apiGroup, deps.Project.Service(), deps.Container.Service(), deps.Swarm.Service(), deps.System.Service(), deps.Diagnostics, authMiddleware, cfg)
// Register edge tunnel endpoint for manager to accept agent connections
// This is only registered when NOT in agent mode (i.e., running as manager)
var tunnelServer *edge.TunnelServer
if !cfg.AgentMode {
- tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, p.ActorRuntime, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), deps.Notification.Service(), p.TunnelRegistry)
+ tunnelServer = registerEdgeTunnelRoutes(ctx, p.Lifecycle, cfg, apiGroup, deps.Environment.Service(), deps.Event.Service(), deps.Notification.Service(), p.TunnelRegistry)
}
if cfg.Environment != "production" {
diff --git a/backend/internal/bootstrap/server_bootstrap.go b/backend/internal/bootstrap/server_bootstrap.go
index 2f12d82fb..e10080016 100644
--- a/backend/internal/bootstrap/server_bootstrap.go
+++ b/backend/internal/bootstrap/server_bootstrap.go
@@ -14,6 +14,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/labstack/echo/v5"
"go.uber.org/fx"
"google.golang.org/grpc"
@@ -34,6 +35,7 @@ type HTTPServerParams struct {
Config *config.Config
Router *echo.Echo
TunnelServer *edge.TunnelServer
+ Scheduler schedulertypes.JobScheduler
}
// NewHTTPServer builds the shared HTTP/gRPC server and registers its lifecycle.
diff --git a/backend/internal/build/workspace_service_test.go b/backend/internal/build/workspace_service_test.go
index de8c26f67..8b17d8054 100644
--- a/backend/internal/build/workspace_service_test.go
+++ b/backend/internal/build/workspace_service_test.go
@@ -5,14 +5,11 @@ import (
"os"
"path/filepath"
"testing"
- "time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -44,19 +41,9 @@ func setupSettingsTestDB(t *testing.T) *database.DB {
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go
index 678a0b05b..7ca63f608 100644
--- a/backend/internal/config/config.go
+++ b/backend/internal/config/config.go
@@ -30,7 +30,6 @@ const (
// Fields with `options:"file"` support Docker secrets via the _FILE suffix.
// Available options: file, toLower, trimTrailingSlash, deprecated
type Config struct {
- // BuildablesConfig contains feature-specific configuration that can be conditionally compiled
BuildablesConfig
AppUrl string `env:"APP_URL" default:"http://localhost:3552"`
@@ -38,6 +37,7 @@ type Config struct {
AllowDowngrade bool `env:"ALLOW_DOWNGRADE" default:"false"`
AllowCLIPasswordReset bool `env:"ALLOW_CLI_PASSWORD_RESET" default:"false"`
AllowCLIMFAReset bool `env:"ALLOW_CLI_MFA_RESET" default:"false"`
+ ActorPort string `env:"ACTOR_PORT" default:"3551"`
Port string `env:"PORT" default:"3552"`
Listen string `env:"LISTEN" default:""`
TLSEnabled bool `env:"TLS_ENABLED" default:"false"`
@@ -62,11 +62,7 @@ type Config struct {
OidcProviderName string `env:"OIDC_PROVIDER_NAME" default:""`
OidcProviderLogoUrl string `env:"OIDC_PROVIDER_LOGO_URL" default:""`
OidcMobileRedirectUris string `env:"OIDC_MOBILE_REDIRECT_URIS" default:"arcane-mobile://oidc-callback"`
- // OidcRoleMappings declaratively defines OIDC group→role mappings as a
- // JSON array of role.OidcRoleMappingSpec. Reconciled into source='env'
- // rows on every boot; rows are read-only at runtime. Supports *_FILE for
- // Docker secrets. Leave empty to manage mappings purely via the UI/API.
- OidcRoleMappings string `env:"OIDC_ROLE_MAPPINGS" default:"" options:"file"`
+ OidcRoleMappings string `env:"OIDC_ROLE_MAPPINGS" default:"" options:"file"`
PUID string `env:"PUID" default:""`
PGID string `env:"PGID" default:""`
@@ -102,23 +98,16 @@ type Config struct {
EdgeMTLSServerName string `env:"EDGE_MTLS_SERVER_NAME" default:""`
EdgeMTLSAssetsDir string `env:"EDGE_MTLS_ASSETS_DIR" default:""`
- TrustedProxies string `env:"TRUSTED_PROXIES" default:""`
-
- FilePerm os.FileMode `env:"FILE_PERM" default:"0644"`
- DirPerm os.FileMode `env:"DIR_PERM" default:"0755"`
- GitWorkDir string `env:"GIT_WORK_DIR" default:"data/git"`
-
- HTTPClientTimeout int `env:"HTTP_CLIENT_TIMEOUT" default:"0"`
- BackupVolumeName string `env:"ARCANE_BACKUP_VOLUME_NAME" default:"arcane-backups"`
-
- // HTTPProxy, HTTPSProxy and NoProxy are forwarded to temporary Trivy scan containers.
- HTTPProxy string `env:"HTTP_PROXY" default:""`
- HTTPSProxy string `env:"HTTPS_PROXY" default:""`
- NoProxy string `env:"NO_PROXY" default:""`
-
- // Timezone for cron job scheduling. Uses IANA timezone names (e.g., "America/New_York", "Europe/London").
- // "Local" uses the system's local timezone, "UTC" for Coordinated Universal Time.
- Timezone string `env:"TZ" default:"Local"`
+ TrustedProxies string `env:"TRUSTED_PROXIES" default:""`
+ FilePerm os.FileMode `env:"FILE_PERM" default:"0644"`
+ DirPerm os.FileMode `env:"DIR_PERM" default:"0755"`
+ GitWorkDir string `env:"GIT_WORK_DIR" default:"data/git"`
+ HTTPClientTimeout int `env:"HTTP_CLIENT_TIMEOUT" default:"0"`
+ BackupVolumeName string `env:"ARCANE_BACKUP_VOLUME_NAME" default:"arcane-backups"`
+ HTTPProxy string `env:"HTTP_PROXY" default:""`
+ HTTPSProxy string `env:"HTTPS_PROXY" default:""`
+ NoProxy string `env:"NO_PROXY" default:""`
+ Timezone string `env:"TZ" default:"Local"`
}
func Load() *Config {
diff --git a/backend/internal/config/schema/schema_test.go b/backend/internal/config/schema/schema_test.go
index 983e1dd69..5ae2949c8 100644
--- a/backend/internal/config/schema/schema_test.go
+++ b/backend/internal/config/schema/schema_test.go
@@ -195,6 +195,7 @@ func countDocumentedSettingOverrides() int {
}
var expectedEnvConfigVars = []string{
+ "ACTOR_PORT",
"ADMIN_STATIC_API_KEY",
"AGENT_MODE",
"AGENT_TOKEN",
diff --git a/backend/internal/container/module.go b/backend/internal/container/module.go
index bfcaf2ba5..68fba5fe2 100644
--- a/backend/internal/container/module.go
+++ b/backend/internal/container/module.go
@@ -6,35 +6,29 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
- "github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/image"
- "github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-// Dependencies are the collaborators the container domain needs.
-type Dependencies struct {
- Event *event.EventService
- Docker *docker.DockerClientService
- Image *image.ImageService
- Settings *settings.SettingsService
- Project *project.ProjectService
- Activity *activity.ActivityService
-}
-
// Module wires the container domain and mounts its routes.
type Module struct {
- Service *ContainerService
- deps Dependencies
+ service *ContainerService
+ dockerService *docker.DockerClientService
+ settingsService *settings.SettingsService
+ activityService *activity.ActivityService
}
-// New builds the container domain from its dependencies.
-func New(deps Dependencies) *Module {
- return &Module{
- Service: NewContainerService(deps.Event, deps.Docker, deps.Image, deps.Settings, deps.Project),
- deps: deps,
+// New wires container routes around an existing service.
+func New(service *ContainerService, dockerService *docker.DockerClientService, settingsService *settings.SettingsService, activityService *activity.ActivityService) *Module {
+ return &Module{service: service, dockerService: dockerService, settingsService: settingsService, activityService: activityService}
+}
+
+// Service exposes the container service to collaborators.
+func (m *Module) Service() *ContainerService {
+ if m == nil {
+ return nil
}
+ return m.service
}
// RegisterRoutes mounts the container endpoints. A nil module still registers,
@@ -44,5 +38,5 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
RegisterContainers(api, nil, nil, nil, nil, appCtx)
return
}
- RegisterContainers(api, m.Service, m.deps.Docker, m.deps.Settings, m.deps.Activity, appCtx)
+ RegisterContainers(api, m.service, m.dockerService, m.settingsService, m.activityService, appCtx)
}
diff --git a/backend/internal/dashboard/module.go b/backend/internal/dashboard/module.go
index 3086891bd..d5c210a07 100644
--- a/backend/internal/dashboard/module.go
+++ b/backend/internal/dashboard/module.go
@@ -5,32 +5,9 @@ package dashboard
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/container"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
- "github.com/getarcaneapp/arcane/backend/v2/internal/image"
- "github.com/getarcaneapp/arcane/backend/v2/internal/project"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
- "github.com/getarcaneapp/arcane/backend/v2/internal/version"
- "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
- "github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
)
-// Dependencies are the collaborators the dashboard domain needs.
-type Dependencies struct {
- DB *database.DB
- Docker *docker.DockerClientService
- Container *container.ContainerService
- Project *project.ProjectService
- Image *image.ImageService
- Settings *settings.SettingsService
- Vulnerability *vulnerability.VulnerabilityService
- Environment *environment.EnvironmentService
- Version *version.VersionService
- Volume *volume.VolumeService
-}
-
// Module is the dashboard domain's wiring seam: it owns the service and the
// handler, and mounts the domain's routes.
type Module struct {
@@ -38,21 +15,9 @@ type Module struct {
handler *DashboardHandler
}
-// New builds the dashboard domain from its dependencies.
-func New(deps Dependencies) *Module {
- service := NewDashboardService(
- deps.DB,
- deps.Docker,
- deps.Container,
- deps.Project,
- deps.Image,
- deps.Settings,
- deps.Vulnerability,
- deps.Environment,
- deps.Version,
- deps.Volume,
- )
- return &Module{service: service, handler: NewHandler(service, deps.Environment)}
+// New wires dashboard routes around an existing service.
+func New(service *DashboardService, environmentService *environment.EnvironmentService) *Module {
+ return &Module{service: service, handler: NewHandler(service, environmentService)}
}
// Handler exposes the dashboard stream producer.
diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.go
index 45d1de22c..fafd88521 100644
--- a/backend/internal/dashboard/service_test.go
+++ b/backend/internal/dashboard/service_test.go
@@ -11,7 +11,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -37,7 +36,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.getarcane.app/updater/labels"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -198,7 +196,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {
Status: project.ProjectStatusStopped,
}).Error)
imageSvc := image.NewImageService(db, nil, nil, nil, nil, nil)
- projectSvc := project.NewProjectService(db, settingsSvc, nil, imageSvc, nil, nil, nil, nil, config.Load())
+ projectSvc := project.NewProjectService(db, settingsSvc, nil, imageSvc, nil, nil, nil, nil, config.Load(), nil, nil)
svc := NewDashboardService(db, dockerSvc, nil, projectSvc, imageSvc, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil))
snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)
@@ -342,21 +340,11 @@ func createTestRemoteEnvironmentInternal(t *testing.T, db *database.DB, environm
// stopped when the test ends.
func newSettingsServiceForTestInternal(ctx context.Context, t testing.TB, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(ctx, lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(ctx, runtime, "settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(ctx, runtime, "settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() { //nolint:contextcheck // cleanup must outlive ctx to stop the runtime
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func TestDashboardService_GetSnapshot_TrimmedOmitsTablesAndSharesBuilds(t *testing.T) {
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go
index 68259ca0f..fbbdfd981 100644
--- a/backend/internal/database/database.go
+++ b/backend/internal/database/database.go
@@ -17,6 +17,7 @@ import (
"emperror.dev/errors"
sqliteutil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/sqlite"
"github.com/getarcaneapp/arcane/backend/v2/resources"
+ dbtypes "github.com/getarcaneapp/arcane/types/v2/database"
"github.com/libtnb/sqlite"
"github.com/pressly/goose/v3"
"github.com/pressly/goose/v3/lock"
@@ -121,7 +122,7 @@ func connectDatabaseInternal(ctx context.Context, databaseURL string) (*DB, erro
if err := sqliteutil.RegisterFunctions(); err != nil {
return nil, errors.WrapIf(err, "failed to register SQLite functions")
}
- connString, err := ParseSQLiteConnectionString(databaseURL)
+ connString, err := ParseSQLiteConnectionString(databaseURL, dbtypes.SQLiteConnectionOptions{})
if err != nil {
return nil, errors.WrapIf(err, "failed to parse SQLite connection string")
}
@@ -603,7 +604,7 @@ func missingEmbeddedDowngradeMigrationsInternal(ctx context.Context, db *sql.DB,
return missing, nil
}
-func ParseSQLiteConnectionString(connString string) (string, error) {
+func ParseSQLiteConnectionString(connString string, options dbtypes.SQLiteConnectionOptions) (string, error) {
if !strings.HasPrefix(connString, "file:") {
connString = "file:" + connString
}
@@ -634,11 +635,42 @@ func ParseSQLiteConnectionString(connString string) (string, error) {
qs.Add("_pragma", "journal_mode("+v[0]+")")
case "_txlock":
qs.Add("_txlock", v[0])
+ case "_pragma":
+ qs["_pragma"] = append(qs["_pragma"], v...)
default:
qs[k] = v
}
}
+ pragmas := make([]string, 0, len(qs["_pragma"])+2)
+ journalMode, busyTimeout := false, false
+ for _, pragma := range qs["_pragma"] {
+ name, _, _ := strings.Cut(pragma, "(")
+ name, _, _ = strings.Cut(name, "=")
+ switch strings.ToLower(strings.TrimSpace(name)) {
+ case "foreign_keys":
+ if options.IgnoreForeignKeys {
+ continue
+ }
+ case "journal_mode":
+ journalMode = true
+ case "busy_timeout":
+ busyTimeout = true
+ }
+ pragmas = append(pragmas, pragma)
+ }
+ if !journalMode && options.JournalMode != "" {
+ pragmas = append(pragmas, "journal_mode("+options.JournalMode+")")
+ }
+ if !busyTimeout && options.BusyTimeout > 0 {
+ pragmas = append(pragmas, "busy_timeout("+strconv.FormatInt(options.BusyTimeout.Milliseconds(), 10)+")")
+ }
+ if len(pragmas) > 0 {
+ qs["_pragma"] = pragmas
+ } else {
+ qs.Del("_pragma")
+ }
+
connStringUrl.RawQuery = qs.Encode()
return connStringUrl.String(), nil
}
diff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.go
index d5873cf3a..3458fd8f3 100644
--- a/backend/internal/database/database_test.go
+++ b/backend/internal/database/database_test.go
@@ -14,6 +14,7 @@ import (
"time"
sqliteutil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/sqlite"
+ dbtypes "github.com/getarcaneapp/arcane/types/v2/database"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -901,3 +902,19 @@ func TestIdentityNormalizationPostgresUpgrade(t *testing.T) {
require.NoError(t, err)
require.Equal(t, highest, version)
}
+
+func TestSQLiteDSN(t *testing.T) {
+ original := "file:arcane.db?_fk=0&_pragma=foreign_keys%3D1&_pragma=synchronous(NORMAL)&_txlock=immediate"
+ dsn, err := ParseSQLiteConnectionString(original, dbtypes.SQLiteConnectionOptions{
+ IgnoreForeignKeys: true,
+ JournalMode: "WAL",
+ BusyTimeout: 2500 * time.Millisecond,
+ })
+ require.NoError(t, err)
+ parsed, err := url.Parse(dsn)
+ require.NoError(t, err)
+ require.Equal(t, "immediate", parsed.Query().Get("_txlock"))
+ require.ElementsMatch(t, []string{"synchronous(NORMAL)", "journal_mode(WAL)", "busy_timeout(2500)"}, parsed.Query()["_pragma"])
+ require.Empty(t, parsed.Query().Get("_fk"))
+ require.Contains(t, original, "_fk=0")
+}
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index ba1593cde..d6d943f05 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -2,18 +2,23 @@
package di
import (
+ "context"
+ "log/slog"
+
"emperror.dev/emperror"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
"github.com/getarcaneapp/arcane/backend/v2/internal/appimages"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/build"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/dashboard"
"github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/federated"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
@@ -27,118 +32,137 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/passkey"
"github.com/getarcaneapp/arcane/backend/v2/internal/port"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
- "github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
- s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/search"
"github.com/getarcaneapp/arcane/backend/v2/internal/session"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
"github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
"github.com/getarcaneapp/arcane/backend/v2/internal/template"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/updater"
"github.com/getarcaneapp/arcane/backend/v2/internal/upload"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/internal/variable"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/webhook"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/logging"
+ "github.com/getarcaneapp/arcane/backend/v2/resources"
"go.uber.org/fx"
)
-// ActorOptions provides the shared in-process actor runtime.
+// ActorOptions provides the durable Francis runtime and admission.
var ActorOptions = fx.Options(
- fx.Provide(actors.NewRuntime),
+ fx.Provide(provideActorRuntimeInternal, provideRunCoordinatorInternal),
fx.Provide(provideAdmissionGateInternal),
fx.Provide(provideTunnelRegistryInternal),
)
-// ServiceOptions provides the backend service graph.
+// ServiceOptions constructs services once and injects them into route modules.
var ServiceOptions = fx.Options(
- fx.Supply(fx.Annotate(new(logging.SlogErrorHandler), fx.As(new(emperror.ErrorHandler)))),
+ fx.Supply(
+ fx.Annotate(new(logging.SlogErrorHandler), fx.As(new(emperror.ErrorHandler))),
+ resources.FS,
+ ),
fx.Provide(
- // Infrastructure values consumed by services.
- provideResourcesFSInternal,
- provideJWKSetManagerInternal,
-
- // Services constructed directly through their public constructors.
- provideEventModuleInternal,
- (*event.Module).Service,
- provideActivityModuleInternal,
- (*activity.Module).Service,
- provideSettingsServiceInternal,
+ fx.Annotate(activity.NewActivityService, fx.As(fx.Self()), fx.As(new(activitylib.Service))),
+ event.NewEventService,
+ job.NewJobService,
+ role.NewRoleService,
+ apikey.NewApiKeyService,
+ user.NewUserService,
+ federated.NewFederatedCredentialService,
kv.NewKVService,
- provideJobModuleInternal,
- (*job.Module).Service,
- search.New,
appimages.NewApplicationImagesService,
- provideDockerClientServiceInternal,
- provideRoleModuleInternal,
- (*role.Module).Service,
session.NewSessionService,
passkey.NewPasskeyService,
environment.NewEnvironmentService,
- provideEnvironmentModuleInternal,
- provideSettingsModuleInternal,
apns.NewApnsService,
- apns.New,
notification.NewNotificationService,
- notification.New,
vulnerability.NewVulnerabilityService,
- vulnerability.New,
imagepatch.NewImagePatchService,
- imagepatch.New,
imageupdate.NewImageUpdateService,
image.NewImageService,
- provideImageUpdateModuleInternal,
- provideImageModuleInternal,
build.NewBuildService,
build.NewBuildWorkspaceService,
project.NewLifecycleService,
- provideProjectServiceInternal,
- project.New,
- provideContainerModuleInternal,
- provideDashboardModuleInternal,
+ container.NewContainerService,
+ dashboard.NewDashboardService,
network.NewNetworkService,
port.NewPortService,
swarm.NewSwarmService,
- provideSwarmModuleInternal,
- provideTemplateModuleInternal,
- (*template.Module).Service,
+ template.NewTemplateService,
oidc.NewOidcService,
- provideSystemModuleInternal,
+ system.NewSystemService,
system.NewSystemUpgradeService,
diagnostics.NewDiagnosticsService,
gitops.NewGitOpsSyncService,
- gitops.New,
- provideWebhookModuleInternal,
variable.NewVariableService,
- variable.New,
- provideS3ModuleInternal,
- (*s3domain.Module).Service,
- provideBackupEngineInternal,
backup.NewRecoveryKeyStore,
upload.NewUploadService,
- upload.New,
-
- // Adapters for scalar config fields, unexported parameters, builders, and lifecycle hooks.
- provideVersionServiceInternal,
- provideGitRepositoryModuleInternal,
- (*gitrepo.Module).Service,
- provideVolumeModuleInternal,
- (*volume.Module).Service,
systembackup.NewSystemBackupService,
- systembackup.New,
auth.NewAuthService,
- provideAuthModuleInternal,
+ settings.NewSettingsSearchService,
+ fx.Annotate(settings.NewSettingsService, fx.OnStop(func(ctx context.Context, service *settings.SettingsService) error { return service.Stop(ctx) })),
+ fx.Annotate(volume.NewVolumeService, fx.OnStop(func(ctx context.Context, service *volume.VolumeService) { service.CleanupHelperContainers(ctx) })),
+ fx.Annotate(webhook.NewWebhookService,
+ fx.OnStart(func(ctx context.Context, service *webhook.WebhookService) error { return service.LoadTokenHashes(ctx) }),
+ fx.OnStop(func(ctx context.Context, service *webhook.WebhookService) {
+ if err := service.DrainActions(ctx); err != nil {
+ slog.WarnContext(ctx, "shutdown proceeding with webhook actions still in flight", "error", err)
+ }
+ }),
+ ),
+ ),
+ // These providers bind config values, callbacks, or managed resource lifetimes.
+ fx.Provide(
+ provideJWKSetManagerInternal,
+ provideDockerClientServiceInternal,
+ provideProjectServiceInternal,
+ provideBackupEngineInternal,
+ provideVersionServiceInternal,
+ provideGitRepositoryServiceInternal,
+ provideS3ServiceInternal,
+ provideContainerRegistryServiceInternal,
+ provideUpdaterServiceInternal,
+ provideAuthMiddlewareInternal,
+ ),
+ fx.Provide(
+ event.New,
+ role.New,
+ apikey.New,
+ template.New,
+ gitrepo.New,
+ job.New,
+ environment.New,
+ image.New,
+ container.New,
+ dashboard.New,
+ swarm.New,
+ system.New,
+ volume.New,
+ updater.New,
+ webhook.New,
+ apns.New,
+ notification.New,
+ vulnerability.New,
+ imagepatch.New,
+ project.New,
+ gitops.New,
+ variable.New,
+ upload.New,
+ systembackup.New,
+ search.New,
+ provideActivityModuleInternal,
+ provideImageUpdateModuleInternal,
+ provideSettingsModuleInternal,
+ provideS3ModuleInternal,
provideContainerRegistryModuleInternal,
- (*registry.Module).Service,
- provideUpdaterModuleInternal,
- provideUserServiceInternal,
+ provideAuthModuleInternal,
provideUserModuleInternal,
- provideApiKeyModuleInternal,
- (*apikey.Module).Service,
- provideFederatedCredentialServiceInternal,
- provideAuthMiddlewareInternal,
),
)
diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.go
index d04de42c9..9616981ad 100644
--- a/backend/internal/di/di_test.go
+++ b/backend/internal/di/di_test.go
@@ -11,7 +11,6 @@ import (
"time"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/appimages"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
@@ -128,8 +127,11 @@ type graphParams struct {
}
func TestOptionsValidate(t *testing.T) {
+ _, cancel := context.WithCancel(t.Context())
+ defer cancel()
err := fx.ValidateApp(
fx.Supply(
+ cancel,
&config.Config{},
(*database.DB)(nil),
&http.Client{},
@@ -177,10 +179,8 @@ func TestDockerEventLifecycleInternal(t *testing.T) {
databaseDB := &database.DB{DB: db}
cfg := &config.Config{DockerHost: server.URL, AgentMode: mode != "manager", EdgeAgent: mode == "edge"}
lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
eventService := event.NewEventService(databaseDB, cfg, nil)
- provideDockerClientServiceInternal(t.Context(), lifecycle, runtime, databaseDB, cfg, nil, eventService)
+ provideDockerClientServiceInternal(t.Context(), lifecycle, databaseDB, cfg, nil, eventService)
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 5d4ba5c55..5ea358e9a 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -2,33 +2,26 @@ package di
import (
"context"
- "embed"
"log/slog"
"net/http"
"time"
+ "uuid"
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/build"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
- "github.com/getarcaneapp/arcane/backend/v2/internal/container"
- "github.com/getarcaneapp/arcane/backend/v2/internal/dashboard"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/federated"
- "github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
- "github.com/getarcaneapp/arcane/backend/v2/internal/job"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
- "github.com/getarcaneapp/arcane/backend/v2/internal/network"
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/internal/passkey"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
@@ -36,174 +29,140 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
- "github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
"github.com/getarcaneapp/arcane/backend/v2/internal/template"
"github.com/getarcaneapp/arcane/backend/v2/internal/updater"
- "github.com/getarcaneapp/arcane/backend/v2/internal/upload"
"github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
- "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
- "github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
- "github.com/getarcaneapp/arcane/backend/v2/internal/webhook"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"
- "github.com/getarcaneapp/arcane/backend/v2/resources"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/moby/moby/api/types/events"
"go.getarcane.app/streams/bus"
"go.uber.org/fx"
)
-func provideResourcesFSInternal() embed.FS {
- return resources.FS
-}
-
-func provideEventModuleInternal(db *database.DB, cfg *config.Config, httpClient *http.Client) *event.Module {
- return event.New(event.Dependencies{DB: db, Config: cfg, HTTPClient: httpClient})
-}
-
-func provideActivityModuleInternal(db *database.DB, settingsService *settings.SettingsService, environment *environment.EnvironmentService) *activity.Module {
- return activity.New(activity.Dependencies{
- DB: db,
- Settings: settingsService,
- Environment: activity.EnvironmentDependencies{
- ProxyJSONRequest: environment.ProxyJSONRequest,
- ListActiveRemoteEnvironments: environment.ListActiveRemoteEnvironments,
- GetActiveRemoteEnvironment: environment.GetActiveRemoteEnvironmentSnapshot,
- ProxyJSONRequestForEnvironment: environment.ProxyJSONRequestForEnvironment,
- ResolveEnvironmentName: environment.ResolveEnvironmentName,
- },
- })
-}
-
-func provideEnvironmentModuleInternal(service *environment.EnvironmentService, settingsService *settings.SettingsService, apiKey *apikey.ApiKeyService, eventService *event.EventService, cfg *config.Config, activityService *activity.ActivityService) *environment.Module {
- return environment.New(service, environment.Dependencies{
- Settings: settingsService,
- ApiKey: apiKey,
- Event: eventService,
- Config: cfg,
- Activity: activityService,
- })
-}
-
-func provideSwarmModuleInternal(service *swarm.SwarmService, environmentService *environment.EnvironmentService, eventService *event.EventService, cfg *config.Config) *swarm.Module {
- return swarm.New(service, swarm.Dependencies{
- Environment: environmentService,
- Event: eventService,
- Config: cfg,
+func provideActivityModuleInternal(service *activity.ActivityService, environment *environment.EnvironmentService) *activity.Module {
+ return activity.New(service, activity.EnvironmentDependencies{
+ ProxyJSONRequest: environment.ProxyJSONRequest,
+ ListActiveRemoteEnvironments: environment.ListActiveRemoteEnvironments,
+ GetActiveRemoteEnvironment: environment.GetActiveRemoteEnvironmentSnapshot,
+ ProxyJSONRequestForEnvironment: environment.ProxyJSONRequestForEnvironment,
+ ResolveEnvironmentName: environment.ResolveEnvironmentName,
})
}
func provideImageUpdateModuleInternal(service *imageupdate.ImageUpdateService, imageService *image.ImageService) *imageupdate.Module {
- return imageupdate.New(service, imageupdate.Dependencies{
- GetUpdateInfoByImageRefs: imageService.GetUpdateInfoByImageRefs,
- })
+ return imageupdate.New(service, imageService.GetUpdateInfoByImageRefs)
}
-func provideImageModuleInternal(service *image.ImageService, dockerService *docker.DockerClientService, imageUpdate *imageupdate.ImageUpdateService, settingsService *settings.SettingsService, buildService *build.BuildService, activityService *activity.ActivityService, uploadService *upload.UploadService) *image.Module {
- return image.New(service, image.Dependencies{
- Docker: dockerService,
- ImageUpdate: imageUpdate,
- Settings: settingsService,
- Build: buildService,
- Activity: activityService,
- Upload: uploadService,
- })
+func provideSettingsModuleInternal(service *settings.SettingsService, search *settings.SettingsSearchService, environment *environment.EnvironmentService, cfg *config.Config) *settings.Module {
+ return settings.New(service, search, environment.ProxyJSONRequest, cfg)
}
-func provideRoleModuleInternal(db *database.DB) *role.Module {
- return role.New(role.Dependencies{DB: db})
-}
-
-func provideSettingsServiceInternal(ctx context.Context, lc fx.Lifecycle, db *database.DB, runtime *actors.Runtime) (*settings.SettingsService, error) {
- executor, err := actors.NewExecutor(ctx, runtime, "services", "settings", 3)
- if err != nil {
+func provideBackupEngineInternal(ctx context.Context, lc fx.Lifecycle, admission *runs.Admission, imageService *image.ImageService, runtime *francis.Runtime, coordinator *runs.Coordinator, roles *role.RoleService, cfg *config.Config) (*backup.Engine, error) {
+ engine := backup.NewEngine(ctx, admission, imageService)
+ if err := engine.Register(runtime); err != nil {
return nil, err
}
- effects, err := actors.NewExecutor(ctx, runtime, "services", "settings-effects", 3)
- if err != nil {
- return nil, errors.Combine(err, executor.Stop(ctx))
- }
- service, err := settings.NewSettingsService(ctx, db, executor, effects)
- if err != nil {
- return nil, errors.Combine(err, executor.Stop(ctx), effects.Stop(ctx))
- }
- lc.Append(fx.Hook{
- OnStop: func(ctx context.Context) error {
- return errors.Combine(executor.Stop(ctx), effects.Stop(ctx))
- },
- })
- return service, nil
-}
-
-func provideSettingsModuleInternal(service *settings.SettingsService, environment *environment.EnvironmentService, cfg *config.Config) *settings.Module {
- return settings.New(service, settings.Dependencies{
- Search: settings.NewSettingsSearchService(),
- ProxyRemoteJSON: environment.ProxyJSONRequest,
- Config: cfg,
+ engine.SetExecutionReady(coordinator.Active)
+ engine.SetAuthorize(func(ctx context.Context, command backuptypes.DurableRunCommand) error {
+ if cfg.AgentMode && command.UserID == "agent" && command.RequestedWithKey == "" {
+ return nil
+ }
+ if command.UserID == "" {
+ return errors.New("requesting user unavailable")
+ }
+ permissions, err := roles.ResolveExecutionPermissions(ctx, command.UserID, command.RequestedWithKey)
+ if err != nil {
+ return err
+ }
+ if !permissions.Allows(command.Permission, command.EnvironmentID) {
+ return errors.New("requesting user no longer has permission for this backup")
+ }
+ if command.Permission == authz.PermSystemBackupsManage && !permissions.IsGlobalAdmin() {
+ return errors.New("system backups require a global administrator")
+ }
+ return nil
})
-}
-
-func provideBackupEngineInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime, admission *actors.Gate[actors.AdmissionKey], imageService *image.ImageService) *backup.Engine {
- engine := backup.NewEngine(ctx, runtime, admission, imageService)
lc.Append(fx.Hook{OnStop: engine.Stop})
- return engine
+ return engine, nil
}
-func provideAdmissionGateInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*actors.Gate[actors.AdmissionKey], error) {
- gate, err := actors.NewGate[actors.AdmissionKey](ctx, runtime, "admission", "application")
+func provideActorRuntimeInternal(appCtx context.Context, cfg *config.Config, settingsService *settings.SettingsService, lc fx.Lifecycle, cancelApp context.CancelFunc) (*francis.Runtime, error) {
+ runtime, err := francis.New(cfg.DatabaseURL, "", "", cfg.ActorPort)
if err != nil {
return nil, err
}
- lc.Append(fx.Hook{OnStop: gate.Stop})
- return gate, nil
+ lc.Append(fx.Hook{OnStart: func(ctx context.Context) error {
+ if err := runtime.ConfigureIdentity(cfg.EncryptionKey, settingsService.GetSettingsConfig().InstanceID.Value); err != nil {
+ return err
+ }
+ return runtime.Start(ctx, appCtx, func(err error) { slog.ErrorContext(appCtx, "Francis host failed", "error", err); cancelApp() })
+ }, OnStop: runtime.Stop})
+ return runtime, nil
}
-func provideTunnelRegistryInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*edge.TunnelRegistry, error) {
- registry, err := edge.NewActorTunnelRegistry(ctx, runtime)
- if err != nil {
+func provideRunCoordinatorInternal(runtime *francis.Runtime, store *kv.KVService, cfg *config.Config) (*runs.Coordinator, error) {
+ coordinator := runs.New(store, runtime.Service(), cfg.GetLocation())
+ if err := coordinator.Register(runtime); err != nil {
return nil, err
}
+ return coordinator, nil
+}
+
+func provideAdmissionGateInternal(runtime *francis.Runtime) (*runs.Admission, error) {
+ admission := runs.NewAdmission(runtime.Service(), uuid.New().String())
+ return admission, admission.Register(runtime)
+}
+
+func provideTunnelRegistryInternal(lc fx.Lifecycle) *edge.TunnelRegistry {
+ registry := edge.NewTunnelRegistry()
edge.SetDefaultRegistry(registry)
- lc.Append(fx.Hook{
- OnStop: func(ctx context.Context) error {
- defer edge.ClearDefaultRegistry(registry)
- return registry.Stop(ctx)
- },
- })
- return registry, nil
+ lc.Append(fx.Hook{OnStop: func(ctx context.Context) error { defer edge.ClearDefaultRegistry(registry); return registry.Stop(ctx) }})
+ return registry
}
-func provideDockerClientServiceInternal(ctx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, db *database.DB, cfg *config.Config, settings *settings.SettingsService, eventService *event.EventService) *docker.DockerClientService {
+func provideDockerClientServiceInternal(ctx context.Context, lc fx.Lifecycle, db *database.DB, cfg *config.Config, settings *settings.SettingsService, eventService *event.EventService) *docker.DockerClientService {
service := docker.NewDockerClientService(ctx, db, cfg, settings, bus.WithDroppedEventCallback(func(message events.Message) {
// Overflow is recovered synchronously, applying backpressure instead of losing log entries.
eventService.RecordDockerEvent(ctx, message)
}))
- var runner, logRunner *actors.Runner
+ var stopStream, stopLog func(context.Context) error
var unsubscribe func()
lc.Append(fx.Hook{
OnStart: func(startCtx context.Context) error {
run, cleanup := eventService.SubscribeDockerEvents(service.EventBus())
unsubscribe = cleanup
var err error
- logRunner, err = actors.NewRunner(ctx, actorRuntime, "docker-events", "log", "Docker event log", 3, run)
+ stopLog, err = concurrency.StartSupervised(ctx, "Docker event log", run)
if err != nil {
cleanup()
return err
}
- runner, err = actors.NewRunner(ctx, actorRuntime, "docker-events", "stream", "Docker event watcher", 3, func(runCtx context.Context) error {
+ stopStream, err = concurrency.StartSupervised(ctx, "Docker event watcher", func(runCtx context.Context) error {
service.WatchEvents(runCtx)
return nil
})
if err != nil {
- err = errors.Combine(err, logRunner.Stop(startCtx))
+ err = errors.Combine(err, stopLog(startCtx))
cleanup()
}
return err
},
OnStop: func(stopCtx context.Context) error {
- err := errors.Combine(runner.Stop(stopCtx), logRunner.Stop(stopCtx))
+ var err error
+ if stopStream != nil {
+ err = stopStream(stopCtx)
+ }
+ if stopLog != nil {
+ err = errors.Combine(err, stopLog(stopCtx))
+ }
if unsubscribe != nil {
unsubscribe()
}
@@ -218,84 +177,45 @@ func provideVersionServiceInternal(httpClient *http.Client, cfg *config.Config,
return version.NewVersionService(httpClient, cfg.UpdateCheckDisabled, config.Version, config.Revision, registry, docker, imageUpdate, settingsService)
}
-func provideGitRepositoryModuleInternal(db *database.DB, cfg *config.Config, eventService *event.EventService, settingsService *settings.SettingsService) *gitrepo.Module {
- return gitrepo.New(gitrepo.Dependencies{DB: db, WorkDir: cfg.GitWorkDir, Event: eventService, Settings: settingsService})
+func provideGitRepositoryServiceInternal(db *database.DB, cfg *config.Config, eventService *event.EventService, settingsService *settings.SettingsService) *gitrepo.GitRepositoryService {
+ return gitrepo.NewGitRepositoryService(db, cfg.GitWorkDir, eventService, settingsService)
}
-func provideS3ModuleInternal(db *database.DB, environmentService *environment.EnvironmentService) *s3domain.Module {
- return s3domain.New(s3domain.Dependencies{
- DB: db,
- SyncRemoteDestinations: environmentService.SyncS3DestinationsToRemoteEnvironments,
- CheckRemoteReferences: environmentService.CheckS3DestinationReferences,
- })
+func provideS3ModuleInternal(service *s3domain.S3DestinationService, environment *environment.EnvironmentService) *s3domain.Module {
+ return s3domain.New(service, environment.SyncS3DestinationsToRemoteEnvironments)
}
-func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, activityService *activity.ActivityService, containerModule *container.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, environmentService *environment.EnvironmentService, cfg *config.Config, uploadService *upload.UploadService, recoveryKeys *backup.RecoveryKeyStore) *volume.Module {
- module := volume.New(volume.Dependencies{
- DB: db,
- Docker: dockerService,
- Event: eventService,
- Settings: settingsService,
- Image: imageService,
- Activity: activityService,
- Environment: environmentService,
- Container: containerModule.Service,
- Engine: engine,
- S3: s3Service,
- Config: cfg,
- Upload: uploadService,
- RecoveryKeys: recoveryKeys,
- })
- lc.Append(fx.Hook{
- OnStop: func(ctx context.Context) error {
- module.Service().CleanupHelperContainers(ctx)
- return nil
- },
- })
- return module
+func provideS3ServiceInternal(db *database.DB, environment *environment.EnvironmentService) *s3domain.S3DestinationService {
+ return s3domain.NewS3DestinationService(db, environment.CheckS3DestinationReferences)
}
-func provideContainerModuleInternal(event *event.EventService, docker *docker.DockerClientService, image *image.ImageService, settings *settings.SettingsService, project *project.ProjectService, activity *activity.ActivityService) *container.Module {
- return container.New(container.Dependencies{
- Event: event,
- Docker: docker,
- Image: image,
- Settings: settings,
- Project: project,
- Activity: activity,
- })
+func provideAuthModuleInternal(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {
+ return auth.New(service, userService, settingsService, passkeyService.BeginMFAAuthentication)
}
-func provideAuthModuleInternal(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {
- return auth.New(auth.Dependencies{
- Service: service,
- User: userService,
- Settings: settingsService,
- BeginMFAAuthentication: passkeyService.BeginMFAAuthentication,
- })
+func provideContainerRegistryServiceInternal(db *database.DB, docker *docker.DockerClientService, kv *kv.KVService, settings *settings.SettingsService) *registry.ContainerRegistryService {
+ return registry.NewContainerRegistryService(db, func(ctx context.Context) (registry.RegistryDaemonClient, error) { return docker.GetClient(ctx) }, kv, settings)
}
-func provideContainerRegistryModuleInternal(db *database.DB, dockerService *docker.DockerClientService, kvService *kv.KVService, settingsService *settings.SettingsService, environment *environment.EnvironmentService) *registry.Module {
- return registry.New(registry.Dependencies{DB: db, Docker: dockerService, KV: kvService, Settings: settingsService, SyncRemoteRegistries: environment.SyncRegistriesToRemoteEnvironments})
+func provideContainerRegistryModuleInternal(service *registry.ContainerRegistryService, environment *environment.EnvironmentService) *registry.Module {
+ return registry.New(service, environment.SyncRegistriesToRemoteEnvironments)
}
func provideProjectServiceInternal(db *database.DB, settings *settings.SettingsService, event *event.EventService, image *image.ImageService, docker *docker.DockerClientService, build *build.BuildService, lifecycleService *project.LifecycleService, kv *kv.KVService, registry *registry.ContainerRegistryService, environment *environment.EnvironmentService, cfg *config.Config) *project.ProjectService {
- service := project.NewProjectService(db, settings, event, image, docker, build, lifecycleService, registry, cfg)
- service.KVService = kv
- service.RegistryCredentialsProvider = environment.GetEnabledRegistryCredentials
- return service
+ return project.NewProjectService(db, settings, event, image, docker, build, lifecycleService, registry, cfg, kv, environment.GetEnabledRegistryCredentials)
}
-// updaterServiceParams collects the updater's dependencies. The dedicated
-// provider exists because NewUpdaterService takes its self-upgrade dependency
-// as an unexported interface, which fx cannot supply directly; the adapter
-// narrows *SystemUpgradeService to that interface.
+// updaterServiceParams includes actor registration and worker lifecycle dependencies.
type updaterServiceParams struct {
fx.In
Context context.Context
+ Config *config.Config
Lifecycle fx.Lifecycle
- ActorRuntime *actors.Runtime
+ ActorRuntime *francis.Runtime
+ Coordinator *runs.Coordinator
+ Admission *runs.Admission
+ Roles *role.RoleService
DB *database.DB
Settings *settings.SettingsService
Docker *docker.DockerClientService
@@ -309,50 +229,20 @@ type updaterServiceParams struct {
Activity *activity.ActivityService
}
-func provideUpdaterModuleInternal(p updaterServiceParams) (*updater.Module, error) {
- executor, err := actors.NewExecutor(p.Context, p.ActorRuntime, "updater", "single-container", 3)
+func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterService, error) {
+ service, err := updater.NewUpdaterService(p.DB, p.Settings, p.Docker, p.Project, p.ImageUpdate, p.Registry, p.Event, p.Image, p.Notification, p.SystemUpgrade, p.Activity, p.Config, p.Coordinator, p.Admission, p.Roles)
if err != nil {
return nil, err
}
- module, err := updater.New(updater.Dependencies{
- DB: p.DB,
- Settings: p.Settings,
- Docker: p.Docker,
- Project: p.Project,
- ImageUpdate: p.ImageUpdate,
- Registry: p.Registry,
- Event: p.Event,
- Image: p.Image,
- Notification: p.Notification,
- SelfUpgrade: p.SystemUpgrade,
- Activity: p.Activity,
- SingleUpdates: executor,
- })
- if err != nil {
- return nil, errors.Combine(err, executor.Stop(p.Context))
+ if err := service.RegisterActors(p.ActorRuntime); err != nil {
+ return nil, err
}
- p.Lifecycle.Append(fx.Hook{OnStop: executor.Stop})
- return module, nil
-}
-
-func provideUserServiceInternal(db *database.DB, role *role.RoleService) *user.UserService {
- return user.NewUserService(db).WithRoleService(role)
+ p.Lifecycle.Append(fx.Hook{OnStart: func(context.Context) error { return service.Start(p.Context) }, OnStop: service.Stop}) //nolint:contextcheck // Workers inherit the application lifetime after startup returns.
+ return service, nil
}
func provideUserModuleInternal(service *user.UserService, auth *auth.AuthService, settingsService *settings.SettingsService) *user.Module {
- return user.New(user.Dependencies{Service: service, InvalidateUserTokenCache: auth.InvalidateUserTokenCache, Settings: settingsService})
-}
-
-func provideJobModuleInternal(db *database.DB, settingsService *settings.SettingsService, cfg *config.Config, environment *environment.EnvironmentService, roles *role.RoleService, activityService *activity.ActivityService) *job.Module {
- return job.New(job.Dependencies{DB: db, Settings: settingsService, Config: cfg, Environment: environment, Roles: roles, Activity: activityService})
-}
-
-func provideTemplateModuleInternal(ctx context.Context, db *database.DB, httpClient *http.Client, settingsService *settings.SettingsService) *template.Module {
- return template.New(template.Dependencies{Context: ctx, DB: db, HTTPClient: httpClient, Settings: settingsService})
-}
-
-func provideApiKeyModuleInternal(db *database.DB, userService *user.UserService, roleService *role.RoleService) *apikey.Module {
- return apikey.New(apikey.Dependencies{DB: db, User: userService, Role: roleService})
+ return user.New(service, auth.InvalidateUserTokenCache, settingsService)
}
func provideJWKSetManagerInternal(ctx context.Context, lc fx.Lifecycle) *oidcjwk.KeySetManager {
@@ -361,10 +251,6 @@ func provideJWKSetManagerInternal(ctx context.Context, lc fx.Lifecycle) *oidcjwk
return manager
}
-func provideFederatedCredentialServiceInternal(db *database.DB, auth *auth.AuthService, user *user.UserService, settings *settings.SettingsService, event *event.EventService, httpClient *http.Client, role *role.RoleService, keySetManager *oidcjwk.KeySetManager) *federated.FederatedCredentialService {
- return federated.NewFederatedCredentialService(db, auth, user, settings, event, httpClient, keySetManager).WithRoleService(role)
-}
-
func provideAuthMiddlewareInternal(authService *auth.AuthService, apiKey *apikey.ApiKeyService, env *environment.EnvironmentService, role *role.RoleService, cfg *config.Config) *auth.AuthMiddleware {
return auth.NewAuthMiddleware(authService, cfg).
WithApiKeyValidator(apiKey).
@@ -372,15 +258,15 @@ func provideAuthMiddlewareInternal(authService *auth.AuthService, apiKey *apikey
WithPermissionResolver(role)
}
-func provideFilesystemWatcherJobInternal(ctx context.Context, lc fx.Lifecycle, actorRuntime *actors.Runtime, project *project.ProjectService, template *template.TemplateService, settings *settings.SettingsService, cfg *config.Config) (*scheduler.FilesystemWatcherJob, error) {
- job, err := scheduler.NewFilesystemWatcherJob(ctx, actorRuntime, project, template, settings, cfg.ProjectScanMaxDepth)
+func provideFilesystemWatcherJobInternal(ctx context.Context, lc fx.Lifecycle, project *project.ProjectService, template *template.TemplateService, settings *settings.SettingsService, cfg *config.Config) (*scheduler.FilesystemWatcherJob, error) {
+ job, err := scheduler.NewFilesystemWatcherJob(ctx, project, template, settings, cfg.ProjectScanMaxDepth)
if err != nil {
return nil, err
}
- var runner *actors.Runner
+ var stop func(context.Context) error
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
- runner, err = actors.NewRunner(ctx, actorRuntime, "filesystem-watcher", "lifecycle", "filesystem watcher", 3, func(runCtx context.Context) error {
+ stop, err = concurrency.StartSupervised(ctx, "filesystem watcher", func(runCtx context.Context) error {
delay := 5 * time.Second
for runCtx.Err() == nil {
startErr := job.Start(runCtx)
@@ -408,65 +294,12 @@ func provideFilesystemWatcherJobInternal(ctx context.Context, lc fx.Lifecycle, a
return nil
},
OnStop: func(stopCtx context.Context) error {
- return errors.Combine(runner.Stop(stopCtx), job.Stop(stopCtx))
- },
- })
- return job, nil
-}
-
-func provideDashboardModuleInternal(db *database.DB, docker *docker.DockerClientService, containerModule *container.Module, project *project.ProjectService, image *image.ImageService, settings *settings.SettingsService, vulnerability *vulnerability.VulnerabilityService, environment *environment.EnvironmentService, version *version.VersionService, volumeModule *volume.Module) *dashboard.Module {
- return dashboard.New(dashboard.Dependencies{
- DB: db,
- Docker: docker,
- Container: containerModule.Service,
- Project: project,
- Image: image,
- Settings: settings,
- Vulnerability: vulnerability,
- Environment: environment,
- Version: version,
- Volume: volumeModule.Service(),
- })
-}
-
-func provideSystemModuleInternal(db *database.DB, cfg *config.Config, docker *docker.DockerClientService, containerModule *container.Module, image *image.ImageService, imageUpdate *imageupdate.ImageUpdateService, volumeModule *volume.Module, network *network.NetworkService, settings *settings.SettingsService, activity *activity.ActivityService, upgrade *system.SystemUpgradeService, environment *environment.EnvironmentService) *system.Module {
- return system.New(system.Dependencies{
- DB: db,
- Config: cfg,
- Docker: docker,
- Container: containerModule.Service,
- Image: image,
- ImageUpdate: imageUpdate,
- Volume: volumeModule.Service(),
- Network: network,
- Settings: settings,
- Activity: activity,
- SystemUpgrade: upgrade,
- Environment: environment,
- })
-}
-
-func provideWebhookModuleInternal(lc fx.Lifecycle, db *database.DB, containerModule *container.Module, updaterModule *updater.Module, project *project.ProjectService, gitOpsSync *gitops.GitOpsSyncService, event *event.EventService, environment *environment.EnvironmentService) *webhook.Module {
- module := webhook.New(webhook.Dependencies{
- DB: db,
- Container: containerModule.Service,
- Updater: updaterModule.Service(),
- Project: project,
- GitOpsSync: gitOpsSync,
- Event: event,
- Environment: environment,
- })
- lc.Append(fx.Hook{
- OnStart: module.Service().LoadTokenHashes,
- OnStop: func(ctx context.Context) error {
- // Drain fails only when the stop context expires; fx then skips all
- // remaining teardown regardless of what is returned here, so an
- // error would only mark an expected long action as a failed stop.
- if err := module.Service().DrainActions(ctx); err != nil {
- slog.WarnContext(ctx, "shutdown proceeding with webhook actions still in flight", "error", err)
+ var err error
+ if stop != nil {
+ err = stop(stopCtx)
}
- return nil
+ return errors.Combine(err, job.Stop(stopCtx))
},
})
- return module
+ return job, nil
}
diff --git a/backend/internal/docker/service_test.go b/backend/internal/docker/service_test.go
index 7abc9598d..ff2fb3517 100644
--- a/backend/internal/docker/service_test.go
+++ b/backend/internal/docker/service_test.go
@@ -10,8 +10,8 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
imagetypes "github.com/getarcaneapp/arcane/types/v2/image"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/events"
@@ -19,7 +19,6 @@ import (
"github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
)
func TestNewDockerClient_PinsEffectiveAPIVersion(t *testing.T) {
@@ -212,15 +211,12 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *test
}
})
- lifecycle := fxtest.NewLifecycle(t)
- actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
service := newDockerClientServiceForTestInternal(server.URL)
t.Cleanup(service.Close)
eventsCh, unsubscribe := service.EventBus().Subscribe(events.ImageEventType)
t.Cleanup(unsubscribe)
- runner, err := actors.NewRunner(t.Context(), actorRuntime, "docker-events-test", "stream", "Docker event watcher", 3, func(ctx context.Context) error {
+ stop, err := concurrency.StartSupervised(t.Context(), "Docker event watcher", func(ctx context.Context) error {
service.WatchEvents(ctx)
return nil
})
@@ -239,7 +235,7 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *test
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
- require.NoError(t, runner.Stop(stopCtx))
+ require.NoError(t, stop(stopCtx))
require.Eventually(t, func() bool {
select {
case <-streamStopped:
@@ -248,7 +244,6 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *test
return false
}
}, time.Second, time.Millisecond)
- require.NoError(t, lifecycle.Stop(stopCtx))
}
func TestCountImageUsage_UsesContainerImageIDs(t *testing.T) {
diff --git a/backend/internal/environment/environment_health.go b/backend/internal/environment/environment_health.go
index 1d2c19d36..af34a6ebe 100644
--- a/backend/internal/environment/environment_health.go
+++ b/backend/internal/environment/environment_health.go
@@ -7,10 +7,10 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/types/v2/environment"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/moby/moby/client"
@@ -27,7 +27,7 @@ const (
// SetScheduler injects the job scheduler and app lifecycle context. Called during
// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job
// registration becomes a no-op).
-func (s *EnvironmentService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {
+func (s *EnvironmentService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *runs.Admission) error {
return s.jobs.SetScheduler(ctx, scheduler, admissionGate)
}
@@ -131,7 +131,7 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s
slog.WarnContext(ctx, "environment health check skipped; previous run still in progress", "environment_id", envID)
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
- defer lease.Release()
+ defer lease.Release(ctx)
environment, err := s.GetEnvironmentByID(ctx, envID)
if err != nil {
diff --git a/backend/internal/environment/module.go b/backend/internal/environment/module.go
index 5fe80e62b..9ffe9aa82 100644
--- a/backend/internal/environment/module.go
+++ b/backend/internal/environment/module.go
@@ -12,23 +12,15 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-type Dependencies struct {
- Settings *settings.SettingsService
- ApiKey *apikey.ApiKeyService
- Event *event.EventService
- Config *config.Config
- Activity activitylib.Service
-}
-
type Module struct {
service *EnvironmentService
handler *EnvironmentHandler
}
-func New(service *EnvironmentService, deps Dependencies) *Module {
+func New(service *EnvironmentService, settingsService *settings.SettingsService, apiKey *apikey.ApiKeyService, eventService *event.EventService, cfg *config.Config, activityService activitylib.Service) *Module {
return &Module{
service: service,
- handler: NewHandler(service, deps.Settings, deps.ApiKey, deps.Event, deps.Config, deps.Activity),
+ handler: NewHandler(service, settingsService, apiKey, eventService, cfg, activityService),
}
}
diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.go
index 75debfc79..dff31395b 100644
--- a/backend/internal/environment/service_test.go
+++ b/backend/internal/environment/service_test.go
@@ -15,7 +15,6 @@ import (
"time"
"github.com/coder/websocket"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -28,6 +27,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/environment"
@@ -37,18 +38,13 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.getarcane.app/sys/crypto"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), actorRuntime, "environment-test-admission", "overlap")
- require.NoError(t, err)
+ gate := newAdmissionGateForEnvironmentTestInternal(t)
- key := actors.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: "environment-id"}
+ key := schedulertypes.AdmissionKey{Scope: environmentHealthAdmissionScopeInternal, ID: "environment-id"}
lease, admitted, err := gate.TryAcquire(t.Context(), key)
require.NoError(t, err)
require.True(t, admitted)
@@ -56,12 +52,7 @@ func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T
service := &EnvironmentService{jobs: entityjobs.New(environmentHealthJobPrefix, environmentHealthAdmissionScopeInternal)}
require.NoError(t, service.SetScheduler(t.Context(), &environmentTestSchedulerInternal{}, gate))
service.runHealthCheckInternal(t.Context(), "environment-id")
- lease.Release()
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
+ lease.Release(t.Context())
}
type environmentTestSchedulerInternal struct {
@@ -83,25 +74,16 @@ func (s *environmentTestSchedulerInternal) HasJob(_ string) bool {
return false
}
-func newAdmissionGateForEnvironmentTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {
+func newAdmissionGateForEnvironmentTestInternal(t testing.TB) *runs.Admission {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "environment-test-admission", t.Name())
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
+ runtime := francistest.New(t)
+ gate := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, gate.Register(runtime))
+ francistest.Start(t, runtime)
return gate
}
-// testProjectRow / testGitOpsSyncRow are minimal stand-ins for the project
-// domain's models: the project package imports environment, so this in-package
-// test cannot import it. Only the columns the cascade path touches are declared.
+// Project imports environment, so these rows avoid an import cycle in tests.
type testProjectRow struct {
database.BaseModel
Name string
@@ -947,8 +929,8 @@ func TestAgentHostPortInternal(t *testing.T) {
func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_CreatesVisibleEnvironmentAndReusesToken(t *testing.T) {
ctx := context.Background()
db := setupEnvironmentServiceTestDB(t)
- userService := user.NewUserService(db)
- apiKeyService := apikey.NewApiKeyService(db, userService)
+ userService := user.NewUserService(db, nil)
+ apiKeyService := apikey.NewApiKeyService(db, userService, nil)
svc := NewEnvironmentService(db, nil, nil, nil, nil, apiKeyService)
user := createTestEnvironmentServiceUser(t, ctx, userService, "swarm-admin")
@@ -1087,8 +1069,8 @@ func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_ReusesLegacyHiddenRe
func TestEnvironmentService_EnsureSwarmNodeAgentEnvironment_TokenResolvesEndToEnd(t *testing.T) {
ctx := context.Background()
db := setupEnvironmentServiceTestDB(t)
- userService := user.NewUserService(db)
- apiKeyService := apikey.NewApiKeyService(db, userService)
+ userService := user.NewUserService(db, nil)
+ apiKeyService := apikey.NewApiKeyService(db, userService, nil)
svc := NewEnvironmentService(db, nil, nil, nil, nil, apiKeyService)
user := createTestEnvironmentServiceUser(t, ctx, userService, "swarm-resolve-admin")
diff --git a/backend/internal/event/docker_events.go b/backend/internal/event/docker_events.go
index 0ba892d34..46c73d4f0 100644
--- a/backend/internal/event/docker_events.go
+++ b/backend/internal/event/docker_events.go
@@ -3,6 +3,7 @@ package event
import (
"cmp"
"context"
+ "crypto/sha256"
"encoding/json/v2"
"fmt"
"log/slog"
@@ -10,7 +11,6 @@ import (
"time"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/google/uuid"
"github.com/moby/moby/api/types/events"
kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/streams/bus"
@@ -94,7 +94,7 @@ func (s *EventService) RecordDockerEvent(ctx context.Context, msg events.Message
slog.WarnContext(ctx, "Failed to identify Docker daemon event", "type", req.Type, "resourceId", msg.Actor.ID, "error", err)
return
}
- req.deduplicationID = uuid.NewSHA1(uuid.NameSpaceOID, identity).String()
+ req.deduplicationKey = fmt.Sprintf("%x", sha256.Sum256(identity))
}
if _, err := s.CreateEvent(ctx, req); err != nil {
slog.WarnContext(ctx, "Failed to log Docker daemon event", "type", req.Type, "resourceId", msg.Actor.ID, "error", err)
diff --git a/backend/internal/event/model.go b/backend/internal/event/model.go
index 37d5f5b6d..a270bac0f 100644
--- a/backend/internal/event/model.go
+++ b/backend/internal/event/model.go
@@ -130,18 +130,19 @@ const (
type Event struct {
database.BaseModel
- Type EventType `json:"type" sortable:"true"`
- Severity EventSeverity `json:"severity" sortable:"true"`
- Title string `json:"title" sortable:"true"`
- Description string `json:"description"`
- ResourceType *string `json:"resourceType,omitempty" sortable:"true"`
- ResourceID *string `json:"resourceId,omitempty" sortable:"true"`
- ResourceName *string `json:"resourceName,omitempty" sortable:"true"`
- UserID *string `json:"userId,omitempty" sortable:"true"`
- Username *string `json:"username,omitempty" sortable:"true"`
- EnvironmentID *string `json:"environmentId,omitempty"`
- Metadata database.JSON `json:"metadata,omitempty" gorm:"type:text"`
- Timestamp time.Time `json:"timestamp" sortable:"true"`
+ DeduplicationKey *string `json:"-" gorm:"uniqueIndex:idx_events_deduplication_key"`
+ Type EventType `json:"type" sortable:"true"`
+ Severity EventSeverity `json:"severity" sortable:"true"`
+ Title string `json:"title" sortable:"true"`
+ Description string `json:"description"`
+ ResourceType *string `json:"resourceType,omitempty" sortable:"true"`
+ ResourceID *string `json:"resourceId,omitempty" sortable:"true"`
+ ResourceName *string `json:"resourceName,omitempty" sortable:"true"`
+ UserID *string `json:"userId,omitempty" sortable:"true"`
+ Username *string `json:"username,omitempty" sortable:"true"`
+ EnvironmentID *string `json:"environmentId,omitempty"`
+ Metadata database.JSON `json:"metadata,omitempty" gorm:"type:text"`
+ Timestamp time.Time `json:"timestamp" sortable:"true"`
}
func (Event) TableName() string {
diff --git a/backend/internal/event/module.go b/backend/internal/event/module.go
index 80d3c80bb..184e3df79 100644
--- a/backend/internal/event/module.go
+++ b/backend/internal/event/module.go
@@ -3,31 +3,19 @@ package event
import (
"context"
- "net/http"
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/config"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/labstack/echo/v5"
)
-// Dependencies are the collaborators the event domain needs.
-type Dependencies struct {
- DB *database.DB
- Config *config.Config
- HTTPClient *http.Client
-}
-
// Module owns event persistence and both of the domain's HTTP surfaces.
type Module struct {
service *EventService
}
-// New builds the event domain from its dependencies.
-func New(deps Dependencies) *Module {
- return &Module{
- service: NewEventService(deps.DB, deps.Config, deps.HTTPClient),
- }
+// New assembles the event routes around its service.
+func New(service *EventService) *Module {
+ return &Module{service: service}
}
// Service exposes event operations to collaborating domains.
diff --git a/backend/internal/event/service.go b/backend/internal/event/service.go
index 1f710cdde..8e92ae719 100644
--- a/backend/internal/event/service.go
+++ b/backend/internal/event/service.go
@@ -14,12 +14,12 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
eventtypes "github.com/getarcaneapp/arcane/types/v2/event"
"github.com/samber/mo"
@@ -32,7 +32,7 @@ import (
)
type EventService struct {
- changes *actors.Signal[struct{}]
+ changes *concurrency.Signal[struct{}]
dockerCorrelation dockerCorrelationInternal
db *database.DB
cfg *config.Config
@@ -46,7 +46,7 @@ func NewEventService(db *database.DB, cfg *config.Config, httpClient *http.Clien
}
}
return &EventService{
- changes: actors.NewSignal[struct{}](),
+ changes: concurrency.NewSignal[struct{}](),
dockerCorrelation: dockerCorrelationInternal{now: time.Now},
db: db,
cfg: cfg,
@@ -56,18 +56,18 @@ func NewEventService(db *database.DB, cfg *config.Config, httpClient *http.Clien
type CreateEventRequest struct {
// Internal identity for replay-safe daemon delivery; never accepted from API payloads.
- deduplicationID string
- Type EventType `json:"type"`
- Severity EventSeverity `json:"severity,omitempty"`
- Title string `json:"title"`
- Description string `json:"description,omitempty"`
- ResourceType *string `json:"resourceType,omitempty"`
- ResourceID *string `json:"resourceId,omitempty"`
- ResourceName *string `json:"resourceName,omitempty"`
- UserID *string `json:"userId,omitempty"`
- Username *string `json:"username,omitempty"`
- EnvironmentID *string `json:"environmentId,omitempty"`
- Metadata database.JSON `json:"metadata,omitempty"`
+ deduplicationKey string
+ Type EventType `json:"type"`
+ Severity EventSeverity `json:"severity,omitempty"`
+ Title string `json:"title"`
+ Description string `json:"description,omitempty"`
+ ResourceType *string `json:"resourceType,omitempty"`
+ ResourceID *string `json:"resourceId,omitempty"`
+ ResourceName *string `json:"resourceName,omitempty"`
+ UserID *string `json:"userId,omitempty"`
+ Username *string `json:"username,omitempty"`
+ EnvironmentID *string `json:"environmentId,omitempty"`
+ Metadata database.JSON `json:"metadata,omitempty"`
}
func (s *EventService) CreateEvent(ctx context.Context, req CreateEventRequest) (*Event, error) {
@@ -75,7 +75,6 @@ func (s *EventService) CreateEvent(ctx context.Context, req CreateEventRequest)
userID, username := normalizeEventActor(req.UserID, req.Username)
eventRecord := &Event{
- ID: req.deduplicationID,
CreatedAt: time.Now(),
Type: req.Type,
Severity: severity,
@@ -91,10 +90,14 @@ func (s *EventService) CreateEvent(ctx context.Context, req CreateEventRequest)
Timestamp: time.Now(),
}
+ if req.deduplicationKey != "" {
+ eventRecord.DeduplicationKey = &req.deduplicationKey
+ }
+
var inserted bool
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- if req.deduplicationID != "" {
- tx = tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "id"}}, DoNothing: true})
+ if req.deduplicationKey != "" {
+ tx = tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "deduplication_key"}}, DoNothing: true})
}
result := tx.Create(eventRecord)
if result.Error != nil {
diff --git a/backend/internal/federated/service.go b/backend/internal/federated/service.go
index a7103f948..456847252 100644
--- a/backend/internal/federated/service.go
+++ b/backend/internal/federated/service.go
@@ -56,6 +56,7 @@ func NewFederatedCredentialService(
eventService *event.EventService,
httpClient *http.Client,
keySetManager *oidcjwk.KeySetManager,
+ roleService *role.RoleService,
) *FederatedCredentialService {
if httpClient == nil {
httpClient = httpx.NewHTTPClient(httpxtypes.ClientOptions{Timeout: 15 * time.Second, TLSHandshakeTimeout: 10 * time.Second})
@@ -69,15 +70,11 @@ func NewFederatedCredentialService(
eventService: eventService,
httpClient: httpClient,
keySetManager: keySetManager,
+ roleService: roleService,
keySets: make(map[string]oidc.KeySet),
}
}
-func (s *FederatedCredentialService) WithRoleService(roleService *role.RoleService) *FederatedCredentialService {
- s.roleService = roleService
- return s
-}
-
func (s *FederatedCredentialService) Create(ctx context.Context, callerUserID string, req federatedtypes.CreateFederatedCredential) (*federatedtypes.FederatedCredential, error) {
normalized, err := normalizeCreateFederatedCredentialInternal(req)
if err != nil {
diff --git a/backend/internal/federated/service_test.go b/backend/internal/federated/service_test.go
index 1275b8100..71531c213 100644
--- a/backend/internal/federated/service_test.go
+++ b/backend/internal/federated/service_test.go
@@ -13,7 +13,6 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -32,7 +31,6 @@ import (
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -142,7 +140,7 @@ func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTest
ctx := context.Background()
db := setupFederatedCredentialServiceTestDBInternal(t)
roleSvc := role.NewRoleService(db)
- userSvc := user.NewUserService(db).WithRoleService(roleSvc)
+ userSvc := user.NewUserService(db, roleSvc)
sessionSvc := session.NewSessionService(db)
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
@@ -159,7 +157,7 @@ func setupFederatedCredentialServiceInternal(t *testing.T, issuer *federatedTest
defer cancel()
require.NoError(t, keySetManager.Shutdown(shutdownCtx))
})
- service := NewFederatedCredentialService(db, authSvc, userSvc, settingsSvc, eventSvc, issuer.server.Client(), keySetManager).WithRoleService(roleSvc)
+ service := NewFederatedCredentialService(db, authSvc, userSvc, settingsSvc, eventSvc, issuer.server.Client(), keySetManager, roleSvc)
viewerRole := role.Role{
ID: "role-federated-viewer",
@@ -385,19 +383,9 @@ func TestFederatedCredentialServiceCreateRejectsBareWildcardGlob(t *testing.T) {
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
diff --git a/backend/internal/gitops/backup.go b/backend/internal/gitops/backup.go
index c3ed54eca..ebb672caa 100644
--- a/backend/internal/gitops/backup.go
+++ b/backend/internal/gitops/backup.go
@@ -576,16 +576,18 @@ func (s *GitOpsSyncService) SubscribeProjectFileChanges(ctx context.Context) {
}
// ReconcileInterruptedBackupsOnStartup turns backups left running by a restart into pending failures.
-func (s *GitOpsSyncService) ReconcileInterruptedBackupsOnStartup(ctx context.Context) error {
- err := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).
- Where("mode = ? AND last_sync_status = ?", gitops.SyncModeBackup, backupStatusRunning).
- Updates(map[string]any{
- "last_sync_status": "failed",
- "last_sync_error": "backup was interrupted by a restart",
- "backup_failure_reason": gitops.BackupFailureRepository,
- "backup_pending": true,
- "backup_pending_since": time.Now(),
- }).Error
+func (s *GitOpsSyncService) ReconcileInterruptedBackupsOnStartup(ctx context.Context, protectedIDs ...string) error {
+ query := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("mode = ? AND last_sync_status = ?", gitops.SyncModeBackup, backupStatusRunning)
+ if len(protectedIDs) > 0 {
+ query = query.Where("id NOT IN ?", protectedIDs)
+ }
+ err := query.Updates(map[string]any{
+ "last_sync_status": "failed",
+ "last_sync_error": "backup was interrupted by a restart",
+ "backup_failure_reason": gitops.BackupFailureRepository,
+ "backup_pending": true,
+ "backup_pending_since": time.Now(),
+ }).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return errors.WrapIf(err, "failed to reconcile interrupted git backups")
}
diff --git a/backend/internal/gitops/backup_test.go b/backend/internal/gitops/backup_test.go
index 062c541ba..ff6288433 100644
--- a/backend/internal/gitops/backup_test.go
+++ b/backend/internal/gitops/backup_test.go
@@ -114,7 +114,7 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
eventService := event.NewEventService(db, config.Load(), nil)
- projectService := projectpkg.NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ projectService := projectpkg.NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
repoService := gitrepo.NewGitRepositoryService(db, t.TempDir(), eventService, settingsService)
service := NewGitOpsSyncService(db, repoService, projectService, nil, eventService, settingsService)
@@ -539,6 +539,11 @@ func TestGitOpsBackup_SaveSignalMarksPendingAndRunsAfterDebounce(t *testing.T) {
require.Positive(t, env.scheduler.submitCount())
require.NotEqual(t, firstHead, env.remoteHeadInternal(t, "main"))
+ require.Eventually(t, func() bool {
+ stored := env.reloadInternal(t, syncRecord.ID)
+ return stored.LastSyncStatus != nil && *stored.LastSyncStatus == "success" && !stored.BackupPending
+ }, 5*time.Second, 25*time.Millisecond)
+
stored := env.reloadInternal(t, syncRecord.ID)
require.NotNil(t, stored.LastBackupAt)
assert.False(t, stored.BackupPending)
diff --git a/backend/internal/gitops/gitops_sync.go b/backend/internal/gitops/gitops_sync.go
index ef4c7b93e..2b03bfed2 100644
--- a/backend/internal/gitops/gitops_sync.go
+++ b/backend/internal/gitops/gitops_sync.go
@@ -17,7 +17,6 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -29,11 +28,14 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/types/v2/gitops"
projecttypes "github.com/getarcaneapp/arcane/types/v2/project"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
swarmtypes "github.com/getarcaneapp/arcane/types/v2/swarm"
+ gogit "github.com/go-git/go-git/v5"
+ "github.com/go-git/go-git/v5/plumbing"
"go.getarcane.app/acfs"
kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
@@ -70,6 +72,10 @@ const (
// preparedSyncSource captures the repository data needed by the sync execution
// paths after the source repository has been cloned and validated.
+type syncRevisionInternal struct {
+ Commit string `json:"commit"`
+}
+
type preparedSyncSource struct {
repoPath string
commitHash string
@@ -367,7 +373,7 @@ func NewGitOpsSyncService(db *database.DB, repoService *gitrepo.GitRepositorySer
// called during bootstrap (after the service graph is built) before any per-sync
// jobs are registered. The lifecycle context is used for background sync kicks so
// they outlive the request/bootstrap goroutine that triggered them.
-func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {
+func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *runs.Admission) error {
return s.jobs.SetScheduler(ctx, scheduler, admissionGate)
}
@@ -423,8 +429,32 @@ func (s *GitOpsSyncService) registerSyncJobInternal(ctx context.Context, syncID,
func(ctx context.Context) (schedulertypes.Outcome, error) {
return s.runScheduledSyncInternal(ctx, environmentID, syncID)
},
- func(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
- return jobcontext.ConfirmedTarget(previous, syncID), nil
+ func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ outcome := jobcontext.ConfirmedTarget(previous, syncID)
+ if outcome.Status == schedulertypes.Succeeded {
+ return outcome, nil
+ }
+ record, err := s.getSyncRecordByIDInternal(ctx, environmentID, syncID)
+ if err != nil {
+ return outcome, err
+ }
+ for _, target := range previous.Outcome.Targets {
+ if target.ID != syncID || len(target.RecoveryData) == 0 {
+ continue
+ }
+ var revision syncRevisionInternal
+ if err := json.Unmarshal(target.RecoveryData, &revision); err != nil {
+ return outcome, err
+ }
+ if revision.Commit != "" && record.LastSyncCommit != nil && *record.LastSyncCommit == revision.Commit && record.LastSyncStatus != nil && *record.LastSyncStatus == "success" {
+ target.Status = schedulertypes.Succeeded
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return outcome, err
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: []schedulertypes.TargetOutcome{target}}, nil
+ }
+ }
+ return outcome, nil
},
)
}
@@ -1056,7 +1086,7 @@ func (s *GitOpsSyncService) performSyncAdmittedInternal(ctx context.Context, env
slog.InfoContext(ctx, "GitOps sync already in progress; skipping", "syncId", id)
return &gitops.SyncResult{Success: false, Message: "sync already in progress", SyncedAt: time.Now()}, nil
}
- defer lease.Release()
+ defer lease.Release(ctx)
syncCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
defer cancel()
@@ -1121,6 +1151,10 @@ func (s *GitOpsSyncService) prepareSyncSource(ctx context.Context, sync *project
commitHash = ""
}
+ commitHash, err = pinSyncRevisionInternal(ctx, sync.ID, repoPath, commitHash)
+ if err != nil {
+ return &preparedSyncSource{repoPath: repoPath}, err
+ }
if !s.repoService.FileExists(ctx, repoPath, sync.ComposePath) {
errMsg := "compose file not found: " + sync.ComposePath
return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, "Compose file not found at "+sync.ComposePath, errMsg)
@@ -1177,6 +1211,45 @@ func (s *GitOpsSyncService) prepareSyncSource(ctx context.Context, sync *project
return source, nil
}
+func pinSyncRevisionInternal(ctx context.Context, syncID, repoPath, commitHash string) (string, error) {
+ if previous, ok := jobcontext.Run(ctx); ok {
+ for _, target := range previous.Outcome.Targets {
+ if target.ID != syncID || len(target.RecoveryData) == 0 {
+ continue
+ }
+ var revision syncRevisionInternal
+ if err := json.Unmarshal(target.RecoveryData, &revision); err != nil {
+ return "", err
+ }
+ if revision.Commit != "" && revision.Commit != commitHash {
+ repository, err := gogit.PlainOpen(repoPath)
+ if err != nil {
+ return "", err
+ }
+ tree, err := repository.Worktree()
+ if err != nil {
+ return "", err
+ }
+ if err := tree.Checkout(&gogit.CheckoutOptions{Hash: plumbing.NewHash(revision.Commit)}); err != nil {
+ return "", err
+ }
+ commitHash = revision.Commit
+ }
+ }
+ }
+ revision, err := json.Marshal(syncRevisionInternal{Commit: commitHash})
+ if err != nil {
+ return "", err
+ }
+ if commitHash == "" {
+ return "", errors.New("GitOps source revision is unavailable")
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "gitops_sync", ID: syncID, Status: schedulertypes.Running, RecoveryData: revision}); err != nil {
+ return "", err
+ }
+ return commitHash, nil
+}
+
// performDirectorySync runs the directory-sync path and only triggers a
// redeploy when an already running project's synced contents changed.
func (s *GitOpsSyncService) performDirectorySync(ctx context.Context, sync *projectpkg.GitOpsSync, id string, actor common.User, result *gitops.SyncResult, source *preparedSyncSource) (*gitops.SyncResult, error) {
diff --git a/backend/internal/gitops/service_test.go b/backend/internal/gitops/service_test.go
index b10713f45..7e1173f23 100644
--- a/backend/internal/gitops/service_test.go
+++ b/backend/internal/gitops/service_test.go
@@ -10,7 +10,6 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -23,13 +22,14 @@ import (
git "github.com/getarcaneapp/arcane/backend/v2/pkg/gitutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
"github.com/getarcaneapp/arcane/types/v2/gitops"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
swarmtypes "github.com/getarcaneapp/arcane/types/v2/swarm"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -37,42 +37,28 @@ func setupGitOpsProjectTestDBInternal(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ sqlDB.SetMaxOpenConns(1)
require.NoError(t, db.AutoMigrate(&projectpkg.Project{}, &settings.SettingVariable{}, &imageupdate.ImageUpdateRecord{}, &event.Event{}))
return &database.DB{DB: db}
}
func newGitOpsSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- fxLifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "gitops-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "gitops-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, fxLifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ service, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, service.Stop(context.WithoutCancel(t.Context()))) })
+ }
+ return service, err
}
-func newGitOpsAdmissionGateForTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {
+func newGitOpsAdmissionGateForTestInternal(t testing.TB) *runs.Admission {
t.Helper()
- fxLifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "gitops-test-admission", t.Name())
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, fxLifecycle.Stop(stopCtx))
- })
+ runtime := francistest.New(t)
+ gate := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, gate.Register(runtime))
+ francistest.Start(t, runtime)
return gate
}
@@ -90,19 +76,14 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *dat
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
eventService := event.NewEventService(db, config.Load(), nil)
- projectService := projectpkg.NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ projectService := projectpkg.NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
return NewGitOpsSyncService(db, nil, projectService, nil, eventService, settingsService), db, projectsDir
}
func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), actorRuntime, "gitops-test-admission", "overlap")
- require.NoError(t, err)
-
- key := actors.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: "sync-id"}
+ gate := newGitOpsAdmissionGateForTestInternal(t)
+ key := schedulertypes.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: "sync-id"}
lease, admitted, err := gate.TryAcquire(t.Context(), key)
require.NoError(t, err)
require.True(t, admitted)
@@ -113,12 +94,7 @@ func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *
require.NoError(t, err)
require.False(t, result.Success)
require.Equal(t, "sync already in progress", result.Message)
- lease.Release()
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
+ lease.Release(t.Context())
}
type gitOpsSyncTestSchedulerInternal struct {
diff --git a/backend/internal/gitrepo/module.go b/backend/internal/gitrepo/module.go
index 1a94f72ec..b3d3e5311 100644
--- a/backend/internal/gitrepo/module.go
+++ b/backend/internal/gitrepo/module.go
@@ -3,24 +3,14 @@ package gitrepo
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-type Dependencies struct {
- DB *database.DB
- WorkDir string
- Event *event.EventService
- Settings *settings.SettingsService
-}
-
type Module struct {
service *GitRepositoryService
}
-func New(deps Dependencies) *Module {
- return &Module{service: NewGitRepositoryService(deps.DB, deps.WorkDir, deps.Event, deps.Settings)}
+func New(service *GitRepositoryService) *Module {
+ return &Module{service: service}
}
func (m *Module) Service() *GitRepositoryService {
diff --git a/backend/internal/gitrepo/service_test.go b/backend/internal/gitrepo/service_test.go
index 0608132b4..a192f45e9 100644
--- a/backend/internal/gitrepo/service_test.go
+++ b/backend/internal/gitrepo/service_test.go
@@ -5,10 +5,8 @@ import (
"fmt"
"strings"
"testing"
- "time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -21,7 +19,6 @@ import (
"github.com/stretchr/testify/require"
"go.getarcane.app/kit/pkg/mapping"
"go.getarcane.app/sys/crypto"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -61,21 +58,11 @@ func setupGitRepositoryServiceTestInternal(t *testing.T) (*GitRepositoryService,
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "gitrepo-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "gitrepo-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func createGitRepositoryServiceTestRepoInternal(t *testing.T, svc *GitRepositoryService, req gitops.CreateRepositoryRequest) *GitRepository {
diff --git a/backend/internal/image/module.go b/backend/internal/image/module.go
index 75a656a46..9d8886090 100644
--- a/backend/internal/image/module.go
+++ b/backend/internal/image/module.go
@@ -12,22 +12,18 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-type Dependencies struct {
- Docker *docker.DockerClientService
- ImageUpdate *imageupdate.ImageUpdateService
- Settings *settings.SettingsService
- Build *build.BuildService
- Activity *activity.ActivityService
- Upload *upload.UploadService
-}
-
type Module struct {
- service *ImageService
- deps Dependencies
+ service *ImageService
+ docker *docker.DockerClientService
+ imageUpdate *imageupdate.ImageUpdateService
+ settings *settings.SettingsService
+ build *build.BuildService
+ activity *activity.ActivityService
+ upload *upload.UploadService
}
-func New(service *ImageService, deps Dependencies) *Module {
- return &Module{service: service, deps: deps}
+func New(service *ImageService, dockerService *docker.DockerClientService, imageUpdate *imageupdate.ImageUpdateService, settingsService *settings.SettingsService, buildService *build.BuildService, activityService *activity.ActivityService, uploadService *upload.UploadService) *Module {
+ return &Module{service: service, docker: dockerService, imageUpdate: imageUpdate, settings: settingsService, build: buildService, activity: activityService, upload: uploadService}
}
func (m *Module) Service() *ImageService {
@@ -42,5 +38,5 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
RegisterImages(api, nil, nil, nil, nil, nil, nil, nil, appCtx)
return
}
- RegisterImages(api, m.deps.Docker, m.service, m.deps.ImageUpdate, m.deps.Settings, m.deps.Build, m.deps.Activity, m.deps.Upload, appCtx)
+ RegisterImages(api, m.docker, m.service, m.imageUpdate, m.settings, m.build, m.activity, m.upload, appCtx)
}
diff --git a/backend/internal/image/service_test.go b/backend/internal/image/service_test.go
index 58bbba9dd..698322c55 100644
--- a/backend/internal/image/service_test.go
+++ b/backend/internal/image/service_test.go
@@ -335,7 +335,7 @@ func setupImageServiceAuthTest(t *testing.T) (*ImageService, *database.DB) {
dbWrap := &database.DB{DB: db}
svc := &ImageService{
- registryService: registry.NewContainerRegistryService(dbWrap, nil, kv.NewKVService(dbWrap)),
+ registryService: registry.NewContainerRegistryService(dbWrap, nil, kv.NewKVService(dbWrap), nil),
}
return svc, dbWrap
diff --git a/backend/internal/imagepatch/imagepatch_feature_test.go b/backend/internal/imagepatch/imagepatch_feature_test.go
index 3c18bce94..9976c0dea 100644
--- a/backend/internal/imagepatch/imagepatch_feature_test.go
+++ b/backend/internal/imagepatch/imagepatch_feature_test.go
@@ -8,7 +8,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -21,7 +20,6 @@ import (
"github.com/libtnb/sqlite"
"github.com/moby/moby/client"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -31,21 +29,10 @@ func patchFeatureSettingsInternal(t *testing.T) (*settings.SettingsService, *dat
require.NoError(t, err)
require.NoError(t, gdb.AutoMigrate(&settings.SettingVariable{}, &vulnerability.VulnerabilityScanRecord{}, &vulnerability.VulnerabilityReportRecord{}, &ImagePatchRecord{}))
db := &database.DB{DB: gdb}
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- writes, err := actors.NewExecutor(t.Context(), runtime, "patch-feature-writes", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "patch-feature-effects", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, writes.Stop(ctx))
- require.NoError(t, effects.Stop(ctx))
- require.NoError(t, lifecycle.Stop(ctx))
- })
- svc, err := settings.NewSettingsService(t.Context(), db, writes, effects)
+ svc, err := settings.NewSettingsService(t.Context(), db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
require.NoError(t, err)
return svc, db
}
diff --git a/backend/internal/imageupdate/module.go b/backend/internal/imageupdate/module.go
index e87d18f1b..2af798dba 100644
--- a/backend/internal/imageupdate/module.go
+++ b/backend/internal/imageupdate/module.go
@@ -9,17 +9,13 @@ import (
imagetypes "github.com/getarcaneapp/arcane/types/v2/image"
)
-type Dependencies struct {
- GetUpdateInfoByImageRefs func(context.Context, []string) (map[string]*imagetypes.UpdateInfo, error)
-}
-
type Module struct {
- service *ImageUpdateService
- deps Dependencies
+ service *ImageUpdateService
+ getUpdateInfoByImageRefs func(context.Context, []string) (map[string]*imagetypes.UpdateInfo, error)
}
-func New(service *ImageUpdateService, deps Dependencies) *Module {
- return &Module{service: service, deps: deps}
+func New(service *ImageUpdateService, getUpdateInfo func(context.Context, []string) (map[string]*imagetypes.UpdateInfo, error)) *Module {
+ return &Module{service: service, getUpdateInfoByImageRefs: getUpdateInfo}
}
func (m *Module) Service() *ImageUpdateService {
@@ -34,5 +30,5 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
RegisterImageUpdates(api, nil, nil, appCtx)
return
}
- RegisterImageUpdates(api, m.service, m.deps.GetUpdateInfoByImageRefs, appCtx)
+ RegisterImageUpdates(api, m.service, m.getUpdateInfoByImageRefs, appCtx)
}
diff --git a/backend/internal/imageupdate/service_test.go b/backend/internal/imageupdate/service_test.go
index f0bfa4d8b..68df16ce5 100644
--- a/backend/internal/imageupdate/service_test.go
+++ b/backend/internal/imageupdate/service_test.go
@@ -15,7 +15,6 @@ import (
ref "github.com/distribution/reference"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -43,7 +42,6 @@ import (
kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"go.getarcane.app/updater/labels"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -383,7 +381,7 @@ func newComposeBuildImageUpdateServiceInternal(t *testing.T) (*ImageUpdateServic
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}
eventService := event.NewEventService(db, nil, nil)
@@ -454,7 +452,7 @@ func TestImageUpdateService_CheckMultipleImages_ComposeBuildMissingLocallySkipsR
return client.DistributionInspectResult{}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}
eventService := event.NewEventService(db, nil, nil)
@@ -505,7 +503,7 @@ func newArcaneLocalImageUpdateServiceInternal(t *testing.T, imageExists bool) (*
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}
eventService := event.NewEventService(db, nil, nil)
@@ -898,7 +896,7 @@ func TestImageUpdateService_CheckImageUpdate_UsesRegistryFallback(t *testing.T)
return client.DistributionInspectResult{}, errors.New("error response from daemon: Not Found")
},
}, nil
- }, nil, server.Client())
+ }, nil, nil, server.Client())
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}
eventService := event.NewEventService(db, nil, nil)
@@ -937,7 +935,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.
return client.DistributionInspectResult{}, errors.New("error response from daemon: <html><body><h1>403 Forbidden</h1> Request forbidden by administrative rules. </body></html>")
},
}, nil
- }, nil, server.Client())
+ }, nil, nil, server.Client())
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}
eventService := event.NewEventService(db, nil, nil)
@@ -1022,7 +1020,7 @@ func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInter
return client.DistributionInspectResult{}, ctx.Err()
},
}, nil
- }, nil)
+ }, nil, nil)
parentCtx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond)
defer cancel()
@@ -1062,7 +1060,7 @@ func TestImageUpdateService_CheckMultipleImagesPanicMarksActivityFailedInternal(
panic("registry check exploded")
},
}, nil
- }, nil)
+ }, nil, nil)
svc := NewImageUpdateService(db, settingsService, registryService, &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}, nil, nil, activityService)
@@ -1144,7 +1142,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstA
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}
eventService := event.NewEventService(db, nil, nil)
@@ -1183,7 +1181,7 @@ func TestImageUpdateService_CheckMultipleImages_ReportsNotPulledWhenLocalImageMi
return client.DistributionInspectResult{}, errors.New("error response from daemon: Not Found")
},
}, nil
- }, nil, server.Client())
+ }, nil, nil, server.Client())
dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}
eventService := event.NewEventService(db, nil, nil)
@@ -2029,21 +2027,10 @@ func newImageUpdateTestSettingsServiceInternal(t *testing.T, registryTimeout, do
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}))
dbWrap := &database.DB{DB: db}
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- writes, err := actors.NewExecutor(t.Context(), runtime, "image-update-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "image-update-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, writes.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- service, err := settings.NewSettingsService(ctx, dbWrap, writes, effects)
+ service, err := settings.NewSettingsService(ctx, dbWrap)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, service.Stop(context.Background())) })
+ }
require.NoError(t, err)
return service
}
@@ -2527,7 +2514,7 @@ func TestContainerTagChecksPersistIndependentPoliciesInternal(t *testing.T) {
return &fakeRegistryDaemonClient{distributionInspectFn: func(context.Context, string, client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
return client.DistributionInspectResult{}, errors.New("manifest not found")
}}, nil
- }, nil, registryServer.Client())
+ }, nil, nil, registryServer.Client())
svc := NewImageUpdateService(db, nil, registryService, &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}, nil, nil, nil)
checks, err := svc.checkContainerTagUpdatesInternal(t.Context(), []string{imageRef}, nil)
require.NoError(t, err)
@@ -2626,7 +2613,7 @@ func TestContainerTagChecksSeparateMonitoringFromInstallationInternal(t *testing
defer dockerServer.Close()
registryService := registry.NewContainerRegistryService(db, func(context.Context) (registry.RegistryDaemonClient, error) {
return &fakeRegistryDaemonClient{}, nil
- }, nil, registryServer.Client())
+ }, nil, nil, registryServer.Client())
settingsService := newImageUpdateTestSettingsServiceInternal(t, "5", "5")
// The UI exclusion governs installation only; it must not reach the checker.
require.NoError(t, settingsService.SetStringSetting(t.Context(), "autoUpdateExcludedContainers", "install-excluded"))
@@ -2719,7 +2706,7 @@ func TestContainerTagChecksUseRegistryTagTimeoutInternal(t *testing.T) {
return &fakeRegistryDaemonClient{distributionInspectFn: func(context.Context, string, client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
return client.DistributionInspectResult{}, errors.New("manifest not found")
}}, nil
- }, nil, registryServer.Client()).WithSettingsService(settingsService)
+ }, nil, settingsService, registryServer.Client())
svc := NewImageUpdateService(db, settingsService, registryService, &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}, nil, nil, nil)
start := time.Now()
@@ -2786,7 +2773,7 @@ func TestContainerTagChecksPersistResultsFinishedBeforeScanDeadlineInternal(t *t
return &fakeRegistryDaemonClient{distributionInspectFn: func(context.Context, string, client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
return client.DistributionInspectResult{}, errors.New("manifest not found")
}}, nil
- }, nil, registryServer.Client()).WithSettingsService(settingsService)
+ }, nil, settingsService, registryServer.Client())
svc := NewImageUpdateService(db, settingsService, registryService, &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}, nil, nil, nil)
scanCtx, cancel := context.WithTimeout(t.Context(), time.Second)
diff --git a/backend/internal/job/activity.go b/backend/internal/job/activity.go
index 176d79e5d..3c6907150 100644
--- a/backend/internal/job/activity.go
+++ b/backend/internal/job/activity.go
@@ -3,15 +3,15 @@ package job
import (
"context"
"strings"
+ "uuid"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/types/v2/meta"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/google/uuid"
)
-// ActivityID gives visible runs a stable summary identity before execution.
+// ActivityID assigns a summary identity before a visible run is persisted.
func (s *JobService) ActivityID(run st.Run) string {
// A remotely delivered run already has a summary on its accepting manager.
if run.Trigger == "remote" {
@@ -27,7 +27,7 @@ func (s *JobService) ActivityID(run st.Run) string {
if !visible {
return ""
}
- return uuid.NewSHA1(uuid.NameSpaceURL, []byte("arcane-job/"+run.EnvironmentID+"/"+run.JobID+"/"+run.ID)).String()
+ return uuid.New().String()
}
// SyncRunActivity projects the latest committed run without changing execution state.
@@ -37,8 +37,8 @@ func (s *JobService) SyncRunActivity(ctx context.Context, run st.Run) error {
}
s.activityMu.Lock()
defer s.activityMu.Unlock()
- current, err := s.Queue.Get(ctx, run.EnvironmentID, run.JobID, run.ID)
- if errors.Is(err, queue.ErrRunNotFound) {
+ current, err := s.runs.Get(ctx, run.EnvironmentID, run.JobID, run.ID)
+ if errors.Is(err, runs.ErrRunNotFound) {
return nil
}
if err != nil {
@@ -53,3 +53,43 @@ func (s *JobService) SyncRunActivity(ctx context.Context, run st.Run) error {
}
return s.activity.SyncJobRun(ctx, current, name)
}
+
+func (s *JobService) ReconcileStartupActivities(ctx context.Context, extraProtectedIDs ...string) error {
+ if s.activity == nil {
+ return nil
+ }
+ records, err := s.runs.Records(ctx)
+ if err != nil {
+ return err
+ }
+ protected := append([]string(nil), extraProtectedIDs...)
+ for _, record := range records {
+ for _, run := range record.Runs {
+ if run.Status.Terminal() {
+ continue
+ }
+ if run.ActivityID != "" {
+ protected = append(protected, run.ActivityID)
+ }
+ if run.Outcome.ActivityID != "" {
+ protected = append(protected, run.Outcome.ActivityID)
+ }
+ for _, target := range run.Outcome.Targets {
+ if target.ActivityID != "" {
+ protected = append(protected, target.ActivityID)
+ }
+ }
+ }
+ }
+ if err := s.activity.FailInterruptedBackups(ctx, protected...); err != nil {
+ return err
+ }
+ if _, err := s.activity.FailStaleImageUpdateChecks(ctx); err != nil {
+ return err
+ }
+ if _, err := s.activity.ResolveStaleAutoUpdateActivities(ctx, protected...); err != nil {
+ return err
+ }
+ _, err = s.activity.ResolveOrphanedQueuedActivities(ctx, protected...)
+ return err
+}
diff --git a/backend/internal/job/activity_test.go b/backend/internal/job/activity_test.go
index 90ea1b79a..c75152eb1 100644
--- a/backend/internal/job/activity_test.go
+++ b/backend/internal/job/activity_test.go
@@ -4,6 +4,7 @@ import (
"context"
"testing"
"time"
+ "uuid"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -19,19 +20,20 @@ func newJobActivityTestServiceInternal(t *testing.T) (*JobService, *activity.Act
sqlDB, err := db.DB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
- t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
activities := activity.NewActivityService(db, nil)
- svc := New(Dependencies{DB: db, Config: &config.Config{}, Activity: activities}).Service()
+ svc := NewJobService(db, nil, &config.Config{}, newJobCoordinatorForTestInternal(t, db), nil, nil, activities)
return svc, activities, db
}
func TestJobActivityLifecycle(t *testing.T) {
svc, activities, db := newJobActivityTestServiceInternal(t)
ctx := t.Context()
- run, err := svc.Queue.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "scheduled"})
+ run, err := svc.runs.Submit(ctx, st.Request{JobID: "auto-update", RunID: "b6d679c2-bdf5-4af1-985f-49c468983ba0", Trigger: "scheduled"})
require.NoError(t, err)
- require.NotEmpty(t, run.ActivityID)
+ activityID, err := uuid.Parse(run.ActivityID)
+ require.NoError(t, err)
+ require.NotEqual(t, uuid.Nil(), activityID)
require.Equal(t, "0", run.ActivityEnvironmentID)
check := func(status activitytypes.Status, precise st.RunStatus) {
t.Helper()
@@ -42,11 +44,11 @@ func TestJobActivityLifecycle(t *testing.T) {
require.Equal(t, run.ID, *detail.Activity.BatchID)
}
check(activitytypes.StatusQueued, st.Queued)
- duplicate, err := svc.Queue.Submit(ctx, st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "scheduled"})
+ duplicate, err := svc.runs.Submit(ctx, st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "scheduled"})
require.NoError(t, err)
require.Equal(t, run.ActivityID, duplicate.ActivityID)
for _, status := range []st.RunStatus{st.Running, st.Waiting, st.NeedsAttention} {
- require.NoError(t, svc.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *st.Run) error {
current.Status = status
current.UpdatedAt = time.Now().UTC()
current.Outcome = st.Outcome{Status: status, Message: "operation detail"}
@@ -54,18 +56,18 @@ func TestJobActivityLifecycle(t *testing.T) {
}))
}
check(activitytypes.StatusFailed, st.NeedsAttention)
- _, err = svc.Queue.Retry(ctx, "0", run.JobID, run.ID)
+ _, err = svc.runs.Retry(ctx, "0", run.JobID, run.ID)
require.NoError(t, err)
check(activitytypes.StatusQueued, st.Queued)
_, err = activities.CancelActivity(ctx, "0", run.ActivityID, "operator")
require.ErrorIs(t, err, activity.ErrActivityNotCancelable)
check(activitytypes.StatusQueued, st.Queued)
- require.NoError(t, svc.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *st.Run) error {
current.Status = st.NeedsAttention
current.UpdatedAt = time.Now().UTC()
return nil
}))
- _, err = svc.Queue.Resolve(ctx, "0", run.JobID, run.ID, "operator")
+ _, err = svc.runs.Resolve(ctx, "0", run.JobID, run.ID, "operator")
require.NoError(t, err)
check(activitytypes.StatusCancelled, st.Canceled)
var count int64
@@ -91,7 +93,7 @@ func TestJobActivityVisibilityAndGrouping(t *testing.T) {
{"activity-sweep", "scheduled", "0", false},
} {
t.Run(test.job+"/"+test.trigger+"/"+test.environment, func(t *testing.T) {
- run, err := svc.Queue.Submit(t.Context(), st.Request{JobID: test.job, Trigger: test.trigger, EnvironmentID: test.environment})
+ run, err := svc.runs.Submit(t.Context(), st.Request{JobID: test.job, Trigger: test.trigger, EnvironmentID: test.environment})
require.NoError(t, err)
require.Equal(t, test.visible, run.ActivityID != "")
if test.visible {
@@ -111,28 +113,28 @@ func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {
svc, activities, db := newJobActivityTestServiceInternal(t)
ctx := t.Context()
require.NoError(t, db.Migrator().DropTable(&activity.ActivityMessage{}, &activity.Activity{}))
- run, err := svc.Queue.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "scheduled"})
+ run, err := svc.runs.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "scheduled"})
require.NoError(t, err, "activity failure must not reject accepted work")
require.NotEmpty(t, run.ActivityID)
- require.NoError(t, svc.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *st.Run) error {
current.Status = st.NeedsAttention
current.Outcome = st.Outcome{Message: "interrupted before applying updates"}
current.UpdatedAt = time.Now().UTC()
return nil
}))
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
- restarted := New(Dependencies{DB: db, Config: &config.Config{}, Activity: activities}).Service()
- require.NoError(t, restarted.Queue.Start(ctx))
+ restarted := NewJobService(db, nil, &config.Config{}, svc.runs, nil, nil, activities)
+ require.NoError(t, restarted.runs.Start(ctx))
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
- require.NoError(t, restarted.Queue.Stop(stopCtx))
+ require.NoError(t, restarted.runs.Stop(stopCtx))
})
detail, err := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)
require.NoError(t, err)
require.Equal(t, activitytypes.StatusFailed, detail.Activity.Status)
require.Contains(t, detail.Activity.LatestMessage, "interrupted")
- persisted, err := restarted.Queue.Get(ctx, "0", run.JobID, run.ID)
+ persisted, err := restarted.runs.Get(ctx, "0", run.JobID, run.ID)
require.NoError(t, err)
require.Zero(t, persisted.AttemptCount, "repair must not execute blocked work")
require.Equal(t, run.ActivityID, persisted.ActivityID)
diff --git a/backend/internal/job/execution.go b/backend/internal/job/execution.go
index 51f028c6a..75a637e45 100644
--- a/backend/internal/job/execution.go
+++ b/backend/internal/job/execution.go
@@ -6,16 +6,12 @@ import (
"log/slog"
"net"
"strings"
- "time"
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
- "github.com/getarcaneapp/arcane/backend/v2/internal/common"
- "github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/meta"
@@ -47,7 +43,10 @@ func (s *JobService) Submit(ctx context.Context, request st.Request) (st.Run, er
} else if err := s.validateLocalJobInternal(ctx, request.JobID); err != nil {
return st.Run{}, err
}
- return s.Queue.Submit(ctx, request)
+ if err := s.authorizeRunInternal(ctx, st.Run{Trigger: request.Trigger, RequestedBy: request.RequestedBy, RequestedWithKey: request.RequestedWithKey, EnvironmentID: request.EnvironmentID}); err != nil {
+ return st.Run{}, err
+ }
+ return s.runs.Submit(ctx, request)
}
func (s *JobService) validateLocalJobInternal(ctx context.Context, jobID string) error {
@@ -93,14 +92,7 @@ func (s *JobService) authorizeRunInternal(ctx context.Context, run st.Run) error
if s.roles == nil {
return errors.New("permission resolver unavailable")
}
- var user common.User
- if err := s.db.WithContext(ctx).First(&user, "id = ?", run.RequestedBy).Error; err != nil {
- return err
- }
- if run.RequestedWithKey != "" {
- return s.authorizeKeyInternal(ctx, run)
- }
- permissions, err := s.roles.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), user.ID)
+ permissions, err := s.roles.ResolveExecutionPermissions(ctx, run.RequestedBy, run.RequestedWithKey)
if err != nil {
return err
}
@@ -193,6 +185,13 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.O
if safeJobInternal(run.JobID) {
return s.executeRunInternal(ctx, run)
}
+ if s.scheduler != nil {
+ if job, ok := s.scheduler.GetJob(run.JobID); ok {
+ if reconciler, ok := job.(st.Reconciler); ok {
+ return reconciler.Reconcile(s.runContextInternal(ctx, run), run)
+ }
+ }
+ }
// A successful activity proves only that target, never the entire batch.
for index, target := range run.Outcome.Targets {
if run.JobID == "auto-update" {
@@ -228,12 +227,15 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.O
func (s *JobService) runContextInternal(ctx context.Context, run st.Run) context.Context {
ctx = utils.WithActivityBatchID(ctx, run.ID)
ctx = jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {
- return s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ return s.runs.UpdateRun(ctx, run, func(current *st.Run) error {
if current.Status != st.Running || current.Owner != run.Owner {
- return queue.ErrRunConflict
+ return runs.ErrRunConflict
}
for index := range current.Outcome.Targets {
if current.Outcome.Targets[index].ID == target.ID {
+ if len(target.RecoveryData) == 0 {
+ target.RecoveryData = current.Outcome.Targets[index].RecoveryData
+ }
current.Outcome.Targets[index] = target
return nil
}
@@ -245,42 +247,6 @@ func (s *JobService) runContextInternal(ctx context.Context, run st.Run) context
return ctx
}
-func (s *JobService) authorizeKeyInternal(ctx context.Context, run st.Run) error {
- var key apikey.ApiKey
- if err := s.db.WithContext(ctx).First(&key, "id = ?", run.RequestedWithKey).Error; err != nil {
- return err
- }
- if key.UserID == nil || *key.UserID != run.RequestedBy || (key.ExpiresAt != nil && !key.ExpiresAt.After(time.Now())) {
- return errors.New("requesting API key is no longer valid")
- }
- if key.Kind == apikey.ApiKeyKindPersonal {
- permissions, err := s.roles.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), run.RequestedBy)
- if err != nil {
- return err
- }
- if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {
- return errors.New("requesting user no longer has permission to manage jobs")
- }
- return nil
- }
- var grants []role.ApiKeyPermission
- if err := s.db.WithContext(ctx).Where("api_key_id = ?", key.ID).Find(&grants).Error; err != nil {
- return err
- }
- permissions := authz.NewPermissionSet()
- for _, grant := range grants {
- if grant.EnvironmentID == nil {
- permissions.AddGlobal(grant.Permission)
- } else {
- permissions.AddEnv(*grant.EnvironmentID, grant.Permission)
- }
- }
- if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {
- return errors.New("requesting API key no longer has permission to manage jobs")
- }
- return nil
-}
-
func requiresDockerInternal(jobID string) bool {
switch jobID {
case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan", "image-polling":
diff --git a/backend/internal/job/module.go b/backend/internal/job/module.go
index 36fd4c8fc..e1fc6bfb0 100644
--- a/backend/internal/job/module.go
+++ b/backend/internal/job/module.go
@@ -3,40 +3,16 @@ package job
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/config"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
- "github.com/getarcaneapp/arcane/backend/v2/internal/role"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-type Dependencies struct {
- Activity *activity.ActivityService
- Roles *role.RoleService
- DB *database.DB
- Settings *settings.SettingsService
- Config *config.Config
- Environment *environment.EnvironmentService
-}
-
type Module struct {
service *JobService
environment *environment.EnvironmentService
}
-func New(deps Dependencies) *Module {
- service := NewJobService(deps.DB, deps.Settings, deps.Config)
- service.environment = deps.Environment
- service.roles = deps.Roles
- service.activity = deps.Activity
- if deps.Activity != nil {
- service.Queue.SetObserver(service)
- }
- return &Module{
- service: service,
- environment: deps.Environment,
- }
+func New(service *JobService, environment *environment.EnvironmentService) *Module {
+ return &Module{service: service, environment: environment}
}
func (m *Module) Service() *JobService {
diff --git a/backend/internal/job/remote.go b/backend/internal/job/remote.go
index 090a23bf0..1bd80a7e4 100644
--- a/backend/internal/job/remote.go
+++ b/backend/internal/job/remote.go
@@ -10,7 +10,7 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
"github.com/getarcaneapp/arcane/types/v2/meta"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -85,7 +85,7 @@ func (s *JobService) confirmRemoteRunInternal(ctx context.Context, run, remoteRu
now := time.Now().UTC()
outcome := remoteRun.Outcome
outcome.Status = remoteRun.Status
- if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error {
current.RemoteAccepted = true
current.RemoteDeliveryAttempted = true
current.LastConfirmedAt = &now
@@ -114,7 +114,7 @@ func (s *JobService) acknowledgeRemoteInternal(ctx context.Context, run st.Run,
if response.ID != run.ID || !response.RemoteSettled {
return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; acknowledgement not confirmed"}, nil
}
- if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error { current.RemoteSettled = true; return nil }); err != nil {
+ if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error { current.RemoteSettled = true; return nil }); err != nil {
return st.Outcome{Status: st.Retrying}, err
}
return outcome, nil
@@ -170,8 +170,8 @@ func (s *JobService) ListRemoteJobs(ctx context.Context, environmentID string) (
// RetryRemoteRun queues one explicit retry while retaining the agent's target progress.
func (s *JobService) RetryRemoteRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
- run, err := s.Queue.Get(ctx, environmentID, jobID, runID)
- if errors.Is(err, queue.ErrRunNotFound) {
+ run, err := s.runs.Get(ctx, environmentID, jobID, runID)
+ if errors.Is(err, runs.ErrRunNotFound) {
return s.mutateAgentRunInternal(ctx, environmentID, jobID, runID, "retry")
}
if err != nil {
@@ -180,7 +180,7 @@ func (s *JobService) RetryRemoteRun(ctx context.Context, environmentID, jobID, r
if err := s.authorizeRunInternal(ctx, run); err != nil {
return run, err
}
- err = s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ err = s.runs.UpdateRun(ctx, run, func(current *st.Run) error {
if current.Status != st.Failed && current.Status != st.Partial && current.Status != st.NeedsAttention {
return errors.New("run is not eligible for retry")
}
@@ -242,7 +242,7 @@ func (s *JobService) admitRemoteInternal(ctx context.Context, run st.Run, catalo
return st.Run{}, errRemoteIneligibleInternal
}
// A confirmed 404 permits another delivery of the same deduplicated ID.
- if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error {
current.RemoteDeliveryAttempted = true
return nil
}); err != nil {
@@ -267,7 +267,7 @@ func (s *JobService) retryDeliveryInternal(ctx context.Context, run st.Run, path
if run.RemoteRetryAttempted {
return st.Run{}, errRemoteRetryUncertainInternal
}
- if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error {
current.RemoteRetryAttempted = true
return nil
}); err != nil {
diff --git a/backend/internal/job/remote_history.go b/backend/internal/job/remote_history.go
index 806679bc3..4549d9392 100644
--- a/backend/internal/job/remote_history.go
+++ b/backend/internal/job/remote_history.go
@@ -9,19 +9,19 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
// ListRuns merges agent history with requests accepted by this manager.
func (s *JobService) ListRuns(ctx context.Context, environmentID, jobID string, page, limit int) (st.RunList, error) {
if environmentID == "0" {
- return s.Queue.List(ctx, environmentID, jobID, page, limit)
+ return s.runs.List(ctx, environmentID, jobID, page, limit)
}
if _, err := s.environment.GetEnvironmentByID(ctx, environmentID); err != nil {
return st.RunList{}, err
}
- records, err := s.Queue.Records(ctx)
+ records, err := s.runs.Records(ctx)
if err != nil {
return st.RunList{}, err
}
@@ -84,8 +84,8 @@ func (s *JobService) remoteHistoryInternal(ctx context.Context, environmentID, j
// GetRun prefers the manager's durable delivery record, then queries agent-owned history.
func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
- run, err := s.Queue.Get(ctx, environmentID, jobID, runID)
- if environmentID == "0" || !errors.Is(err, queue.ErrRunNotFound) {
+ run, err := s.runs.Get(ctx, environmentID, jobID, runID)
+ if environmentID == "0" || !errors.Is(err, runs.ErrRunNotFound) {
return run, err
}
path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)
@@ -104,14 +104,14 @@ func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID str
// CancelRun cancels manager admission or explicitly forwards an agent-owned cancellation.
func (s *JobService) CancelRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
- _, err := s.Queue.Get(ctx, environmentID, jobID, runID)
- if environmentID != "0" && errors.Is(err, queue.ErrRunNotFound) {
+ _, err := s.runs.Get(ctx, environmentID, jobID, runID)
+ if environmentID != "0" && errors.Is(err, runs.ErrRunNotFound) {
return s.mutateAgentRunInternal(ctx, environmentID, jobID, runID, "cancel")
}
if err != nil {
return st.Run{}, err
}
- return s.Queue.Cancel(ctx, environmentID, jobID, runID)
+ return s.runs.Cancel(ctx, environmentID, jobID, runID)
}
func (s *JobService) mutateAgentRunInternal(ctx context.Context, environmentID, jobID, runID, action string) (st.Run, error) {
diff --git a/backend/internal/job/remote_resolution.go b/backend/internal/job/remote_resolution.go
index e07647a7a..28c76bd66 100644
--- a/backend/internal/job/remote_resolution.go
+++ b/backend/internal/job/remote_resolution.go
@@ -8,13 +8,13 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {
- local, localErr := s.Queue.Get(ctx, environmentID, jobID, runID)
- if localErr != nil && !errors.Is(localErr, queue.ErrRunNotFound) {
+ local, localErr := s.runs.Get(ctx, environmentID, jobID, runID)
+ if localErr != nil && !errors.Is(localErr, runs.ErrRunNotFound) {
return st.Run{}, localErr
}
if localErr == nil {
@@ -25,7 +25,7 @@ func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID
return local, errors.New("only runs needing attention can be resolved")
}
if !local.RemoteDeliveryAttempted && !local.RemoteAccepted {
- return s.Queue.Resolve(ctx, environmentID, jobID, runID, actor)
+ return s.runs.Resolve(ctx, environmentID, jobID, runID, actor)
}
}
acknowledged, err := s.resolveAgentReviewInternal(ctx, environmentID, jobID, runID, actor)
@@ -40,9 +40,9 @@ func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID
return acknowledged, nil
}
now := time.Now().UTC()
- if err := s.Queue.UpdateRun(ctx, local, func(current *st.Run) error {
+ if err := s.runs.UpdateRun(ctx, local, func(current *st.Run) error {
if current.Status != st.NeedsAttention {
- return queue.ErrRunConflict
+ return runs.ErrRunConflict
}
outcome := acknowledged.Outcome
outcome.Status = acknowledged.Status
@@ -54,7 +54,7 @@ func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID
}); err != nil {
return st.Run{}, err
}
- return s.Queue.Resolve(ctx, environmentID, jobID, runID, acknowledged.Resolution.ResolvedBy)
+ return s.runs.Resolve(ctx, environmentID, jobID, runID, acknowledged.Resolution.ResolvedBy)
}
// resolveAgentReviewInternal confirms ownership, records review, and settles delivery.
diff --git a/backend/internal/job/remote_resolution_test.go b/backend/internal/job/remote_resolution_test.go
index e69bbfeff..a0a2d6c0d 100644
--- a/backend/internal/job/remote_resolution_test.go
+++ b/backend/internal/job/remote_resolution_test.go
@@ -11,10 +11,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
- "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -23,7 +21,7 @@ import (
func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {
db := setupSettingsTestDBInternal(t)
require.NoError(t, db.AutoMigrate(&environment.Environment{}))
- q := queue.New(kv.NewKVService(db), nil, nil)
+ q := newJobCoordinatorForTestInternal(t, db)
local, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: "remote"})
require.NoError(t, err)
require.NoError(t, q.UpdateRun(t.Context(), local, func(run *st.Run) error {
@@ -58,7 +56,7 @@ func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {
}))
defer server.Close()
require.NoError(t, db.Create(&environment.Environment{BaseModel: database.BaseModel{ID: "remote"}, Name: "agent", ApiUrl: server.URL, Enabled: true}).Error)
- svc := &JobService{Queue: q, environment: environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)}
+ svc := &JobService{runs: q, environment: environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)}
ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("remote"))
_, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")
require.ErrorContains(t, err, "agent run must need attention")
diff --git a/backend/internal/job/resolution.go b/backend/internal/job/resolution.go
index 4964897d5..d66c7d8b4 100644
--- a/backend/internal/job/resolution.go
+++ b/backend/internal/job/resolution.go
@@ -40,5 +40,5 @@ func (s *JobService) ResolveRun(ctx context.Context, environmentID, jobID, runID
if environmentID != "0" {
return s.resolveRemoteRunInternal(ctx, environmentID, jobID, runID, actor)
}
- return s.Queue.Resolve(ctx, environmentID, jobID, runID, actor)
+ return s.runs.Resolve(ctx, environmentID, jobID, runID, actor)
}
diff --git a/backend/internal/job/resolution_test.go b/backend/internal/job/resolution_test.go
index 0ff87c00a..f01f94717 100644
--- a/backend/internal/job/resolution_test.go
+++ b/backend/internal/job/resolution_test.go
@@ -4,20 +4,18 @@ import (
"context"
"testing"
- "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/stretchr/testify/require"
)
func TestResolveRunChecksCurrentOperatorPermission(t *testing.T) {
db := setupSettingsTestDBInternal(t)
- svc := &JobService{Queue: queue.New(kv.NewKVService(db), nil, nil)}
- run, err := svc.Queue.Submit(t.Context(), st.Request{JobID: "auto-update", Trigger: "scheduled"})
+ svc := &JobService{runs: newJobCoordinatorForTestInternal(t, db)}
+ run, err := svc.runs.Submit(t.Context(), st.Request{JobID: "auto-update", Trigger: "scheduled"})
require.NoError(t, err)
- require.NoError(t, svc.Queue.UpdateRun(t.Context(), run, func(current *st.Run) error {
+ require.NoError(t, svc.runs.UpdateRun(t.Context(), run, func(current *st.Run) error {
current.Status = st.NeedsAttention
current.Outcome = st.Outcome{Status: st.NeedsAttention, Message: "interrupted"}
return nil
diff --git a/backend/internal/job/run_routes.go b/backend/internal/job/run_routes.go
index 316ed0c44..6e6f23cec 100644
--- a/backend/internal/job/run_routes.go
+++ b/backend/internal/job/run_routes.go
@@ -9,7 +9,7 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -73,7 +73,7 @@ func (h *JobSchedulesHandler) AcknowledgeRun(ctx context.Context, input *jobsche
if err != nil {
return nil, jobHTTPErrorInternal(err)
}
- err = h.jobService.Queue.UpdateRun(ctx, result, func(current *st.Run) error {
+ err = h.jobService.runs.UpdateRun(ctx, result, func(current *st.Run) error {
if !current.Status.Terminal() {
return errors.New("run is not terminal")
}
@@ -113,10 +113,10 @@ func jobHTTPErrorInternal(err error) error {
if errors.As(err, &status) {
return err
}
- if errors.Is(err, queue.ErrRunNotFound) {
+ if errors.Is(err, runs.ErrRunNotFound) {
return huma.Error404NotFound("Job run not found")
}
- if errors.Is(err, queue.ErrRunConflict) {
+ if errors.Is(err, runs.ErrRunConflict) {
return huma.Error409Conflict("Job run state changed")
}
return huma.Error400BadRequest(err.Error())
@@ -133,7 +133,7 @@ func (s *JobService) RetryRun(ctx context.Context, environmentID, jobID, runID s
return st.Run{}, err
}
}
- run, err := s.Queue.Get(ctx, environmentID, jobID, runID)
+ run, err := s.runs.Get(ctx, environmentID, jobID, runID)
if err != nil {
return st.Run{}, err
}
@@ -146,5 +146,5 @@ func (s *JobService) RetryRun(ctx context.Context, environmentID, jobID, runID s
}
}
}
- return s.Queue.Retry(ctx, environmentID, jobID, runID)
+ return s.runs.Retry(ctx, environmentID, jobID, runID)
}
diff --git a/backend/internal/job/runtime_status.go b/backend/internal/job/runtime_status.go
index 6beb36ce2..16b05b8f8 100644
--- a/backend/internal/job/runtime_status.go
+++ b/backend/internal/job/runtime_status.go
@@ -13,8 +13,8 @@ import (
func (s *JobService) addRuntimeStatusInternal(ctx context.Context, environmentID string, jobs *[]jobschedule.JobStatus) error {
runsByJob := make(map[string][]st.Run)
- if s.Queue != nil {
- records, err := s.Queue.Records(ctx)
+ if s.runs != nil {
+ records, err := s.runs.Records(ctx)
if err != nil {
return errors.WrapIf(err, "load durable job status")
}
diff --git a/backend/internal/job/service.go b/backend/internal/job/service.go
index da2101098..5db4b2e3b 100644
--- a/backend/internal/job/service.go
+++ b/backend/internal/job/service.go
@@ -17,7 +17,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
"github.com/getarcaneapp/arcane/types/v2/meta"
@@ -36,7 +36,7 @@ import (
type JobService struct {
activity *activity.ActivityService
activityMu sync.Mutex
- Queue *queue.Queue
+ runs *runs.Coordinator
store *kv.KVService
environment *environment.EnvironmentService
roles *role.RoleService
@@ -55,15 +55,25 @@ type JobService struct {
RunEnvironmentHealthNow func(ctx context.Context) error
}
-func NewJobService(db *database.DB, settings *settings.SettingsService, cfg *config.Config) *JobService {
+func NewJobService(db *database.DB, settings *settings.SettingsService, cfg *config.Config, coordinator *runs.Coordinator, roles *role.RoleService, environment *environment.EnvironmentService, activity *activity.ActivityService) *JobService {
service := &JobService{
- store: kv.NewKVService(db),
- db: db,
- settings: settings,
- cfg: cfg,
- location: cfg.GetLocation(),
+ roles: roles,
+ environment: environment,
+ activity: activity,
+ store: kv.NewKVService(db),
+ runs: coordinator,
+ db: db,
+ settings: settings,
+ cfg: cfg,
+ location: cfg.GetLocation(),
+ }
+
+ if coordinator != nil {
+ coordinator.SetExecutor(service.executeRunInternal, service.reconcileRunInternal)
+ if activity != nil {
+ coordinator.SetObserver(service)
+ }
}
- service.Queue = queue.New(service.store, service.executeRunInternal, service.reconcileRunInternal)
return service
}
@@ -73,9 +83,6 @@ func (s *JobService) SetScheduler(ctx context.Context, scheduler schedulertypes.
}
s.lifecycleCtx = ctx
s.scheduler = scheduler
- if runtime, ok := scheduler.(schedulertypes.JobScheduler); ok {
- runtime.SetDispatcher(s.Queue)
- }
}
func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {
@@ -404,3 +411,5 @@ func (s *JobService) calculateNextRunInternal(schedule string) *time.Time {
now := time.Now().In(location)
return new(sched.Next(now))
}
+
+func (s *JobService) Coordinator() *runs.Coordinator { return s.runs }
diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.go
index e62a27139..cf719d2ae 100644
--- a/backend/internal/job/service_test.go
+++ b/backend/internal/job/service_test.go
@@ -6,16 +6,19 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -24,26 +27,38 @@ func setupSettingsTestDBInternal(t *testing.T) *database.DB {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}))
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ sqlDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
return &database.DB{DB: db}
}
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "job-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "job-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ service, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second)
+ defer cancel()
+ require.NoError(t, service.Stop(stopCtx))
+ })
+ }
+ return service, err
+}
+
+func newJobCoordinatorForTestInternal(t *testing.T, db *database.DB) *runs.Coordinator {
+ t.Helper()
+ runtime := francistest.New(t)
+ coordinator := runs.New(kv.NewKVService(db), runtime.Service(), time.UTC)
+ require.NoError(t, coordinator.Register(runtime))
+ francistest.Start(t, runtime)
+ return coordinator
+}
+
+func newJobServiceForTestInternal(t *testing.T, db *database.DB, settingsService *settings.SettingsService, cfg *config.Config) *JobService {
+ t.Helper()
+ return NewJobService(db, settingsService, cfg, newJobCoordinatorForTestInternal(t, db), nil, nil, nil)
}
func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testing.T) {
@@ -53,7 +68,7 @@ func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testin
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
cfg := jobSvc.GetJobSchedules(ctx)
require.Equal(t, "0 */5 * * * *", cfg.DockerClientRefreshInterval)
@@ -67,7 +82,7 @@ func TestJobService_ListJobs_AnalyticsHeartbeatIsManagedInternally(t *testing.T)
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
jobs, err := jobSvc.ListJobs(ctx)
require.NoError(t, err)
@@ -87,7 +102,7 @@ func TestJobService_ListJobs_IncludesDisabledAutoHealJob(t *testing.T) {
require.NoError(t, err)
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", false))
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
jobs, err := jobSvc.ListJobs(ctx)
require.NoError(t, err)
@@ -103,7 +118,7 @@ func TestJobService_ListJobs_IncludesDockerClientRefreshJob(t *testing.T) {
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
jobs, err := jobSvc.ListJobs(ctx)
require.NoError(t, err)
@@ -130,7 +145,7 @@ func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {
Scheduled: true,
}
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
jobSvc.SetScheduler(ctx, scheduler)
jobs, err := jobSvc.ListJobs(ctx)
require.NoError(t, err)
@@ -148,7 +163,7 @@ func TestJobService_ListJobs_ImageUpdateWatcherIsContinuousAndRespectsEnabled(t
require.NoError(t, err)
require.NoError(t, settingsSvc.SetBoolSetting(ctx, "pollingEnabled", false))
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
jobs, err := jobSvc.ListJobs(ctx)
require.NoError(t, err)
@@ -169,7 +184,7 @@ func TestJobService_UpdateJobSchedules_ReschedulesChangedJob(t *testing.T) {
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
scheduler := newFakeJobSchedulerInternal("auto-update")
jobSvc.SetScheduler(ctx, scheduler)
@@ -188,7 +203,7 @@ func TestJobService_UpdateJobSchedules_DeprecatedPollingIntervalDoesNotReschedul
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
scheduler := newFakeJobSchedulerInternal("auto-update")
jobSvc.SetScheduler(ctx, scheduler)
@@ -211,7 +226,7 @@ func TestJobService_UpdateJobSchedules_UsesLifecycleContextForReschedule(t *test
lifecycleCtx := context.WithValue(context.Background(), lifecycleContextKey{}, true)
requestCtx, cancelRequest := context.WithCancel(context.Background())
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
scheduler := newFakeJobSchedulerInternal("auto-update")
jobSvc.SetScheduler(lifecycleCtx, scheduler)
@@ -236,7 +251,7 @@ func TestJobService_UpdateJobSchedules_RejectsInvalidCronWithoutChangingSetting(
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
scheduler := newFakeJobSchedulerInternal()
jobSvc.SetScheduler(ctx, scheduler)
@@ -255,7 +270,7 @@ func TestJobService_UpdateJobSchedules_UnchangedScheduleDoesNotReschedule(t *tes
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
updated, err := jobSvc.UpdateJobSchedules(ctx, jobschedule.Update{
PollingInterval: new("0 0 * * * *"),
})
@@ -272,7 +287,7 @@ func TestJobService_UpdateJobSchedules_RestoresPreviousScheduleWhenRescheduleFai
require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))
require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
scheduler := newFakeJobSchedulerInternal("auto-update")
scheduler.rescheduleErr = errors.New("scheduler unavailable")
jobSvc.SetScheduler(ctx, scheduler)
@@ -295,7 +310,7 @@ func TestJobService_UpdateJobSchedules_SkipsManagerOnlyJobsInAgentMode(t *testin
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{AgentMode: true})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{AgentMode: true})
scheduler := newFakeJobSchedulerInternal("environment-health")
jobSvc.SetScheduler(ctx, scheduler)
@@ -314,7 +329,7 @@ func TestJobService_UpdateJobSchedules_DelegatesEnvironmentHealthReschedule(t *t
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
scheduler := newFakeJobSchedulerInternal()
jobSvc.SetScheduler(ctx, scheduler)
@@ -337,12 +352,17 @@ func TestJobService_Submit_PersistsImageUpdateWatcherRun(t *testing.T) {
settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
scheduler := newFakeJobSchedulerInternal()
- jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})
jobSvc.SetScheduler(ctx, scheduler)
- run, err := jobSvc.Submit(ctx, schedulertypes.Request{JobID: "image-polling", EnvironmentID: "0", Trigger: "manual"})
+ require.NoError(t, db.AutoMigrate(&common.User{}, &role.Role{}, &role.UserRoleAssignment{}))
+ require.NoError(t, db.Create(&common.User{BaseModel: database.BaseModel{ID: "operator"}, Username: "operator"}).Error)
+ require.NoError(t, db.Create(&role.Role{BaseModel: database.BaseModel{ID: "jobs-manager"}, Name: "Jobs manager", Permissions: database.StringSlice{authz.PermJobsManage}}).Error)
+ require.NoError(t, db.Create(&role.UserRoleAssignment{UserID: "operator", RoleID: "jobs-manager"}).Error)
+ jobSvc.roles = role.NewRoleService(db)
+ run, err := jobSvc.Submit(ctx, schedulertypes.Request{JobID: "image-polling", EnvironmentID: "0", Trigger: "manual", RequestedBy: "operator"})
require.NoError(t, err)
require.Equal(t, schedulertypes.Queued, run.Status)
- persisted, err := jobSvc.Queue.Get(ctx, "0", "image-polling", run.ID)
+ persisted, err := jobSvc.runs.Get(ctx, "0", "image-polling", run.ID)
require.NoError(t, err)
require.Equal(t, run.ID, persisted.ID)
require.Empty(t, scheduler.busWatcherRuns)
diff --git a/backend/internal/oidc/service_test.go b/backend/internal/oidc/service_test.go
index 5e7b95cc5..ca63ecd1a 100644
--- a/backend/internal/oidc/service_test.go
+++ b/backend/internal/oidc/service_test.go
@@ -3,15 +3,12 @@ package oidc
import (
"context"
"testing"
- "time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -112,19 +109,9 @@ func setupSettingsTestDB(t *testing.T) *database.DB {
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
diff --git a/backend/internal/project/lifecycle_test.go b/backend/internal/project/lifecycle_test.go
index 893c7471f..2b1623626 100644
--- a/backend/internal/project/lifecycle_test.go
+++ b/backend/internal/project/lifecycle_test.go
@@ -8,7 +8,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -16,7 +15,6 @@ import (
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -35,22 +33,9 @@ func setupLifecycleTestDB(t *testing.T) *database.DB {
func newLifecycleTestService(t *testing.T, db *database.DB) (*LifecycleService, *settings.SettingsService) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- writes, err := actors.NewExecutor(t.Context(), runtime, "lifecycle-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "lifecycle-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, writes.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- settings, err := settings.NewSettingsService(context.Background(), db, writes, effects)
+ settings, err := settings.NewSettingsService(t.Context(), db)
require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, settings.Stop(context.Background())) })
events := event.NewEventService(db, nil, nil)
return NewLifecycleService(db, settings, events, nil, nil), settings
}
diff --git a/backend/internal/project/module.go b/backend/internal/project/module.go
index 664fef4aa..e35d847a4 100644
--- a/backend/internal/project/module.go
+++ b/backend/internal/project/module.go
@@ -8,12 +8,19 @@ import (
)
type Module struct {
- Service *ProjectService
+ service *ProjectService
activity *activity.ActivityService
}
func New(service *ProjectService, activityService *activity.ActivityService) *Module {
- return &Module{Service: service, activity: activityService}
+ return &Module{service: service, activity: activityService}
+}
+
+func (m *Module) Service() *ProjectService {
+ if m == nil {
+ return nil
+ }
+ return m.service
}
func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {
@@ -21,5 +28,5 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
RegisterProjects(api, nil, nil, appCtx)
return
}
- RegisterProjects(api, m.Service, m.activity, appCtx)
+ RegisterProjects(api, m.service, m.activity, appCtx)
}
diff --git a/backend/internal/project/service.go b/backend/internal/project/service.go
index cb3c9c577..7a0c4603a 100644
--- a/backend/internal/project/service.go
+++ b/backend/internal/project/service.go
@@ -13,7 +13,6 @@ import (
"emperror.dev/errors"
composetypes "github.com/compose-spec/compose-go/v2/types"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -24,6 +23,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
projecttypes "github.com/getarcaneapp/arcane/types/v2/project"
dockerregistry "github.com/moby/moby/api/types/registry"
@@ -67,7 +67,7 @@ type ProjectService struct {
// FilesChanged fires with the project ID after project files are saved
// through Arcane, so Git backups can react without polling.
- FilesChanged *actors.Signal[string]
+ FilesChanged *concurrency.Signal[string]
}
// EnsureGitOpsProjectLinked persists the bidirectional GitOps/project binding
@@ -302,21 +302,23 @@ func (env *projectMetadataEnvInternal) composeFileInternal(projectID string, res
return path, nil
}
-func NewProjectService(db *database.DB, settingsService *settings.SettingsService, eventService *event.EventService, imageService *image.ImageService, dockerService *docker.DockerClientService, buildService buildServiceInternal, lifecycleService *LifecycleService, containerRegistryService *registry.ContainerRegistryService, cfg *config.Config) *ProjectService {
+func NewProjectService(db *database.DB, settingsService *settings.SettingsService, eventService *event.EventService, imageService *image.ImageService, dockerService *docker.DockerClientService, buildService buildServiceInternal, lifecycleService *LifecycleService, containerRegistryService *registry.ContainerRegistryService, cfg *config.Config, kvService *kv.KVService, registryCredentialsProvider func(context.Context) ([]containerregistry.Credential, error)) *ProjectService {
return &ProjectService{
- composeCoordinator: projects.NewCoordinator(projecttypes.ComposeCommands{Stop: composeStopProjectServicesInternal, Up: composeUpProjectServicesInternal, Create: projects.ComposeCreate}),
- db: db,
- settingsService: settingsService,
- eventService: eventService,
- imageService: imageService,
- dockerService: dockerService,
- buildService: buildService,
- lifecycleService: lifecycleService,
- containerRegistryService: containerRegistryService,
- config: cfg,
- parsedCompose: projects.NewParsedComposeCache(),
- metaCache: projects.NewComposeCache[projects.ArcaneComposeMetadata](1024, nil),
- FilesChanged: actors.NewSignal[string](),
+ KVService: kvService,
+ RegistryCredentialsProvider: registryCredentialsProvider,
+ composeCoordinator: projects.NewCoordinator(projecttypes.ComposeCommands{Stop: composeStopProjectServicesInternal, Up: composeUpProjectServicesInternal, Create: projects.ComposeCreate}),
+ db: db,
+ settingsService: settingsService,
+ eventService: eventService,
+ imageService: imageService,
+ dockerService: dockerService,
+ buildService: buildService,
+ lifecycleService: lifecycleService,
+ containerRegistryService: containerRegistryService,
+ config: cfg,
+ parsedCompose: projects.NewParsedComposeCache(),
+ metaCache: projects.NewComposeCache[projects.ArcaneComposeMetadata](1024, nil),
+ FilesChanged: concurrency.NewSignal[string](),
}
}
diff --git a/backend/internal/project/service_test.go b/backend/internal/project/service_test.go
index 2f2e90d5b..9ac70024a 100644
--- a/backend/internal/project/service_test.go
+++ b/backend/internal/project/service_test.go
@@ -21,7 +21,6 @@ import (
composetypes "github.com/compose-spec/compose-go/v2/types"
composeapi "github.com/docker/compose/v5/pkg/api"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -57,7 +56,6 @@ import (
buildtypes "go.getarcane.app/builds/types"
"go.getarcane.app/updater/labels"
updatertypes "go.getarcane.app/updater/types"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -120,21 +118,11 @@ func TestValidateWorkspaceChangesAgainstGitOps(t *testing.T) {
func newSettingsServiceForTestInternal(t *testing.T, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "project-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "project-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
@@ -229,7 +217,7 @@ func TestProjectService_RefreshProjectImageRefs_PersistsBuildMetadata(t *testing
}
require.NoError(t, db.Create(proj).Error)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.refreshProjectImageRefsInternal(ctx, proj)
var saved Project
@@ -266,7 +254,7 @@ func TestProjectService_BackfillProjectImageRefs_RetriesOnlyMissingMetadata(t *t
require.NoError(t, db.Create(invalid).Error)
require.NoError(t, db.Create(regular).Error)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
count, err := svc.BackfillProjectImageRefs(ctx)
require.NoError(t, err)
require.Equal(t, 3, count)
@@ -310,7 +298,7 @@ func setupProjectDestroyTestServiceInternal(t *testing.T) (*ProjectService, *dat
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
eventService := event.NewEventService(db, config.Load(), nil)
- return NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load()), db, projectsDir
+ return NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil), db, projectsDir
}
func TestProjectService_DestroyProject_RemovesFilesWhenRequested(t *testing.T) {
@@ -375,7 +363,7 @@ func TestProjectService_GetProjectFromDatabaseByID(t *testing.T) {
// Setup dependencies
settingsService, _ := newSettingsServiceForTestInternal(t, ctx, db)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
// Create test project
proj := &Project{
@@ -486,7 +474,7 @@ func TestProjectService_CalculateProjectStatus(t *testing.T) {
func TestProjectService_UpdateProjectStatusInternal(t *testing.T) {
db := setupProjectTestDB(t)
ctx := context.Background()
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
proj := &Project{
ID: "p1",
@@ -529,7 +517,7 @@ func TestProjectService_GetProjectByComposeName(t *testing.T) {
t.Run("exact match", func(t *testing.T) {
db := setupProjectTestDB(t)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
proj := &Project{
ID: "p1",
@@ -545,7 +533,7 @@ func TestProjectService_GetProjectByComposeName(t *testing.T) {
t.Run("normalized fallback", func(t *testing.T) {
db := setupProjectTestDB(t)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
proj := &Project{
ID: "p1",
@@ -561,7 +549,7 @@ func TestProjectService_GetProjectByComposeName(t *testing.T) {
t.Run("display name in db, normalized compose label input", func(t *testing.T) {
db := setupProjectTestDB(t)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
display := &Project{
ID: "p2",
@@ -577,7 +565,7 @@ func TestProjectService_GetProjectByComposeName(t *testing.T) {
t.Run("invalidates stale normalized cache entries after deletion", func(t *testing.T) {
db := setupProjectTestDB(t)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
original := &Project{
ID: "p3",
@@ -614,7 +602,7 @@ func TestProjectService_GetProjectByComposeName(t *testing.T) {
t.Run("invalidates stale normalized cache entries after rename", func(t *testing.T) {
db := setupProjectTestDB(t)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
original := &Project{
ID: "p5",
@@ -674,7 +662,7 @@ func TestProjectService_PullProjectImages_UpdatesCurrentImageRecordAfterPull(t *
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService)
- svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load(), nil, nil)
projectPath := createComposeProjectDir(t, projectsDir, "compose-pull")
composeContent := fmt.Sprintf("services:\n app:\n image: %s\n builder:\n build: .\n", imageRef)
@@ -770,7 +758,7 @@ func TestProjectService_EnsureImagesPresent_UpdatesCurrentImageRecordAfterPull(t
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService)
- svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{
ID: "sha256:old-api",
@@ -820,7 +808,7 @@ func TestProjectService_PullImageForService_UpdatesCurrentImageRecordAfterPull(t
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService)
- svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{
ID: "sha256:old-worker",
@@ -883,7 +871,7 @@ func TestProjectService_ComposePullSelectedServicesInternal_ReconcilesOnlyOnSucc
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService)
- svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load(), nil, nil)
projectDef := &composetypes.Project{
Name: "compose-selected",
@@ -987,7 +975,7 @@ func TestProjectService_ComposePullSelectedServicesInternal_LeavesRecordsWhenPul
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, failingServer)}
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil))
- svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load(), nil, nil)
projectDef := &composetypes.Project{
Name: "compose-selected",
@@ -1077,7 +1065,7 @@ func TestProjectService_UpdateProjectServicesHardFailsWhenPullFailsInternal(t *t
return errors.New("compose up should not run")
}
- svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, dockerService, nil, nil, nil, config.Load(), nil, nil)
err = svc.UpdateProjectServices(ctx, projectRecord.ID, []string{"app"}, common.SystemUser, true)
require.Error(t, err)
require.ErrorContains(t, err, "pull updated service images")
@@ -1173,7 +1161,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
return errors.New("compose up failed after assertion")
}
- svc := NewProjectService(db, settingsService, eventService, imageService, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, imageService, dockerService, nil, nil, nil, config.Load(), nil, nil)
err = svc.UpdateProjectServices(ctx, projectRecord.ID, []string{"app"}, common.SystemUser, true)
require.Error(t, err)
assert.True(t, eventService.ShouldSuppressDaemonEvent("container", "replacement", "app", "compose-update-force"), "failed updates retain correlation through rollback grace")
@@ -1252,7 +1240,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenVolumeMigrationPreparationF
t.Setenv("DOCKER_HOST", dockerHostFromProjectRuntimeServerURLInternal(t, dockerServer.URL))
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, dockerServer)}
- svc := NewProjectService(db, settingsService, eventService, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
originalDirName := "Foo"
originalPath := createComposeProjectDir(t, projectsDir, originalDirName)
@@ -1294,7 +1282,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_AppliesVolumeMigrati
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
migration := &fakeProjectVolumeRenameMigrationInternal{}
@@ -1369,7 +1357,7 @@ func TestProjectService_PrepareProjectRenameVolumeMigrationForUpdate_UsesCompose
t.Cleanup(dockerServer.Close)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, dockerServer)}
- svc := NewProjectService(db, settingsService, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
projectPath := filepath.Join(projectsDir, "nginx")
require.NoError(t, os.MkdirAll(projectPath, 0o755))
@@ -1460,7 +1448,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_RollsBackVolumeMigra
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
migration := &fakeProjectVolumeRenameMigrationInternal{}
@@ -1509,7 +1497,7 @@ func TestProjectService_ApplyProjectUpdateWithRenameJournal_SucceedsCommittedRen
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
migration := &fakeProjectVolumeRenameMigrationInternal{
commitErr: errors.New("source cleanup failed"),
@@ -1561,7 +1549,7 @@ func TestProjectService_UpdateProject_ClearsJournalForNonRenameWhenRecoveryDocke
eventService := event.NewEventService(db, nil, nil)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
oldDir := "nginx"
@@ -1628,7 +1616,7 @@ func TestProjectService_UpdateProject_AllowsRenameAfterJournalRecoveryWithoutDoc
eventService := event.NewEventService(db, nil, nil)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
configureProjectRuntimeDockerInternal(t, nil)
@@ -1691,7 +1679,7 @@ func TestProjectService_UpdateProject_RenamesDirectoryWhenNameChanges(t *testing
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
configureProjectRuntimeDockerInternal(t, nil)
originalDirName := "Foo"
@@ -1740,7 +1728,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenTargetDirectoryExists(t *te
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
configureProjectRuntimeDockerInternal(t, nil)
originalDirName := "Foo"
@@ -1787,7 +1775,7 @@ func TestProjectService_UpdateProject_RenameFailsWhenProjectRunning(t *testing.T
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
originalDirName := "Foo"
originalPath := filepath.Join(projectsDir, originalDirName)
@@ -1832,7 +1820,7 @@ func TestProjectService_UpdateProject_RenameRejectsStaleStoppedWhenRuntimeIsRunn
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
configureProjectRuntimeDockerInternal(t, []container.Summary{
{
ID: "app-container",
@@ -1890,7 +1878,7 @@ func TestProjectService_UpdateProject_RenameResolvesUnknownStoppedStatusBeforeVo
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
server := newProjectRuntimeDockerServerInternal(t, nil)
t.Setenv("DOCKER_HOST", dockerHostFromProjectRuntimeServerURLInternal(t, server.URL))
@@ -1948,7 +1936,7 @@ func TestProjectService_UpdateProject_RenameRejectsUnknownWhenRuntimeIsRunning(t
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
server := newProjectRuntimeDockerServerInternal(t, []container.Summary{
{
@@ -2011,7 +1999,7 @@ func TestProjectService_UpdateProject_ValidatesComposeUsingExistingProjectName(t
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "demo"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2054,7 +2042,7 @@ func TestProjectService_UpdateProject_AllowsMissingEnvFileDuringComposeValidatio
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "env-required"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2099,7 +2087,7 @@ func TestProjectService_UpdateProject_EnvRetargetWritesExplicitIdenticalComposeT
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "retarget-explicit"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2153,7 +2141,7 @@ func TestProjectService_UpdateProject_EnvRetargetWithoutComposePayloadPreservesN
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "retarget-omit"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2205,7 +2193,7 @@ func TestProjectService_UpdateProject_EnvRetargetWritesEditedComposeToSelectedBa
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "retarget-edit"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2252,7 +2240,7 @@ func TestProjectService_UpdateProject_AllowsMissingLocalIncludeDuringComposeVali
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "include-new"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2302,7 +2290,7 @@ func TestProjectService_CreateProject_AllowsExternalInclude(t *testing.T) {
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, os.WriteFile(filepath.Join(projectsDir, "shared.yaml"), []byte("services: {}\n"), 0o644))
compose := `include:
@@ -2333,7 +2321,7 @@ func TestProjectService_UpdateProject_AllowsExternalInclude(t *testing.T) {
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "external-include"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2376,7 +2364,7 @@ func TestProjectService_CreateProject_CommitsWorkspaceAndConfigurationTogether(t
settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- svc := NewProjectService(db, settingsService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, config.Load(), nil, nil)
uploadIndex := 0
manifest := projecttypes.CreateProjectWorkspaceManifest{FileChanges: []projecttypes.WorkspaceFileChange{{
Operation: projecttypes.FileOpCreateFile, RelativePath: "config/app.txt", UploadIndex: &uploadIndex,
@@ -2411,7 +2399,7 @@ func TestProjectService_CreateProject_RollsBackInvalidWorkspaceManifest(t *testi
settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
require.NoError(t, err)
- svc := NewProjectService(db, settingsService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, config.Load(), nil, nil)
manifest := projecttypes.CreateProjectWorkspaceManifest{FileChanges: []projecttypes.WorkspaceFileChange{{
Operation: projecttypes.FileOpDelete, RelativePath: "missing.txt",
}}}
@@ -2447,7 +2435,7 @@ func TestProjectService_UpdateProject_UsesExistingEnvFileDuringComposeValidation
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "env-existing"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2497,7 +2485,7 @@ func newProjectServiceForOverrideTestInternal(t *testing.T, dirName, baseCompose
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
projectPath := filepath.Join(projectsDir, dirName)
require.NoError(t, os.MkdirAll(projectPath, 0o755))
@@ -2608,7 +2596,7 @@ func TestProjectService_UpdateProject_UsesProvidedEnvContentDuringComposeValidat
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "env-updated"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2655,7 +2643,7 @@ func TestProjectService_UpdateProject_ReturnsEnvParseErrorDuringComposeValidatio
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "env-invalid"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2700,7 +2688,7 @@ func TestProjectService_UpdateProject_UsesGlobalEnvDuringComposeValidation(t *te
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "global-env-update"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2749,7 +2737,7 @@ func TestProjectService_UpdateProject_DoesNotResolveHostEnvThroughGlobalEnvDurin
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "host-env-guard"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2793,7 +2781,7 @@ func TestProjectService_UpdateProject_DerivesProjectOverrideEnvWhenGitSourceExis
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "override-edit"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2840,7 +2828,7 @@ func TestProjectService_UpdateProject_UnchangedGitEnvLeavesFilesUntouched(t *tes
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "unchanged-git-env"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2937,7 +2925,7 @@ func TestProjectService_UpdateProject_DeletingGitBackedKeyFallsBackToGit(t *test
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "override-delete"
projectPath := filepath.Join(projectsDir, dirName)
@@ -2987,7 +2975,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_MigratesDirectEnvIntoProjectOve
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-migrate"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3038,7 +3026,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_PreservesGitEnvSyntax(t *testin
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-env-syntax"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3100,7 +3088,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_NormalizesStaleCopiedGitOverrid
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-normalize"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3148,7 +3136,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_RemovesLegacyDeletedGitMasks(t
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-delete-mask"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3197,7 +3185,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_RemovesGitEnvSource(t *testing.
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-remove"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3241,7 +3229,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_WritesAndRemovesComposeOverride
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-override"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3292,7 +3280,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_UsesGlobalEnvDuringComposeValid
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-global-env"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3343,7 +3331,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_TolerantOfUndefinedComposeVar(t
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-undefined-var"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3391,7 +3379,7 @@ func TestProjectService_PersistGitSyncEnvFiles_UsesPreparedState(t *testing.T) {
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-prepared-state"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3429,7 +3417,7 @@ func TestProjectService_GetProjectDetails_ReturnsEffectiveEnvContent(t *testing.
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "details-override"
projectPath := filepath.Join(projectsDir, dirName)
@@ -3569,7 +3557,7 @@ func TestProjectService_GetProjectDetails_IncludesUpdateInfo(t *testing.T) {
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
imageService := image.NewImageService(db, nil, nil, nil, nil, nil)
- svc := NewProjectService(db, settingsService, nil, imageService, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, nil, nil, nil, nil, config.Load(), nil, nil)
projectPath := createComposeProjectDir(t, projectsDir, "updates-demo")
require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services:\n app:\n image: nginx:latest\n"), 0o644))
@@ -3668,7 +3656,7 @@ func TestProjectService_GetProjectDetails_RefreshesRuntimeStatusWithoutRuntimeSe
}
require.NoError(t, db.Create(projectRecord).Error)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
details, err := svc.GetProjectDetails(ctx, projectRecord.ID, projecttypes.DetailsOptions{})
require.NoError(t, err)
@@ -3720,7 +3708,7 @@ func TestProjectService_GetProjectDetails_PopulatesRuntimeServicesFromComposePs(
}
require.NoError(t, db.Create(projectRecord).Error)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
details, err := svc.GetProjectDetails(ctx, projectRecord.ID, projecttypes.DetailsOptions{IncludeRuntimeServices: true})
require.NoError(t, err)
@@ -3778,7 +3766,7 @@ func TestProjectService_ListProjects_FiltersByUpdateStatus(t *testing.T) {
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
imageService := image.NewImageService(db, nil, nil, nil, nil, nil)
- svc := NewProjectService(db, settingsService, nil, imageService, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, imageService, nil, nil, nil, nil, config.Load(), nil, nil)
updatedPath := createComposeProjectDir(t, projectsDir, "updated-demo")
require.NoError(t, os.WriteFile(filepath.Join(updatedPath, "compose.yaml"), []byte("services:\n app:\n image: nginx:latest\n"), 0o644))
@@ -4042,7 +4030,7 @@ func TestProjectService_ListProjects_FiltersArchivedProjects(t *testing.T) {
ArchivedAt: new(time.Now().UTC()),
}).Error)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
items, page, err := svc.ListProjects(ctx, pagination.QueryParams{
Limit: -1,
@@ -4110,7 +4098,7 @@ func TestProjectService_ArchiveProject_RequiresStoppedProject(t *testing.T) {
},
})
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
err = svc.ArchiveProject(ctx, "project-running", common.User{ID: "user-1", Username: "tester"})
require.Error(t, err)
require.ErrorIs(t, err, common.ErrProjectMustBeStopped)
@@ -4141,7 +4129,7 @@ func TestProjectService_ArchiveProject_TogglesArchiveFlag(t *testing.T) {
configureProjectRuntimeDockerInternal(t, nil)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
user := common.User{ID: "user-1", Username: "tester"}
require.NoError(t, svc.ArchiveProject(ctx, "project-stopped", user))
@@ -4187,7 +4175,7 @@ func TestProjectService_ArchiveProject_LiveVerificationErrorPolicy(t *testing.T)
t.Setenv("DOCKER_HOST", "tcp://127.0.0.1:1")
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
user := common.User{ID: "user-1", Username: "tester"}
err = svc.ArchiveProject(ctx, "project-unreachable", user)
@@ -4377,7 +4365,7 @@ func TestProjectService_ListProjects_WithDerivedStatusFilter_AllowsAllPageSizeSe
}).Error)
}
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
configureProjectRuntimeDockerInternal(t, nil)
items, page, err := svc.ListProjects(ctx, pagination.QueryParams{
@@ -4420,7 +4408,7 @@ func TestProjectService_DeployProject_StopsOnBuildPreparationError(t *testing.T)
require.NoError(t, db.Create(proj).Error)
buildSvc := testBuildBuilder{err: errors.New("boom build")}
- svc := NewProjectService(db, settingsService, nil, nil, nil, buildSvc, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, buildSvc, nil, nil, config.Load(), nil, nil)
err = svc.DeployProject(ctx, "p1", common.User{ID: "u1", Username: "tester"}, nil)
require.Error(t, err)
@@ -4462,7 +4450,7 @@ func TestProjectService_DeployProject_BuildsGeneratedImageWithoutPull(t *testing
require.NoError(t, db.Create(proj).Error)
buildSvc := testBuildBuilder{err: errors.New("boom build")}
- svc := NewProjectService(db, settingsService, nil, nil, nil, buildSvc, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, buildSvc, nil, nil, config.Load(), nil, nil)
err = svc.DeployProject(ctx, proj.ID, common.User{ID: "u1", Username: "tester"}, nil)
require.Error(t, err)
@@ -4488,7 +4476,7 @@ func TestProjectService_SyncProjectsFromFileSystem_IgnoresSymlinkedProjectDirsWh
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
require.NoError(t, settingsService.SetStringSetting(ctx, "followProjectSymlinks", "false"))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -4514,7 +4502,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DetectsSymlinkedProjectDirsWh
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
require.NoError(t, settingsService.SetStringSetting(ctx, "followProjectSymlinks", "true"))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -4539,7 +4527,7 @@ func TestProjectService_CountProjectFolders_RespectsFollowProjectSymlinks(t *tes
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, settingsService.SetStringSetting(ctx, "followProjectSymlinks", "false"))
count, err := svc.countProjectFolders(ctx)
@@ -4565,7 +4553,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversNestedProjectsAndRel
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, page, err := svc.ListProjects(ctx, pagination.QueryParams{
@@ -4599,7 +4587,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RespectsConfiguredScanMaxDept
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -4646,7 +4634,7 @@ services:
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, page, err := svc.ListProjects(ctx, pagination.QueryParams{
@@ -4674,7 +4662,7 @@ func TestProjectService_CountProjectFolders_RecursivelyCountsNestedProjects(t *t
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
count, err := svc.countProjectFolders(ctx)
require.NoError(t, err)
@@ -4695,7 +4683,7 @@ func TestProjectService_CountProjectFolders_RespectsConfiguredScanMaxDepth(t *te
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
count, err := svc.countProjectFolders(ctx)
require.NoError(t, err)
@@ -4714,7 +4702,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(t
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -4754,7 +4742,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PrunesLeakedScratchRow(t *tes
Path: scratchPath,
}).Error)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -4790,7 +4778,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectsWhenDirector
}).Error)
}
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -4810,7 +4798,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectWithAmbiguous
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -4849,7 +4837,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesProjectsBeyondReducedS
// Initial sync at the default scan depth discovers both the top-level and
// the nested project, persisting them to the database.
- defaultSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ defaultSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, defaultSvc.SyncProjectsFromFileSystem(ctx))
items, err := defaultSvc.ListAllProjects(ctx)
@@ -4859,7 +4847,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesProjectsBeyondReducedS
// Lowering the scan depth must prune the nested project from the database on
// the next sync, even though its compose file still exists on disk.
t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")
- depthLimitedSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ depthLimitedSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, depthLimitedSvc.SyncProjectsFromFileSystem(ctx))
items, err = depthLimitedSvc.ListAllProjects(ctx)
@@ -4892,7 +4880,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesDBRecordsWhenDirecto
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
before, err := svc.ListAllProjects(ctx)
@@ -4957,7 +4945,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversReadableProjectsDesp
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx), "sync must succeed despite the unreadable nested directory")
after, err := svc.ListAllProjects(ctx)
@@ -4993,7 +4981,7 @@ func TestProjectService_SyncProjectsFromFileSystem_AllowsDuplicateLeafDirectorie
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
var items []Project
@@ -5024,7 +5012,7 @@ func TestProjectService_SyncProjectsFromFileSystem_DetectsNestedSymlinkedProject
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
require.NoError(t, settingsService.SetStringSetting(ctx, "followProjectSymlinks", "true"))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, page, err := svc.ListProjects(ctx, pagination.QueryParams{
@@ -5055,7 +5043,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesSymlinkedProjectsWhenD
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
require.NoError(t, settingsService.SetStringSetting(ctx, "followProjectSymlinks", "true"))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -5085,7 +5073,7 @@ func TestProjectService_SyncProjectsFromFileSystem_RefreshesServiceCountOnCompos
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
var project Project
@@ -5117,7 +5105,7 @@ services:
image: nginx:alpine
`), 0o644))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
var project Project
@@ -5150,7 +5138,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesValidCustomNameWitho
}
require.NoError(t, db.Create(project).Error)
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
var fromDB Project
@@ -5200,7 +5188,7 @@ func TestProjectService_SyncProjectsFromFileSystem_PreservesGitOpsProjectWithCus
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
items, err := svc.ListAllProjects(ctx)
@@ -5250,7 +5238,7 @@ func TestProjectService_GetProjectDetails_UsesGitOpsCustomComposeFilename(t *tes
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
- svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
composeFromContent, envFromContent, _, err := svc.GetProjectContent(ctx, syncProjectID)
require.NoError(t, err)
@@ -5282,7 +5270,7 @@ func TestProjectService_UpdateProject_WritesThroughSymlinkedProjectPath(t *testi
require.NoError(t, settingsService.SetStringSetting(ctx, "followProjectSymlinks", "true"))
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
project := &Project{
ID: "proj-symlink-update",
@@ -5337,7 +5325,7 @@ func TestProjectService_UpdateProject_WritesThroughExternalEnvSymlink(t *testing
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "demo"
project := &Project{
ID: "proj-external-env-symlink-update",
@@ -5396,7 +5384,7 @@ func TestProjectService_UpdateProject_RestoresExternalEnvSymlinkTargetWhenProjec
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "demo"
project := &Project{
ID: "proj-external-env-symlink-rollback",
@@ -5557,7 +5545,7 @@ func TestProjectService_RecoverProjectRenameJournals_RollsBackUncommittedDirecto
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -5612,7 +5600,7 @@ func TestProjectService_RecoverProjectRenameJournals_StartedPhaseSkipsVolumeRoll
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -5676,7 +5664,7 @@ func TestProjectService_RecoverProjectRenameJournals_RelocatesTargetWhenBothPath
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -5740,7 +5728,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsStartedJournalWhenDir
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -5806,7 +5794,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsPreservedTargetJourna
t.Cleanup(server.Close)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -5860,7 +5848,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsCommittedJournal(t *t
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -5901,7 +5889,7 @@ func TestProjectService_FinalizeProjectRenameAfterCommit_ClearsJournalAfterSourc
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := &projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -5943,7 +5931,7 @@ func TestProjectService_FinalizeProjectRenameAfterCommit_KeepsJournalWhenSourceC
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := &projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6029,7 +6017,7 @@ func TestProjectService_RecoverProjectRenameJournals_KeepsJournalWhenDirectoryRo
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6124,7 +6112,7 @@ func TestProjectService_RecoverProjectRenameJournals_CompletesCommittedVolumeJou
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6198,7 +6186,7 @@ func TestProjectService_RecoverProjectRenameJournals_RollsBackCommittedJournalWh
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6309,7 +6297,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsJournalAfterDBRestore
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6387,7 +6375,7 @@ func TestProjectService_RecoverProjectRenameJournals_KeepsRollbackCleanupWhenDoc
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
cleanup := projecttypes.RenameRollbackCleanup{
ProjectID: project.ID,
@@ -6451,7 +6439,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsCommittedJournalWhenS
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6538,7 +6526,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsCommittedJournalAndCl
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6622,7 +6610,7 @@ func TestProjectService_RecoverProjectRenameJournals_MarksSourceCleanupPendingWh
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6708,7 +6696,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsSourceCleanupPendingJ
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6808,7 +6796,7 @@ func TestProjectService_RecoverProjectRenameJournals_RollsBackSourceCleanupPendi
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6899,7 +6887,7 @@ func TestProjectService_RecoverProjectRenameJournals_KeepsSourceCleanupPendingJo
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -6966,7 +6954,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsStartedJournalWhenDir
require.NoError(t, db.Create(project).Error)
kvService := kv.NewKVService(db)
- svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -7037,7 +7025,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsMissingPathJournalWhe
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -7112,7 +7100,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsJournalWhenRollbackSo
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -7192,7 +7180,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsJournalWhenRollbackTa
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -7278,7 +7266,7 @@ func TestProjectService_RecoverProjectRenameJournals_ClearsJournalWhenTargetPres
kvService := kv.NewKVService(db)
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
- svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, nil, nil, nil, dockerService, nil, nil, nil, config.Load(), nil, nil)
svc.KVService = kvService
journal := projecttypes.RenameJournal{
ProjectID: project.ID,
@@ -7692,7 +7680,7 @@ func TestProjectServiceManualUpdateDiscoversTags(t *testing.T) {
deploymentError := errors.New("deployment reached")
coordinator := &serviceImageCoordinatorInternal{err: deploymentError}
transport := &serviceTagTransportInternal{tags: tt.tags}
- registryService := registry.NewContainerRegistryService(db, nil, nil, &http.Client{Transport: transport})
+ registryService := registry.NewContainerRegistryService(db, nil, nil, nil, &http.Client{Transport: transport})
service := &ProjectService{db: db, settingsService: settingsService, eventService: event.NewEventService(db, nil, nil), composeCoordinator: coordinator, containerRegistryService: registryService}
err = service.UpdateProjectServices(ctx, proj.ID, []string{"app", "selected-digest"}, common.SystemUser, !tt.skipDiscovery)
require.ErrorIs(t, err, deploymentError)
@@ -7781,7 +7769,7 @@ func TestStoppedProjectTagPolicyNeverInheritsSharedDigestCheck(t *testing.T) {
require.NoError(t, db.Create(projectRecord).Error)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{ID: "shared-digest", Repository: "docker.io/library/nginx", Tag: "3.1.0", UpdateType: "digest", CheckTime: time.Now()}).Error)
imageService := image.NewImageService(db, nil, nil, nil, nil, nil)
- service := NewProjectService(db, settingsService, nil, imageService, nil, nil, nil, nil, config.Load())
+ service := NewProjectService(db, settingsService, nil, imageService, nil, nil, nil, nil, config.Load(), nil, nil)
detail, err := service.GetProjectDetails(t.Context(), projectRecord.ID, projecttypes.AllDetails())
require.NoError(t, err)
require.Equal(t, "unknown", detail.UpdateInfo.Status)
diff --git a/backend/internal/project/service_unix_test.go b/backend/internal/project/service_unix_test.go
index 072f70d6a..26beca368 100644
--- a/backend/internal/project/service_unix_test.go
+++ b/backend/internal/project/service_unix_test.go
@@ -37,7 +37,7 @@ func TestProjectService_ApplyGitSyncProjectFiles_TolerantOfPermissionLockedEnv(t
require.NoError(t, err)
eventService := event.NewEventService(db, nil, nil)
- svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, eventService, nil, nil, nil, nil, nil, config.Load(), nil, nil)
dirName := "git-sync-locked-env"
projectPath := filepath.Join(projectsDir, dirName)
@@ -170,7 +170,7 @@ func newEnvDirectoryProjectInternal(t *testing.T, id string) (*ProjectService, *
require.NoError(t, err)
require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
- svc := NewProjectService(db, settingsService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, config.Load())
+ svc := NewProjectService(db, settingsService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, config.Load(), nil, nil)
projectPath := createComposeProjectDir(t, projectsDir, id)
envDir := filepath.Join(projectPath, ".env")
diff --git a/backend/internal/registry/browse_test.go b/backend/internal/registry/browse_test.go
index 1ef815917..b60940753 100644
--- a/backend/internal/registry/browse_test.go
+++ b/backend/internal/registry/browse_test.go
@@ -93,7 +93,7 @@ func TestContainerRegistryService_ListRepositoriesInternal(t *testing.T) {
writeImageInternal(t, host+"/other/tool:1.0", img)
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
ctx := context.Background()
id := createBrowseTestRegistryInternal(t, db, host, "", "")
@@ -120,7 +120,7 @@ func TestContainerRegistryService_ListRepositoryTagsSingleImageInternal(t *testi
writeImageInternal(t, host+"/team/api:2.0", img)
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
id := createBrowseTestRegistryInternal(t, db, host, "", "")
tags, page, err := svc.ListRepositoryTags(context.Background(), id, "team/api", pagination.QueryParams{Params: pagination.Params{Start: 0, Limit: 1}})
@@ -164,7 +164,7 @@ func TestContainerRegistryService_ListRepositoryTagsIndexInternal(t *testing.T)
require.NoError(t, remote.WriteIndex(ref, index))
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
id := createBrowseTestRegistryInternal(t, db, host, "", "")
tags, _, err := svc.ListRepositoryTags(context.Background(), id, "team/api", browseParamsInternal(""))
@@ -191,7 +191,7 @@ func TestContainerRegistryService_DeleteRepositoryTagInternal(t *testing.T) {
writeImageInternal(t, host+"/team/api:2.0", platformImageInternal(t, v1.Platform{OS: "linux", Architecture: "amd64"}, time.Now()))
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
id := createBrowseTestRegistryInternal(t, db, host, "", "")
ctx := context.Background()
@@ -227,7 +227,7 @@ func TestContainerRegistryService_BrowseUsesStoredCredentialsInternal(t *testing
remote.WithAuth(&authn.Basic{Username: "robot", Password: "secret"}))
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
ctx := context.Background()
id := createBrowseTestRegistryInternal(t, db, host, "robot", "secret")
@@ -245,7 +245,7 @@ func TestContainerRegistryService_BrowseErrorsInternal(t *testing.T) {
writeImageInternal(t, host+"/team/api:1.0", platformImageInternal(t, v1.Platform{OS: "linux", Architecture: "amd64"}, time.Now()))
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
id := createBrowseTestRegistryInternal(t, db, host, "", "")
namespacedID := createBrowseTestRegistryInternal(t, db, "http://"+host+"/team/", "", "")
diff --git a/backend/internal/registry/module.go b/backend/internal/registry/module.go
index 97a1ad783..c8fd046a3 100644
--- a/backend/internal/registry/module.go
+++ b/backend/internal/registry/module.go
@@ -6,30 +6,15 @@ import (
"context"
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
- "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-type Dependencies struct {
- DB *database.DB
- Docker *docker.DockerClientService
- KV *kv.KVService
- Settings *settings.SettingsService
- SyncRemoteRegistries func(context.Context) error
-}
-
type Module struct {
service *ContainerRegistryService
handler *ContainerRegistryHandler
}
-func New(deps Dependencies) *Module {
- service := NewContainerRegistryService(deps.DB, func(ctx context.Context) (RegistryDaemonClient, error) {
- return deps.Docker.GetClient(ctx)
- }, deps.KV).WithSettingsService(deps.Settings)
- return &Module{service: service, handler: NewHandler(service, deps.SyncRemoteRegistries)}
+func New(service *ContainerRegistryService, syncRemoteRegistries func(context.Context) error) *Module {
+ return &Module{service: service, handler: NewHandler(service, syncRemoteRegistries)}
}
func (m *Module) Handler() *ContainerRegistryHandler {
diff --git a/backend/internal/registry/service.go b/backend/internal/registry/service.go
index 6b9f3035a..2c1f728d0 100644
--- a/backend/internal/registry/service.go
+++ b/backend/internal/registry/service.go
@@ -84,14 +84,9 @@ type ContainerRegistryService struct {
settingsService *settings.SettingsService
}
-func (s *ContainerRegistryService) WithSettingsService(settingsService *settings.SettingsService) *ContainerRegistryService {
- s.settingsService = settingsService
- return s
-}
-
// NewContainerRegistryService creates a registry service. kvService may be nil
// in tests that do not need pull tracking or rate-limit caching.
-func NewContainerRegistryService(db *database.DB, dockerClient registryDaemonGetter, kvService *kv.KVService, distributionHTTPClients ...*http.Client) *ContainerRegistryService {
+func NewContainerRegistryService(db *database.DB, dockerClient registryDaemonGetter, kvService *kv.KVService, settingsService *settings.SettingsService, distributionHTTPClients ...*http.Client) *ContainerRegistryService {
distributionHTTPClient := &http.Client{Timeout: 30 * time.Second}
if len(distributionHTTPClients) > 0 && distributionHTTPClients[0] != nil {
distributionHTTPClient = distributionHTTPClients[0]
@@ -101,6 +96,7 @@ func NewContainerRegistryService(db *database.DB, dockerClient registryDaemonGet
dockerClient: dockerClient,
distributionHTTPClient: distributionHTTPClient,
kvService: kvService,
+ settingsService: settingsService,
}
backgroundLoader := func(imageRefs []string) (map[string]string, error) {
digests := make(map[string]string, len(imageRefs))
diff --git a/backend/internal/registry/service_label_test.go b/backend/internal/registry/service_label_test.go
index 3b28df680..175c9356c 100644
--- a/backend/internal/registry/service_label_test.go
+++ b/backend/internal/registry/service_label_test.go
@@ -56,7 +56,7 @@ func TestContainerRegistryService_ImageVersionLabelInternal(t *testing.T) {
ociImageVersionLabel: "v2.8.0-next.66",
})
- svc := NewContainerRegistryService(nil, nil, nil)
+ svc := NewContainerRegistryService(nil, nil, nil, nil)
label, err := svc.ImageVersionLabel(context.Background(), imageRef)
require.NoError(t, err)
@@ -75,7 +75,7 @@ func TestContainerRegistryService_ImageVersionLabelMissingLabelInternal(t *testi
imageRef := host + "/getarcaneapp/arcane:next"
pushLabeledImageInternal(t, imageRef, nil)
- svc := NewContainerRegistryService(nil, nil, nil)
+ svc := NewContainerRegistryService(nil, nil, nil, nil)
_, err := svc.ImageVersionLabel(context.Background(), imageRef)
assert.ErrorIs(t, err, ErrNoVersionLabel)
}
@@ -84,7 +84,7 @@ func TestContainerRegistryService_ImageVersionLabelErrorNotCachedInternal(t *tes
host := newLabelTestRegistryInternal(t)
imageRef := host + "/getarcaneapp/arcane:next"
- svc := NewContainerRegistryService(nil, nil, nil)
+ svc := NewContainerRegistryService(nil, nil, nil, nil)
// The image does not exist yet: the lookup must fail...
_, err := svc.ImageVersionLabel(context.Background(), imageRef)
diff --git a/backend/internal/registry/service_test.go b/backend/internal/registry/service_test.go
index feb6bd551..09c2ddec7 100644
--- a/backend/internal/registry/service_test.go
+++ b/backend/internal/registry/service_test.go
@@ -148,7 +148,7 @@ func newDockerHubRateLimitTestClient(t *testing.T, handler http.HandlerFunc) *ht
}
func TestNewContainerRegistryService_InitializesDistributionHTTPClient(t *testing.T) {
- svc := NewContainerRegistryService(nil, nil, nil)
+ svc := NewContainerRegistryService(nil, nil, nil, nil)
require.NotNil(t, svc.distributionHTTPClient)
}
@@ -157,7 +157,7 @@ func TestContainerRegistryService_GetAllRegistryAuthConfigs_NormalizesHosts(t *t
createTestPullRegistryInternal(t, db, "https://index.docker.io/v1/", "docker-user", "docker-token")
createTestPullRegistryInternal(t, db, "https://GHCR.IO/", "gh-user", "gh-token")
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
authConfigs, err := svc.GetAllRegistryAuthConfigs(context.Background())
require.NoError(t, err)
require.NotNil(t, authConfigs)
@@ -184,7 +184,7 @@ func TestContainerRegistryService_GetAllRegistryAuthConfigs_SkipsInvalidEntries(
createTestPullRegistryInternal(t, db, "https://ghcr.io", "gh-user", " ")
createTestPullRegistryInternal(t, db, "https://registry.example.com", "example-user", "example-token")
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
authConfigs, err := svc.GetAllRegistryAuthConfigs(context.Background())
require.NoError(t, err)
require.NotNil(t, authConfigs)
@@ -203,7 +203,7 @@ func TestContainerRegistryService_GetRegistryAuthForHost_UsesDatabaseCredentials
db := setupContainerRegistryTestDBInternal(t)
createTestPullRegistryInternal(t, db, "https://index.docker.io/v1/", "docker-user", "docker-token")
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
auth, err := svc.GetRegistryAuthForHost(context.Background(), "registry-1.docker.io")
require.NoError(t, err)
require.NotEmpty(t, auth)
@@ -218,7 +218,7 @@ func TestContainerRegistryService_GetRegistryAuthForImage_UsesHostLookup(t *test
db := setupContainerRegistryTestDBInternal(t)
createTestPullRegistryInternal(t, db, "https://ghcr.io", "gh-user", "gh-token")
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
auth, err := svc.GetRegistryAuthForImage(context.Background(), "ghcr.io/getarcaneapp/arcane:latest")
require.NoError(t, err)
require.NotEmpty(t, auth)
@@ -256,7 +256,7 @@ func TestContainerRegistryService_GetRegistryPullUsage_AnonymousDockerHubLimit(t
var tokenCalls int
- svc := NewContainerRegistryService(db, nil, kv.NewKVService(db))
+ svc := NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)
svc.distributionHTTPClient = newDockerHubRateLimitTestClient(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/token":
@@ -309,7 +309,7 @@ func TestContainerRegistryService_GetRegistryPullUsage_UsesDockerHubCredential(t
createTestPullRegistryInternal(t, db, "https://ghcr.io", "gh-user", "gh-token")
var tokenAuth string
- svc := NewContainerRegistryService(db, nil, kv.NewKVService(db))
+ svc := NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)
svc.distributionHTTPClient = newDockerHubRateLimitTestClient(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/token":
@@ -350,7 +350,7 @@ func TestContainerRegistryService_GetRegistryPullUsage_CredentialErrorIsNonFatal
db := setupContainerRegistryTestDBInternal(t)
createTestPullRegistryInternal(t, db, "https://docker.io", "docker-user", "docker-token")
- svc := NewContainerRegistryService(db, nil, kv.NewKVService(db))
+ svc := NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)
svc.distributionHTTPClient = newDockerHubRateLimitTestClient(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/token":
@@ -383,7 +383,7 @@ func TestContainerRegistryService_RecordImagePull_IncrementsObservedRegistryCoun
RegistryType: RegistryTypeGeneric,
}).Error)
- svc := NewContainerRegistryService(db, nil, kv.NewKVService(db))
+ svc := NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)
require.NoError(t, svc.RecordImagePull(context.Background(), "ghcr.io/example/app:latest"))
require.NoError(t, svc.RecordImagePull(context.Background(), "ghcr.io/example/worker:latest"))
@@ -397,7 +397,7 @@ func TestContainerRegistryService_RecordImagePull_IncrementsObservedRegistryCoun
func TestContainerRegistryService_CreateRegistry_RejectsUnsupportedRegistryType(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
_, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "registry.example.com",
@@ -411,7 +411,7 @@ func TestContainerRegistryService_CreateRegistry_RejectsUnsupportedRegistryType(
func TestContainerRegistryService_CreateRegistry_RejectsEmptyUsernameForGeneric(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
_, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -426,7 +426,7 @@ func TestContainerRegistryService_CreateRegistry_RejectsEmptyUsernameForGeneric(
func TestContainerRegistryService_CreateRegistry_RejectsEmptyTokenForGeneric(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
_, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -441,7 +441,7 @@ func TestContainerRegistryService_CreateRegistry_RejectsEmptyTokenForGeneric(t *
func TestContainerRegistryService_CreateRegistry_AcceptsValidGenericCredentials(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
reg, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -455,7 +455,7 @@ func TestContainerRegistryService_CreateRegistry_AcceptsValidGenericCredentials(
func TestContainerRegistryService_UpdateRegistry_RejectsBlankingUsername(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
reg, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -475,7 +475,7 @@ func TestContainerRegistryService_UpdateRegistry_RejectsBlankingUsername(t *test
func TestContainerRegistryService_UpdateRegistry_KeepsExistingTokenWhenNotProvided(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
reg, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -505,7 +505,7 @@ func TestContainerRegistryService_UpdateRegistry_RejectsTargetChangeWhenStoredTo
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
registry, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -534,7 +534,7 @@ func TestContainerRegistryService_UpdateRegistry_RejectsTargetChangeWhenStoredTo
func TestContainerRegistryService_UpdateRegistry_AllowsPathChangeOnSameHostWithoutToken(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
registry, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com/one",
@@ -554,7 +554,7 @@ func TestContainerRegistryService_UpdateRegistry_AllowsPathChangeOnSameHostWitho
func TestContainerRegistryService_UpdateRegistry_AllowsTargetChangeWhenTokenIsResupplied(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
registry, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -577,7 +577,7 @@ func TestContainerRegistryService_UpdateRegistry_AllowsTargetChangeWhenTokenIsRe
func TestContainerRegistryService_UpdateRegistryRejectsECRTargetChangeWithStoredCredentialsInternal(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
registry, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "123456789012.dkr.ecr.us-east-1.amazonaws.com",
@@ -619,7 +619,7 @@ func TestContainerRegistryService_UpdateRegistryRejectsECRTargetChangeWithStored
func TestContainerRegistryService_UpdateRegistry_RejectsChangingRegistryType(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
reg, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -639,7 +639,7 @@ func TestContainerRegistryService_UpdateRegistry_RejectsChangingRegistryType(t *
func TestContainerRegistryService_UpdateRegistry_AllowsSameRegistryType(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
reg, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -664,7 +664,7 @@ func TestContainerRegistryService_SyncRegistries_ClearsGenericTokenWhenManagerSe
var existing ContainerRegistry
require.NoError(t, db.WithContext(context.Background()).First(&existing).Error)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
err := svc.SyncRegistries(context.Background(), []containerregistry.Sync{
{
ID: existing.ID,
@@ -697,7 +697,7 @@ func TestContainerRegistryService_TestRegistry_UsesDockerDaemon(t *testing.T) {
return client.RegistryLoginResult{}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
err := svc.TestRegistry(context.Background(), "https://registry.example.com:5443", "user", "token")
require.NoError(t, err)
@@ -711,7 +711,7 @@ func TestContainerRegistryService_TestRegistry_PropagatesDaemonError(t *testing.
return client.RegistryLoginResult{}, expectedErr
},
}, nil
- }, nil)
+ }, nil, nil)
err := svc.TestRegistry(context.Background(), "registry.example.com", "user", "token")
require.Error(t, err)
@@ -726,7 +726,7 @@ func TestContainerRegistryService_TestRegistry_SkipsLoginForEmptyCredentials(t *
return client.RegistryLoginResult{}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
err := svc.TestRegistry(context.Background(), "registry.example.com", "", "")
require.NoError(t, err)
@@ -750,7 +750,7 @@ func TestContainerRegistryService_InspectImageDigest_AnonymousSuccess(t *testing
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "registry.example.com:5443/team/app:1.2.3", nil)
require.NoError(t, err)
@@ -772,7 +772,7 @@ func TestContainerRegistryService_GetImageDigest_HonorsCallerCancellation(t *tes
return client.DistributionInspectResult{}, ctx.Err()
},
}, nil
- }, nil)
+ }, nil, nil)
t.Cleanup(svc.cache.StopJanitor)
ctx, cancel := context.WithCancel(context.Background())
@@ -822,7 +822,7 @@ func TestContainerRegistryService_InspectImageDigest_UsesStoredDockerHubCredenti
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "registry-1.docker.io/library/nginx:latest", nil)
require.NoError(t, err)
@@ -857,7 +857,7 @@ func TestContainerRegistryService_InspectImageDigest_UsesStoredDockerHubCredenti
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "docker.io/library/registry:3", nil)
require.NoError(t, err)
@@ -891,7 +891,7 @@ func TestContainerRegistryService_InspectImageDigest_UsesStoredCredentialsForNon
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "ghcr.io/getarcaneapp/agent:latest", nil)
require.NoError(t, err)
@@ -927,7 +927,7 @@ func TestContainerRegistryService_InspectImageDigest_UsesStoredCredentialsInstea
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "ghcr.io/getarcaneapp/agent:latest", nil)
require.NoError(t, err)
@@ -958,7 +958,7 @@ func TestContainerRegistryService_InspectImageDigest_FallsBackToAnonymousWhenSto
}, nil
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "ghcr.io/immich-app/immich-server:release", nil)
require.NoError(t, err)
@@ -985,7 +985,7 @@ func TestContainerRegistryService_InspectImageDigest_DoesNotFallBackToAnonymousO
)
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "ghcr.io/getarcaneapp/agent:latest", nil)
require.Error(t, err)
@@ -1008,7 +1008,7 @@ func TestContainerRegistryService_InspectImageDigest_FallbackUsesStoredCredentia
return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
},
}, nil
- }, nil)
+ }, nil, nil)
svc.distributionHTTPClient = &http.Client{
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
// The version check never carries credentials; only manifest requests count.
@@ -1069,7 +1069,7 @@ func TestContainerRegistryService_InspectImageDigest_FallsBackWhenDistributionNo
return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
},
}, nil
- }, nil)
+ }, nil, nil)
svc.distributionHTTPClient = server.Client()
result, err := svc.InspectImageDigest(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
@@ -1108,7 +1108,7 @@ func TestContainerRegistryService_InspectImageDigest_FallsBackWhenDistributionFo
return client.DistributionInspectResult{}, errors.New("Error response from daemon: <html><body><h1>403 Forbidden</h1> Request forbidden by administrative rules. </body></html>")
},
}, nil
- }, nil)
+ }, nil, nil)
svc.distributionHTTPClient = server.Client()
result, err := svc.InspectImageDigest(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
@@ -1184,7 +1184,7 @@ func TestContainerRegistryService_InspectImageDigest_RetriesStoredCredentialsAft
return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
},
}, nil
- }, nil)
+ }, nil, nil)
svc.distributionHTTPClient = server.Client()
result, err := svc.InspectImageDigest(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
@@ -1205,7 +1205,7 @@ func TestContainerRegistryService_InspectImageDigest_DoesNotFallbackOnTLSFailure
return client.DistributionInspectResult{}, errors.New("tls: failed to verify certificate: x509: certificate signed by unknown authority")
},
}, nil
- }, nil)
+ }, nil, nil)
result, err := svc.InspectImageDigest(context.Background(), "registry.example.com/team/app:1.2.3", nil)
require.Error(t, err)
@@ -1226,7 +1226,7 @@ func TestContainerRegistryService_InspectImageDigest_PreservesDaemonAndFallbackE
return client.DistributionInspectResult{}, daemonErr
},
}, nil
- }, nil)
+ }, nil, nil)
svc.distributionHTTPClient = &http.Client{
Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, fallbackErr
@@ -1278,7 +1278,7 @@ func TestContainerRegistryService_InspectImageDigest_PreservesAnonymousUnauthori
return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
},
}, nil
- }, nil)
+ }, nil, nil)
svc.distributionHTTPClient = server.Client()
result, err := svc.InspectImageDigest(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
@@ -1315,7 +1315,7 @@ func fetchRegistry(t *testing.T, db *database.DB, id string) ContainerRegistry {
func TestContainerRegistryService_CreateRegistry_NormalizesAndPersistsRepositoryNames(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
// Entries are trimmed, empties dropped and duplicates removed while
// preserving first-occurrence order.
@@ -1326,7 +1326,7 @@ func TestContainerRegistryService_CreateRegistry_NormalizesAndPersistsRepository
func TestContainerRegistryService_CreateRegistry_RejectsInvalidRepositoryName(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
_, err := svc.CreateRegistry(context.Background(), containerregistry.CreateContainerRegistryRequest{
URL: "https://registry.example.com",
@@ -1338,7 +1338,7 @@ func TestContainerRegistryService_CreateRegistry_RejectsInvalidRepositoryName(t
func TestContainerRegistryService_UpdateRegistry_RepositoryNamesPointerSemantics(t *testing.T) {
db := setupContainerRegistryTestDBInternal(t)
- svc := NewContainerRegistryService(db, nil, nil)
+ svc := NewContainerRegistryService(db, nil, nil, nil)
registry := createRegistryWithRepositoryNames(t, svc, "team", "team/platform")
diff --git a/backend/internal/registry/tags_test.go b/backend/internal/registry/tags_test.go
index a768d253c..615852623 100644
--- a/backend/internal/registry/tags_test.go
+++ b/backend/internal/registry/tags_test.go
@@ -42,7 +42,7 @@ func TestListImageTagsCredentialsInternal(t *testing.T) {
assert.Equal(t, tt.wantPassword, password)
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"name":"library/nginx","tags":["1.2.3","1.2.4"]}`)), Header: http.Header{}}, nil
})}
- svc := NewContainerRegistryService(db, nil, nil, client)
+ svc := NewContainerRegistryService(db, nil, nil, nil, client)
tags, err := svc.ListImageTags(t.Context(), "nginx:1.2.3", tt.external)
require.NoError(t, err)
assert.Equal(t, []string{"1.2.3", "1.2.4"}, tags)
@@ -54,7 +54,7 @@ func TestListImageTagsCredentialsInternal(t *testing.T) {
func TestListImageTagsDoesNotFallBackOnRateLimitInternal(t *testing.T) {
// The registry client retries 429s itself, so count credentialed and anonymous listings rather than requests.
credentialed, anonymous := 0, 0
- svc := NewContainerRegistryService(nil, nil, nil, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ svc := NewContainerRegistryService(nil, nil, nil, nil, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Path == "/v2/" {
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Header: http.Header{}}, nil
}
@@ -73,7 +73,7 @@ func TestListImageTagsDoesNotFallBackOnRateLimitInternal(t *testing.T) {
}
func TestListImageTagsCancellationInternal(t *testing.T) {
- svc := NewContainerRegistryService(nil, nil, nil)
+ svc := NewContainerRegistryService(nil, nil, nil, nil)
ctx, cancel := context.WithCancel(t.Context())
cancel()
_, err := svc.ListImageTags(ctx, "registry.test/team/app:1.0.0", nil)
@@ -94,7 +94,7 @@ func TestListImageTagsRejectedCredentialFallbackInternal(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
// The registry challenges at /v2/ and its token endpoint rejects the stored credential but serves anonymous tokens.
var tokenAuth []string
- svc := NewContainerRegistryService(nil, nil, nil, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ svc := NewContainerRegistryService(nil, nil, nil, nil, &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
response := &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Header: http.Header{}}
switch req.URL.Path {
case "/v2/":
diff --git a/backend/internal/role/module.go b/backend/internal/role/module.go
index a2a4549fe..7637e0d26 100644
--- a/backend/internal/role/module.go
+++ b/backend/internal/role/module.go
@@ -3,22 +3,16 @@ package role
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
)
-// Dependencies are the collaborators the role domain needs.
-type Dependencies struct {
- DB *database.DB
-}
-
// Module owns role persistence and its HTTP surface.
type Module struct {
service *RoleService
}
-// New builds the role domain from its dependencies.
-func New(deps Dependencies) *Module {
- return &Module{service: NewRoleService(deps.DB)}
+// New assembles the role routes around its service.
+func New(service *RoleService) *Module {
+ return &Module{service: service}
}
// Service exposes role operations to authentication and authorization collaborators.
diff --git a/backend/internal/role/service.go b/backend/internal/role/service.go
index c9fe0278b..bc0b43792 100644
--- a/backend/internal/role/service.go
+++ b/backend/internal/role/service.go
@@ -607,26 +607,90 @@ func decodePermissionsJSONInternal(raw string) ([]string, error) {
return out, nil
}
+// ResolveExecutionPermissions revalidates durable work against current users,
+// key ownership, expiry and permissions without using authentication caches.
+func (s *RoleService) ResolveExecutionPermissions(ctx context.Context, userID, keyID string) (*authz.PermissionSet, error) {
+ if userID == "" {
+ return nil, errors.New("requesting user unavailable")
+ }
+ db := s.db.WithContext(ctx)
+ var user common.User
+ if err := db.Select("id").First(&user, "id = ?", userID).Error; err != nil {
+ return nil, errors.WrapIf(err, "failed to load requesting user")
+ }
+ if keyID == "" {
+ return s.ResolveUserPermissionsInDB(ctx, db, userID)
+ }
+ var key struct {
+ ID string
+ Kind string
+ UserID *string
+ EnvironmentID *string
+ ExpiresAt *time.Time
+ }
+ if err := db.Table("api_keys").Select("id, kind, user_id, environment_id, expires_at").Where("id = ?", keyID).Take(&key).Error; err != nil {
+ return nil, errors.WrapIf(err, "failed to load requesting API key")
+ }
+ if key.UserID == nil || *key.UserID != userID || (key.ExpiresAt != nil && !key.ExpiresAt.After(time.Now())) {
+ return nil, errors.New("requesting API key is no longer valid")
+ }
+ var permissions *authz.PermissionSet
+ var err error
+ switch key.Kind {
+ case "personal":
+ permissions, err = s.ResolveUserPermissionsInDB(ctx, db, userID)
+ case "scoped":
+ permissions, err = s.resolveApiKeyPermissionsInDBInternal(db, keyID)
+ default:
+ return nil, errors.New("requesting API key kind is invalid")
+ }
+ if err != nil {
+ return nil, err
+ }
+ if key.EnvironmentID == nil {
+ return permissions, nil
+ }
+ if *key.EnvironmentID == "" {
+ return nil, errors.New("requesting API key environment scope is invalid")
+ }
+ restricted := authz.NewPermissionSet()
+ for permission := range permissions.Global {
+ restricted.AddEnv(*key.EnvironmentID, permission)
+ }
+ for permission := range permissions.PerEnv[*key.EnvironmentID] {
+ restricted.AddEnv(*key.EnvironmentID, permission)
+ }
+ return restricted, nil
+}
+
// ResolveApiKeyPermissions returns the PermissionSet for an API key. Caches
// per-key. Falls back to an empty set (deny-all) if the key has no perms.
func (s *RoleService) ResolveApiKeyPermissions(ctx context.Context, apiKeyID string) (*authz.PermissionSet, error) {
if ps, ok, _ := s.apiKeyCache.Get(apiKeyID); ok {
return ps, nil
}
- var perms []ApiKeyPermission
- if err := s.db.WithContext(ctx).Where("api_key_id = ?", apiKeyID).Find(&perms).Error; err != nil {
+ permissions, err := s.resolveApiKeyPermissionsInDBInternal(s.db.WithContext(ctx), apiKeyID)
+ if err != nil {
+ return nil, err
+ }
+ s.apiKeyCache.Set(apiKeyID, permissions)
+ return permissions, nil
+}
+
+func (s *RoleService) resolveApiKeyPermissionsInDBInternal(db *gorm.DB, apiKeyID string) (*authz.PermissionSet, error) {
+ var grants []ApiKeyPermission
+ if err := db.Where("api_key_id = ?", apiKeyID).Find(&grants).Error; err != nil {
return nil, errors.WrapIf(err, "failed to resolve api key permissions")
}
- ps := authz.NewPermissionSet()
- for _, p := range perms {
- if p.EnvironmentID == nil {
- ps.AddGlobal(p.Permission)
+ permissions := authz.NewPermissionSet()
+ for _, grant := range grants {
+ if grant.EnvironmentID == nil {
+ permissions.AddGlobal(grant.Permission)
} else {
- ps.AddEnv(*p.EnvironmentID, p.Permission)
+ permissions.AddEnv(*grant.EnvironmentID, grant.Permission)
}
}
- s.apiKeyCache.Set(apiKeyID, ps)
- return ps, nil
+ return permissions, nil
}
// SetApiKeyPermissions replaces every permission row on the given API key
diff --git a/backend/internal/role/service_test.go b/backend/internal/role/service_test.go
index be513af6a..6280a5b46 100644
--- a/backend/internal/role/service_test.go
+++ b/backend/internal/role/service_test.go
@@ -372,6 +372,75 @@ type testApiKeyRow struct {
UserID *string `gorm:"column:user_id"`
EnvironmentID *string `gorm:"column:environment_id"`
ManagedBy *string `gorm:"column:managed_by"`
+ ExpiresAt *time.Time
}
func (testApiKeyRow) TableName() string { return "api_keys" }
+
+func TestResolveExecutionPermissions(t *testing.T) {
+ t.Run("personal keys use current user permissions and explicit scope", func(t *testing.T) {
+ db, service := setupUserAndRoleServices(t)
+ user := createTestUser(t, db, "owner", "owner")
+ grantGlobalAdmin(t, service, user.ID)
+ environmentID := "env-1"
+ key := testApiKeyRow{BaseModel: database.BaseModel{ID: "personal"}, Kind: "personal", UserID: &user.ID, EnvironmentID: &environmentID}
+ require.NoError(t, db.Create(&key).Error)
+ permissions, err := service.ResolveExecutionPermissions(t.Context(), user.ID, key.ID)
+ require.NoError(t, err)
+ require.True(t, permissions.Allows(authz.PermJobsManage, environmentID))
+ require.False(t, permissions.Allows(authz.PermJobsManage, "env-2"))
+ require.False(t, permissions.IsGlobalAdmin())
+
+ require.NoError(t, db.Where("user_id = ?", user.ID).Delete(&UserRoleAssignment{}).Error)
+ for _, keyID := range []string{"", key.ID} {
+ permissions, err = service.ResolveExecutionPermissions(t.Context(), user.ID, keyID)
+ require.NoError(t, err)
+ require.False(t, permissions.Allows(authz.PermJobsManage, environmentID))
+ }
+ })
+
+ t.Run("scoped keys bypass cached grants", func(t *testing.T) {
+ db, service := setupUserAndRoleServices(t)
+ user := createTestUser(t, db, "owner", "owner")
+ environmentID := "env-1"
+ key := testApiKeyRow{BaseModel: database.BaseModel{ID: "scoped"}, Kind: "scoped", UserID: &user.ID, EnvironmentID: &environmentID}
+ require.NoError(t, db.Create(&key).Error)
+ require.NoError(t, service.SetApiKeyPermissions(t.Context(), key.ID, []ApiKeyPermission{{Permission: authz.PermJobsManage}}))
+ cached, err := service.ResolveApiKeyPermissions(t.Context(), key.ID)
+ require.NoError(t, err)
+ require.True(t, cached.Allows(authz.PermJobsManage, "env-2"))
+ permissions, err := service.ResolveExecutionPermissions(t.Context(), user.ID, key.ID)
+ require.NoError(t, err)
+ require.True(t, permissions.Allows(authz.PermJobsManage, environmentID))
+ require.False(t, permissions.Allows(authz.PermJobsManage, "env-2"))
+
+ require.NoError(t, db.Where("api_key_id = ?", key.ID).Delete(&ApiKeyPermission{}).Error)
+ permissions, err = service.ResolveExecutionPermissions(t.Context(), user.ID, key.ID)
+ require.NoError(t, err)
+ require.False(t, permissions.Allows(authz.PermJobsManage, environmentID))
+ })
+
+ t.Run("rejects invalid persisted identities", func(t *testing.T) {
+ db, service := setupUserAndRoleServices(t)
+ user := createTestUser(t, db, "owner", "owner")
+ otherOwner := "other"
+ emptyEnvironment := ""
+ expired := time.Now().Add(-time.Minute)
+ for _, key := range []testApiKeyRow{
+ {BaseModel: database.BaseModel{ID: "expired"}, Kind: "personal", UserID: &user.ID, ExpiresAt: &expired},
+ {BaseModel: database.BaseModel{ID: "other-owner"}, Kind: "personal", UserID: &otherOwner},
+ {BaseModel: database.BaseModel{ID: "ownerless"}, Kind: "scoped"},
+ {BaseModel: database.BaseModel{ID: "invalid-kind"}, Kind: "unknown", UserID: &user.ID},
+ {BaseModel: database.BaseModel{ID: "invalid-scope"}, Kind: "personal", UserID: &user.ID, EnvironmentID: &emptyEnvironment},
+ } {
+ require.NoError(t, db.Create(&key).Error)
+ _, err := service.ResolveExecutionPermissions(t.Context(), user.ID, key.ID)
+ require.Error(t, err, key.ID)
+ }
+ _, err := service.ResolveExecutionPermissions(t.Context(), user.ID, "deleted-key")
+ require.Error(t, err)
+ require.NoError(t, db.Delete(user).Error)
+ _, err = service.ResolveExecutionPermissions(t.Context(), user.ID, "")
+ require.Error(t, err)
+ })
+}
diff --git a/backend/internal/s3/module.go b/backend/internal/s3/module.go
index f98a3738b..115eacdd5 100644
--- a/backend/internal/s3/module.go
+++ b/backend/internal/s3/module.go
@@ -5,24 +5,15 @@ import (
"context"
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
)
-type Dependencies struct {
- DB *database.DB
- SyncRemoteDestinations func(context.Context) error
- CheckRemoteReferences func(ctx context.Context, destinationID string) error
-}
-
type Module struct {
- service *S3DestinationService
- deps Dependencies
+ service *S3DestinationService
+ syncRemoteDestinations func(context.Context) error
}
-func New(deps Dependencies) *Module {
- service := NewS3DestinationService(deps.DB)
- service.checkRemoteReferences = deps.CheckRemoteReferences
- return &Module{service: service, deps: deps}
+func New(service *S3DestinationService, syncRemoteDestinations func(context.Context) error) *Module {
+ return &Module{service: service, syncRemoteDestinations: syncRemoteDestinations}
}
func (m *Module) Service() *S3DestinationService {
@@ -37,5 +28,5 @@ func (m *Module) RegisterRoutes(api huma.API) {
RegisterS3Destinations(api, nil, nil)
return
}
- RegisterS3Destinations(api, m.service, m.deps.SyncRemoteDestinations)
+ RegisterS3Destinations(api, m.service, m.syncRemoteDestinations)
}
diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.go
index e15ff744a..961cc1233 100644
--- a/backend/internal/s3/service.go
+++ b/backend/internal/s3/service.go
@@ -33,8 +33,8 @@ type S3DestinationService struct {
checkRemoteReferences func(ctx context.Context, destinationID string) error
}
-func NewS3DestinationService(db *database.DB) *S3DestinationService {
- return &S3DestinationService{db: db}
+func NewS3DestinationService(db *database.DB, checkRemoteReferences func(context.Context, string) error) *S3DestinationService {
+ return &S3DestinationService{db: db, checkRemoteReferences: checkRemoteReferences}
}
func destinationConfigurationInternal(id string, input backuptypes.CreateS3Destination) s3config.Configuration {
diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.go
index 4347e20a2..e8472ade8 100644
--- a/backend/internal/s3/service_test.go
+++ b/backend/internal/s3/service_test.go
@@ -39,7 +39,7 @@ func setupS3DestinationServiceTestInternal(t *testing.T) (*S3DestinationService,
EncryptionKey: "test-encryption-key-for-s3-destination-32bytes",
Environment: "test",
})
- return NewS3DestinationService(&database.DB{DB: gormDB}), gormDB
+ return NewS3DestinationService(&database.DB{DB: gormDB}, nil), gormDB
}
func TestS3DestinationService_CRUD(t *testing.T) {
diff --git a/backend/internal/settings/module.go b/backend/internal/settings/module.go
index ca502fb5b..6d478c750 100644
--- a/backend/internal/settings/module.go
+++ b/backend/internal/settings/module.go
@@ -7,22 +7,17 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-// Dependencies are the handler-side collaborators for the settings domain.
-type Dependencies struct {
- Search *SettingsSearchService
- ProxyRemoteJSON handlerutil.RemoteJSONProxy
- Config *config.Config
-}
-
// Module joins the settings service with its route dependencies.
type Module struct {
- service *SettingsService
- deps Dependencies
+ service *SettingsService
+ search *SettingsSearchService
+ proxyRemoteJSON handlerutil.RemoteJSONProxy
+ config *config.Config
}
-// New builds the settings domain around its initialized actor-backed service.
-func New(service *SettingsService, deps Dependencies) *Module {
- return &Module{service: service, deps: deps}
+// New builds the settings domain around its initialized service.
+func New(service *SettingsService, search *SettingsSearchService, proxyRemoteJSON handlerutil.RemoteJSONProxy, cfg *config.Config) *Module {
+ return &Module{service: service, search: search, proxyRemoteJSON: proxyRemoteJSON, config: cfg}
}
// Service exposes settings operations to collaborating domains.
@@ -39,5 +34,5 @@ func (m *Module) RegisterRoutes(api huma.API) {
RegisterSettings(api, nil, nil, nil, nil)
return
}
- RegisterSettings(api, m.service, m.deps.Search, m.deps.ProxyRemoteJSON, m.deps.Config)
+ RegisterSettings(api, m.service, m.search, m.proxyRemoteJSON, m.config)
}
diff --git a/backend/internal/settings/service.go b/backend/internal/settings/service.go
index 4de0607b3..df5c6b50d 100644
--- a/backend/internal/settings/service.go
+++ b/backend/internal/settings/service.go
@@ -15,18 +15,19 @@ import (
"slices"
"strconv"
"strings"
+ "sync"
"sync/atomic"
"time"
"uuid"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/validation"
"github.com/getarcaneapp/arcane/types/v2/features"
@@ -53,10 +54,14 @@ type SettingsService struct {
// The snapshot is shared: readers must never mutate it.
effectiveConfig atomic.Pointer[Settings]
envOverrides []settingsEnvOverride
- writes *actors.Executor
- effects *actors.Executor
+ writes sync.Mutex
+ effectsMu sync.Mutex
+ effects []func()
+ effectsWake chan struct{}
+ effectsDone chan struct{}
+ effectsClosed bool
lifecycleCtx context.Context
- changes *actors.Signal[[]libarcane.SettingUpdate]
+ changes *concurrency.Signal[[]libarcane.SettingUpdate]
}
type settingsUpdateResultInternal struct {
@@ -71,22 +76,16 @@ type settingsEnvOverride struct {
value string
}
-func NewSettingsService(ctx context.Context, db *database.DB, writes, effects *actors.Executor) (*SettingsService, error) {
+func NewSettingsService(ctx context.Context, db *database.DB) (*SettingsService, error) {
if ctx == nil {
return nil, errors.New("settings lifecycle context unavailable")
}
- if writes == nil {
- return nil, errors.New("settings write executor unavailable")
- }
- if effects == nil {
- return nil, errors.New("settings effects executor unavailable")
- }
svc := &SettingsService{
db: db,
- writes: writes,
- effects: effects,
+ effectsWake: make(chan struct{}, 1),
+ effectsDone: make(chan struct{}),
lifecycleCtx: ctx,
- changes: actors.NewSignal[[]libarcane.SettingUpdate](),
+ changes: concurrency.NewSignal[[]libarcane.SettingUpdate](),
}
svc.envOverrides = resolveSettingsEnvOverridesInternal()
if len(svc.envOverrides) > 0 {
@@ -103,12 +102,13 @@ func NewSettingsService(ctx context.Context, db *database.DB, writes, effects *a
return nil, errors.WrapIf(err, "failed to setup instance ID")
}
+ go svc.runEffectsInternal()
return svc, nil
}
// SubscribeSettingsChanges invokes callback once when any subscribed key is
// present in a successful update. The callback receives only matching values.
-// Callbacks run serially on the settings-effects actor, outside the write actor.
+// Callbacks run serially on the ordered effects worker, outside the write mutex.
func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func() {
if callback == nil || len(keys) == 0 {
return func() {}
@@ -125,11 +125,7 @@ func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func(
}
}
if len(matching) > 0 {
- _, err := s.effects.Submit(context.WithoutCancel(s.lifecycleCtx), "apply settings change effects", func(context.Context) (actors.NoPayload, error) {
- callback(matching)
- return actors.NoPayload{}, nil
- }, nil)
- if err != nil && s.lifecycleCtx.Err() == nil {
+ if err := s.enqueueEffectInternal(func() { callback(matching) }); err != nil && s.lifecycleCtx.Err() == nil {
slog.ErrorContext(s.lifecycleCtx, "Failed to queue settings change effects", "error", err)
}
}
@@ -137,25 +133,24 @@ func (s *SettingsService) SubscribeSettingsChanges(keys []string, callback func(
}
// NotifySettingsChanges publishes current values for keys through the settings
-// actor. It is used when another settings-table workflow performs persistence.
+// write mutex. It is used when another settings-table workflow performs persistence.
func (s *SettingsService) NotifySettingsChanges(ctx context.Context, keys ...string) error {
- _, err := s.writes.Execute(ctx, "notify settings changes", func(context.Context) ([]libarcane.SettingUpdate, error) {
- updates := make([]libarcane.SettingUpdate, 0, len(keys))
- cfg := s.GetSettingsConfig()
- for _, key := range keys {
- value, _, _, err := cfg.FieldByKey(key)
- if err != nil {
- return nil, errors.WrapIff(err, "load changed setting '%s'", key)
- }
- updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})
- }
- return updates, nil
- }, func(updates []libarcane.SettingUpdate, err error) {
- if err == nil && len(updates) > 0 {
- s.publishSettingsChangesInternal(updates)
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ updates := make([]libarcane.SettingUpdate, 0, len(keys))
+ cfg := s.GetSettingsConfig()
+ for _, key := range keys {
+ value, _, _, err := cfg.FieldByKey(key)
+ if err != nil {
+ return errors.WrapIff(err, "load changed setting '%s'", key)
}
- })
- return err
+ updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})
+ }
+ s.publishSettingsChangesInternal(updates)
+ return nil
}
func (s *SettingsService) GetSettingsConfig() *Settings {
@@ -520,13 +515,16 @@ func (s *SettingsService) UpdateSetting(ctx context.Context, key, value string)
if err := libarcane.ValidateCronSetting(key, value); err != nil {
return errors.WrapIff(err, "invalid cron expression for %s", key)
}
- _, err := s.writes.Execute(ctx, "update setting", func(writeCtx context.Context) (actors.NoPayload, error) {
- if err := s.updateSettingValueNoRefreshInternal(writeCtx, key, value); err != nil {
- return actors.NoPayload{}, err
- }
- return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))
- }, nil)
- return err
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+
+ if err := s.updateSettingValueNoRefreshInternal(ctx, key, value); err != nil {
+ return err
+ }
+ return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))
}
// UpdateSettingValues persists a group of internal setting values atomically
@@ -539,13 +537,16 @@ func (s *SettingsService) UpdateSettingValues(ctx context.Context, updates []lib
}
values = append(values, SettingVariable{Key: update.Key, Value: update.Value})
}
- _, err := s.writes.Execute(ctx, "update setting values", func(writeCtx context.Context) (actors.NoPayload, error) {
- if err := s.persistSettings(writeCtx, values); err != nil {
- return actors.NoPayload{}, err
- }
- return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))
- }, nil)
- return err
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+
+ if err := s.persistSettings(ctx, values); err != nil {
+ return err
+ }
+ return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))
}
func (s *SettingsService) updateSettingValueNoRefreshInternal(ctx context.Context, key, value string) error {
@@ -567,13 +568,15 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settingsty
if err := normalization.Normalize(&updates); err != nil {
return nil, err
}
- result, err := s.writes.Execute(ctx, "update settings", func(writeCtx context.Context) (settingsUpdateResultInternal, error) {
- return s.updateSettingsInternal(writeCtx, updates)
- }, func(result settingsUpdateResultInternal, err error) {
- if err == nil && len(result.changes) > 0 {
- s.publishSettingsChangesInternal(result.changes)
- }
- })
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ result, err := s.updateSettingsInternal(ctx, updates)
+ if err == nil {
+ s.publishSettingsChangesInternal(result.changes)
+ }
return result.settings, err
}
@@ -785,77 +788,86 @@ var retiredSettingValuesInternal = map[string][]string{
}
func (s *SettingsService) EnsureDefaultSettings(ctx context.Context) error {
- _, err := s.writes.Execute(ctx, "ensure default settings", func(writeCtx context.Context) (actors.NoPayload, error) {
- defaultSettings := s.getDefaultSettings()
- defaultSettingVars := defaultSettings.ToSettingVariableSlice(SettingVisibilityAll, false)
-
- if err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
- for _, defaultSetting := range defaultSettingVars {
- var existing SettingVariable
- err := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error
-
- switch {
- case errors.Is(err, gorm.ErrRecordNotFound):
- if err := tx.Create(&defaultSetting).Error; err != nil {
- return errors.WrapIff(err, "failed to create default setting %s", defaultSetting.Key)
- }
- case err != nil:
- return errors.WrapIff(err, "failed to check for existing setting %s", defaultSetting.Key)
- case slices.Contains(retiredSettingValuesInternal[defaultSetting.Key], existing.Value):
- if err := tx.Model(&SettingVariable{}).Where("key = ?", defaultSetting.Key).Update("value", defaultSetting.Value).Error; err != nil {
- return errors.WrapIff(err, "failed to replace retired default for setting %s", defaultSetting.Key)
- }
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+
+ defaultSettings := s.getDefaultSettings()
+ defaultSettingVars := defaultSettings.ToSettingVariableSlice(SettingVisibilityAll, false)
+
+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ for _, defaultSetting := range defaultSettingVars {
+ var existing SettingVariable
+ err := tx.Where("key = ?", defaultSetting.Key).First(&existing).Error
+
+ switch {
+ case errors.Is(err, gorm.ErrRecordNotFound):
+ if err := tx.Create(&defaultSetting).Error; err != nil {
+ return errors.WrapIff(err, "failed to create default setting %s", defaultSetting.Key)
+ }
+ case err != nil:
+ return errors.WrapIff(err, "failed to check for existing setting %s", defaultSetting.Key)
+ case slices.Contains(retiredSettingValuesInternal[defaultSetting.Key], existing.Value):
+ if err := tx.Model(&SettingVariable{}).Where("key = ?", defaultSetting.Key).Update("value", defaultSetting.Value).Error; err != nil {
+ return errors.WrapIff(err, "failed to replace retired default for setting %s", defaultSetting.Key)
}
}
- return nil
- }); err != nil {
- return actors.NoPayload{}, err
}
- return actors.NoPayload{}, nil
- }, nil)
- return err
+ return nil
+ }); err != nil {
+ return err
+ }
+ return nil
}
func (s *SettingsService) PruneUnknownSettings(ctx context.Context) error {
- _, err := s.writes.Execute(ctx, "prune unknown settings", func(writeCtx context.Context) (actors.NoPayload, error) {
- allowedKeys := allowedSettingKeys()
- if len(allowedKeys) == 0 {
- return actors.NoPayload{}, nil
- }
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
- keys := slices.Collect(maps.Keys(allowedKeys))
+ allowedKeys := allowedSettingKeys()
+ if len(allowedKeys) == 0 {
+ return nil
+ }
- result := s.db.WithContext(writeCtx).Where("key NOT IN ?", keys).Delete(&SettingVariable{})
- if result.Error != nil {
- return actors.NoPayload{}, errors.WrapIf(result.Error, "failed to prune unknown settings")
- }
- if result.RowsAffected > 0 {
- slog.InfoContext(writeCtx, "Pruned unknown settings", "count", result.RowsAffected)
- }
- return actors.NoPayload{}, nil
- }, nil)
- return err
+ keys := slices.Collect(maps.Keys(allowedKeys))
+
+ result := s.db.WithContext(ctx).Where("key NOT IN ?", keys).Delete(&SettingVariable{})
+ if result.Error != nil {
+ return errors.WrapIf(result.Error, "failed to prune unknown settings")
+ }
+ if result.RowsAffected > 0 {
+ slog.InfoContext(ctx, "Pruned unknown settings", "count", result.RowsAffected)
+ }
+ return nil
}
func (s *SettingsService) PersistEnvSettingsIfMissing(ctx context.Context) error {
- _, err := s.writes.Execute(ctx, "persist environment settings", func(writeCtx context.Context) (actors.NoPayload, error) {
- rt := reflect.TypeFor[Settings]()
- appCfg := config.Load()
- isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled
-
- if err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
- for field := range rt.Fields() {
- if err := s.processEnvField(writeCtx, tx, field, isEnvOnlyMode); err != nil {
- return err
- }
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+
+ rt := reflect.TypeFor[Settings]()
+ appCfg := config.Load()
+ isEnvOnlyMode := appCfg.AgentMode || appCfg.UIConfigurationDisabled
+
+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ for field := range rt.Fields() {
+ if err := s.processEnvField(ctx, tx, field, isEnvOnlyMode); err != nil {
+ return err
}
- return nil
- }); err != nil {
- return actors.NoPayload{}, err
}
- return actors.NoPayload{}, s.LoadDatabaseSettings(context.WithoutCancel(writeCtx))
- }, nil)
- return err
+ return nil
+ }); err != nil {
+ return err
+ }
+ return s.LoadDatabaseSettings(context.WithoutCancel(ctx))
}
func allowedSettingKeys() map[string]struct{} {
@@ -1007,76 +1019,83 @@ func (s *SettingsService) SetStringSetting(ctx context.Context, key, value strin
// the autoUpdateExcludedContainers setting. When excluded is true the container
// is added to the list; when false it is removed.
func (s *SettingsService) SetContainerAutoUpdateExclusionInternal(ctx context.Context, containerName string, excluded bool) error {
- _, err := s.writes.Execute(ctx, "update container auto-update exclusion", func(writeCtx context.Context) (actors.NoPayload, error) {
- ordered := kit.Unique(kit.TrimNonEmpty(strings.Split(s.GetStringSetting(writeCtx, "autoUpdateExcludedContainers", ""), ",")))
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
- if excluded {
- ordered = kit.Unique(append(ordered, containerName))
- } else {
- filtered := ordered[:0]
- for _, name := range ordered {
- if name != containerName {
- filtered = append(filtered, name)
- }
+ ordered := kit.Unique(kit.TrimNonEmpty(strings.Split(s.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""), ",")))
+
+ if excluded {
+ ordered = kit.Unique(append(ordered, containerName))
+ } else {
+ filtered := ordered[:0]
+ for _, name := range ordered {
+ if name != containerName {
+ filtered = append(filtered, name)
}
- ordered = filtered
}
+ ordered = filtered
+ }
- if err := s.updateSettingValueNoRefreshInternal(writeCtx, "autoUpdateExcludedContainers", strings.Join(ordered, ",")); err != nil {
- return actors.NoPayload{}, err
- }
- return actors.NoPayload{}, s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx))
- }, nil)
- return err
+ if err := s.updateSettingValueNoRefreshInternal(ctx, "autoUpdateExcludedContainers", strings.Join(ordered, ",")); err != nil {
+ return err
+ }
+ return s.refreshSettingsCacheInternal(context.WithoutCancel(ctx))
}
func (s *SettingsService) EnsureEncryptionKey(ctx context.Context) (string, error) {
- return s.writes.Execute(ctx, "ensure encryption key", func(writeCtx context.Context) (string, error) {
- const keyName = "encryptionKey"
- var key string
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
- err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
- var sv SettingVariable
- err := tx.Where("key = ?", keyName).First(&sv).Error
+ const keyName = "encryptionKey"
+ var key string
- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
- return errors.WrapIf(err, "failed to load encryption key")
- }
+ err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var sv SettingVariable
+ err := tx.Where("key = ?", keyName).First(&sv).Error
- if sv.Value != "" {
- key = sv.Value
- return nil
- }
+ if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.WrapIf(err, "failed to load encryption key")
+ }
- notFound := errors.Is(err, gorm.ErrRecordNotFound)
- sum := sha256.Sum256([]byte(uuid.New().String()))
- generatedKey := base64.StdEncoding.EncodeToString(sum[:])
- key = generatedKey
+ if sv.Value != "" {
+ key = sv.Value
+ return nil
+ }
- if notFound {
- if createErr := tx.Create(&SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {
- return errors.WrapIf(createErr, "failed to persist encryption key")
- }
- return nil
- }
+ notFound := errors.Is(err, gorm.ErrRecordNotFound)
+ sum := sha256.Sum256([]byte(uuid.New().String()))
+ generatedKey := base64.StdEncoding.EncodeToString(sum[:])
+ key = generatedKey
- if updErr := tx.Model(&SettingVariable{}).
- Where("key = ?", keyName).
- Update("value", generatedKey).Error; updErr != nil {
- return errors.WrapIf(updErr, "failed to update encryption key")
+ if notFound {
+ if createErr := tx.Create(&SettingVariable{Key: keyName, Value: generatedKey}).Error; createErr != nil {
+ return errors.WrapIf(createErr, "failed to persist encryption key")
}
return nil
- })
- if err != nil {
- return "", err
}
- return key, nil
- }, nil)
+ if updErr := tx.Model(&SettingVariable{}).
+ Where("key = ?", keyName).
+ Update("value", generatedKey).Error; updErr != nil {
+ return errors.WrapIf(updErr, "failed to update encryption key")
+ }
+ return nil
+ })
+ if err != nil {
+ return "", err
+ }
+
+ return key, nil
}
func (s *SettingsService) EnsureJwtSigningKey(ctx context.Context) (*mldsa.PrivateKey, error) {
- seed, err := s.ensureEncryptedKeyInternal(ctx, "ensure jwt signing key", "jwtSigningKeySeed", mldsa.PrivateKeySize)
+ seed, err := s.ensureEncryptedKeyInternal(ctx, "jwtSigningKeySeed", mldsa.PrivateKeySize)
if err != nil {
return nil, err
}
@@ -1085,57 +1104,61 @@ func (s *SettingsService) EnsureJwtSigningKey(ctx context.Context) (*mldsa.Priva
}
func (s *SettingsService) EnsureBrowserSessionSigningKey(ctx context.Context) ([]byte, error) {
- return s.ensureEncryptedKeyInternal(ctx, "ensure browser session signing key", "browserSessionSigningKey", browserSessionSigningKeySize)
+ return s.ensureEncryptedKeyInternal(ctx, "browserSessionSigningKey", browserSessionSigningKeySize)
}
-func (s *SettingsService) ensureEncryptedKeyInternal(ctx context.Context, operation, keyName string, size int) ([]byte, error) {
- return s.writes.Execute(ctx, operation, func(writeCtx context.Context) ([]byte, error) {
- var key []byte
+func (s *SettingsService) ensureEncryptedKeyInternal(ctx context.Context, keyName string, size int) ([]byte, error) {
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
- err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
- var sv SettingVariable
- err := tx.Where("key = ?", keyName).First(&sv).Error
- if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
- return errors.WrapIf(err, "failed to load signing key")
- }
+ var key []byte
- if sv.Value != "" {
- decoded, decErr := libcrypto.Decrypt(sv.Value)
- if decErr != nil {
- return errors.WrapIf(decErr, "failed to decrypt signing key")
- }
- key, decErr = base64.StdEncoding.DecodeString(decoded)
- if decErr != nil {
- return errors.WrapIf(decErr, "failed to decode signing key")
- }
- if len(key) != size {
- return errors.Errorf("invalid signing key length: got %d, want %d", len(key), size)
- }
- return nil
- }
+ err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var sv SettingVariable
+ err := tx.Where("key = ?", keyName).First(&sv).Error
+ if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.WrapIf(err, "failed to load signing key")
+ }
- key = make([]byte, size)
- if _, genErr := rand.Read(key); genErr != nil {
- return errors.WrapIf(genErr, "failed to generate signing key")
+ if sv.Value != "" {
+ decoded, decErr := libcrypto.Decrypt(sv.Value)
+ if decErr != nil {
+ return errors.WrapIf(decErr, "failed to decrypt signing key")
}
- encrypted, encErr := libcrypto.Encrypt(base64.StdEncoding.EncodeToString(key))
- if encErr != nil {
- return errors.WrapIf(encErr, "failed to encrypt signing key")
+ key, decErr = base64.StdEncoding.DecodeString(decoded)
+ if decErr != nil {
+ return errors.WrapIf(decErr, "failed to decode signing key")
}
-
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return errors.WrapIf(tx.Create(&SettingVariable{Key: keyName, Value: encrypted}).Error, "failed to persist signing key")
+ if len(key) != size {
+ return errors.Errorf("invalid signing key length: got %d, want %d", len(key), size)
}
- return errors.WrapIf(tx.Model(&SettingVariable{}).
- Where("key = ?", keyName).
- Update("value", encrypted).Error, "failed to update signing key")
- })
- if err != nil {
- return nil, err
+ return nil
+ }
+
+ key = make([]byte, size)
+ if _, genErr := rand.Read(key); genErr != nil {
+ return errors.WrapIf(genErr, "failed to generate signing key")
}
+ encrypted, encErr := libcrypto.Encrypt(base64.StdEncoding.EncodeToString(key))
+ if encErr != nil {
+ return errors.WrapIf(encErr, "failed to encrypt signing key")
+ }
+
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.WrapIf(tx.Create(&SettingVariable{Key: keyName, Value: encrypted}).Error, "failed to persist signing key")
+ }
+ return errors.WrapIf(tx.Model(&SettingVariable{}).
+ Where("key = ?", keyName).
+ Update("value", encrypted).Error, "failed to update signing key")
+ })
+ if err != nil {
+ return nil, err
+ }
- return key, nil
- }, nil)
+ return key, nil
}
func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projectsDirEnv string) error {
@@ -1144,44 +1167,44 @@ func (s *SettingsService) NormalizeProjectsDirectory(ctx context.Context, projec
return nil
}
- _, err := s.writes.Execute(ctx, "normalize projects directory", func(writeCtx context.Context) (string, error) {
- var projectsDirSetting SettingVariable
- err := s.db.WithContext(writeCtx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error
- if errors.Is(err, gorm.ErrRecordNotFound) {
- slog.DebugContext(writeCtx, "No projectsDirectory setting found, skipping normalization")
- return "", nil
- }
- if err != nil {
- return "", errors.WrapIf(err, "failed to load projectsDirectory setting")
- }
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
- value := strings.TrimSpace(projectsDirSetting.Value)
- isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))
- if filepath.IsAbs(value) || isMapping {
- slog.DebugContext(writeCtx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)
- return "", nil
- }
+ var projectsDirSetting SettingVariable
+ err := s.db.WithContext(ctx).Where("key = ?", "projectsDirectory").First(&projectsDirSetting).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ slog.DebugContext(ctx, "No projectsDirectory setting found, skipping normalization")
+ return nil
+ }
+ if err != nil {
+ return errors.WrapIf(err, "failed to load projectsDirectory setting")
+ }
- cwd, _ := os.Getwd()
- absPath, absErr := filepath.Abs(value)
- if absErr != nil {
- return "", errors.WrapIf(absErr, "failed to resolve relative path to absolute")
- }
- slog.InfoContext(writeCtx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
- if err := s.updateSettingValueNoRefreshInternal(writeCtx, "projectsDirectory", absPath); err != nil {
- return "", errors.WrapIf(err, "failed to update projectsDirectory")
- }
- if err := s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx)); err != nil {
- return "", err
- }
- slog.InfoContext(writeCtx, "Successfully normalized projects directory")
- return absPath, nil
- }, func(value string, err error) {
- if err == nil && value != "" {
- s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: "projectsDirectory", Value: value}})
- }
- })
- return err
+ value := strings.TrimSpace(projectsDirSetting.Value)
+ isMapping := strings.Contains(value, ":") && (strings.HasPrefix(value, "/") || projects.IsWindowsDrivePath(value))
+ if filepath.IsAbs(value) || isMapping {
+ slog.DebugContext(ctx, "Projects directory already normalized or custom, skipping", "value", projectsDirSetting.Value)
+ return nil
+ }
+
+ cwd, _ := os.Getwd()
+ absPath, absErr := filepath.Abs(value)
+ if absErr != nil {
+ return errors.WrapIf(absErr, "failed to resolve relative path to absolute")
+ }
+ slog.InfoContext(ctx, "Normalizing projects directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
+ if err := s.updateSettingValueNoRefreshInternal(ctx, "projectsDirectory", absPath); err != nil {
+ return errors.WrapIf(err, "failed to update projectsDirectory")
+ }
+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); err != nil {
+ return err
+ }
+ slog.InfoContext(ctx, "Successfully normalized projects directory")
+ s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: "projectsDirectory", Value: absPath}})
+ return nil
}
func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {
@@ -1192,41 +1215,101 @@ func (s *SettingsService) NormalizeBuildsDirectory(ctx context.Context) error {
return nil
}
- _, err := s.writes.Execute(ctx, "normalize builds directory", func(writeCtx context.Context) (string, error) {
- var buildsDirSetting SettingVariable
- err := s.db.WithContext(writeCtx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error
- if errors.Is(err, gorm.ErrRecordNotFound) {
- slog.DebugContext(writeCtx, "No buildsDirectory setting found, skipping normalization")
- return "", nil
- }
- if err != nil {
- return "", errors.WrapIf(err, "failed to load buildsDirectory setting")
- }
+ s.writes.Lock()
+ defer s.writes.Unlock()
+ if err := ctx.Err(); err != nil {
+ return err
+ }
- value := strings.TrimSpace(buildsDirSetting.Value)
- if value == "" || filepath.IsAbs(value) {
- slog.DebugContext(writeCtx, "Builds directory already normalized or empty, skipping", "value", buildsDirSetting.Value)
- return "", nil
- }
+ var buildsDirSetting SettingVariable
+ err := s.db.WithContext(ctx).Where("key = ?", buildsKey).First(&buildsDirSetting).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ slog.DebugContext(ctx, "No buildsDirectory setting found, skipping normalization")
+ return nil
+ }
+ if err != nil {
+ return errors.WrapIf(err, "failed to load buildsDirectory setting")
+ }
- cwd, _ := os.Getwd()
- absPath, absErr := filepath.Abs(value)
- if absErr != nil {
- return "", errors.WrapIf(absErr, "failed to resolve relative path to absolute")
- }
- slog.InfoContext(writeCtx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
- if err := s.updateSettingValueNoRefreshInternal(writeCtx, buildsKey, absPath); err != nil {
- return "", errors.WrapIf(err, "failed to update buildsDirectory")
+ value := strings.TrimSpace(buildsDirSetting.Value)
+ if value == "" || filepath.IsAbs(value) {
+ slog.DebugContext(ctx, "Builds directory already normalized or empty, skipping", "value", buildsDirSetting.Value)
+ return nil
+ }
+
+ cwd, _ := os.Getwd()
+ absPath, absErr := filepath.Abs(value)
+ if absErr != nil {
+ return errors.WrapIf(absErr, "failed to resolve relative path to absolute")
+ }
+ slog.InfoContext(ctx, "Normalizing builds directory from relative to absolute path", "from", value, "to", absPath, "base", cwd)
+ if err := s.updateSettingValueNoRefreshInternal(ctx, buildsKey, absPath); err != nil {
+ return errors.WrapIf(err, "failed to update buildsDirectory")
+ }
+ if err := s.refreshSettingsCacheInternal(context.WithoutCancel(ctx)); err != nil {
+ return err
+ }
+ slog.InfoContext(ctx, "Successfully normalized builds directory")
+ s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: buildsKey, Value: absPath}})
+ return nil
+}
+
+func (s *SettingsService) enqueueEffectInternal(effect func()) error {
+ s.effectsMu.Lock()
+ defer s.effectsMu.Unlock()
+ if s.effectsClosed {
+ return errors.New("settings effects stopped")
+ }
+ s.effects = append(s.effects, effect)
+ select {
+ case s.effectsWake <- struct{}{}:
+ default:
+ }
+ return nil
+}
+
+func (s *SettingsService) runEffectsInternal() {
+ defer close(s.effectsDone)
+ for {
+ s.effectsMu.Lock()
+ if len(s.effects) > 0 {
+ effect := s.effects[0]
+ s.effects[0] = nil
+ s.effects = s.effects[1:]
+ s.effectsMu.Unlock()
+ func() { var err error; defer utils.RecoverToError(&err, "settings effect"); effect() }()
+ continue
}
- if err := s.refreshSettingsCacheInternal(context.WithoutCancel(writeCtx)); err != nil {
- return "", err
+ closed := s.effectsClosed
+ s.effectsMu.Unlock()
+ if closed {
+ return
}
- slog.InfoContext(writeCtx, "Successfully normalized builds directory")
- return absPath, nil
- }, func(value string, err error) {
- if err == nil && value != "" {
- s.publishSettingsChangesInternal([]libarcane.SettingUpdate{{Key: buildsKey, Value: value}})
+ select {
+ case <-s.effectsWake:
+ case <-s.lifecycleCtx.Done():
+ s.effectsMu.Lock()
+ s.effectsClosed = true
+ s.effectsMu.Unlock()
}
- })
- return err
+ }
+}
+
+// Stop drains accepted effects and joins the settings worker.
+func (s *SettingsService) Stop(ctx context.Context) error {
+ s.writes.Lock()
+ s.effectsMu.Lock()
+ s.effectsClosed = true
+ s.effectsMu.Unlock()
+ s.writes.Unlock()
+ select {
+ case s.effectsWake <- struct{}{}:
+ default:
+ }
+ select {
+ case <-s.effectsDone:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
}
diff --git a/backend/internal/settings/service_test.go b/backend/internal/settings/service_test.go
index 467d92446..3315cec3a 100644
--- a/backend/internal/settings/service_test.go
+++ b/backend/internal/settings/service_test.go
@@ -10,7 +10,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -19,7 +18,6 @@ import (
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
libcrypto "go.getarcane.app/sys/crypto"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -33,49 +31,25 @@ func setupSettingsTestDB(t *testing.T) *database.DB {
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return NewSettingsService(ctx, db, executor, effects)
-}
-
-func newAdmissionGateForTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {
- t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "services-test-admission", t.Name())
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return gate
+ svc, err := NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func waitForSettingsNotificationsInternal(t *testing.T, svc *SettingsService) {
t.Helper()
- _, err := svc.writes.Execute(t.Context(), "wait for settings notification submission", func(context.Context) (actors.NoPayload, error) {
- return actors.NoPayload{}, nil
- }, nil)
- require.NoError(t, err)
- _, err = svc.effects.Execute(t.Context(), "wait for settings notifications", func(context.Context) (actors.NoPayload, error) {
- return actors.NoPayload{}, nil
- }, nil)
+ done := make(chan struct{})
+ svc.writes.Lock()
+ err := svc.enqueueEffectInternal(func() { close(done) })
+ svc.writes.Unlock()
require.NoError(t, err)
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("settings effects did not finish")
+ }
}
func TestSettingsService_EnsureDefaultSettings_ReplacesRetiredDefaults(t *testing.T) {
diff --git a/backend/internal/swarm/module.go b/backend/internal/swarm/module.go
index ff2fc7450..4253c5dd6 100644
--- a/backend/internal/swarm/module.go
+++ b/backend/internal/swarm/module.go
@@ -8,19 +8,22 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
)
-type Dependencies struct {
- Environment *environment.EnvironmentService
- Event *event.EventService
- Config *config.Config
+type Module struct {
+ service *SwarmService
+ environment *environment.EnvironmentService
+ event *event.EventService
+ config *config.Config
}
-type Module struct {
- Service *SwarmService
- deps Dependencies
+func New(service *SwarmService, environmentService *environment.EnvironmentService, eventService *event.EventService, cfg *config.Config) *Module {
+ return &Module{service: service, environment: environmentService, event: eventService, config: cfg}
}
-func New(service *SwarmService, deps Dependencies) *Module {
- return &Module{Service: service, deps: deps}
+func (m *Module) Service() *SwarmService {
+ if m == nil {
+ return nil
+ }
+ return m.service
}
func (m *Module) RegisterRoutes(api huma.API) {
@@ -28,5 +31,5 @@ func (m *Module) RegisterRoutes(api huma.API) {
RegisterSwarm(api, nil, nil, nil, nil)
return
}
- RegisterSwarm(api, m.Service, m.deps.Environment, m.deps.Event, m.deps.Config)
+ RegisterSwarm(api, m.service, m.environment, m.event, m.config)
}
diff --git a/backend/internal/swarm/service_test.go b/backend/internal/swarm/service_test.go
index 3f18d4d57..ceac33ca9 100644
--- a/backend/internal/swarm/service_test.go
+++ b/backend/internal/swarm/service_test.go
@@ -11,7 +11,6 @@ import (
"time"
cerrdefs "github.com/containerd/errdefs"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -25,7 +24,6 @@ import (
"github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -39,21 +37,11 @@ func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {
func newSettingsServiceForSwarmTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- writes, err := actors.NewExecutor(t.Context(), runtime, "swarm-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "swarm-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, writes.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, writes, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func createSwarmTestEnvironmentInternal(t *testing.T, db *database.DB, id, apiURL, status string, isEdge bool, accessToken *string) {
diff --git a/backend/internal/system/module.go b/backend/internal/system/module.go
index e0bd84eb0..ade18c91c 100644
--- a/backend/internal/system/module.go
+++ b/backend/internal/system/module.go
@@ -6,56 +6,32 @@ import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
- "github.com/getarcaneapp/arcane/backend/v2/internal/container"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
- "github.com/getarcaneapp/arcane/backend/v2/internal/image"
- "github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
- "github.com/getarcaneapp/arcane/backend/v2/internal/network"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
- "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-// Dependencies are the collaborators the system domain needs.
-type Dependencies struct {
- DB *database.DB
- Config *config.Config
- Docker *docker.DockerClientService
- Container *container.ContainerService
- Image *image.ImageService
- ImageUpdate *imageupdate.ImageUpdateService
- Volume *volume.VolumeService
- Network *network.NetworkService
- Settings *settings.SettingsService
- Activity *activity.ActivityService
- SystemUpgrade *SystemUpgradeService
- Environment *environment.EnvironmentService
-}
-
// Module wires the system domain and mounts its routes.
type Module struct {
- Service *SystemService
- deps Dependencies
+ service *SystemService
+ dockerService *docker.DockerClientService
+ upgradeService *SystemUpgradeService
+ environmentService *environment.EnvironmentService
+ cfg *config.Config
+ activityService *activity.ActivityService
}
-// New builds the system domain from its dependencies.
-func New(deps Dependencies) *Module {
- return &Module{
- Service: NewSystemService(
- deps.DB,
- deps.Docker,
- deps.Container,
- deps.Image,
- deps.ImageUpdate,
- deps.Volume,
- deps.Network,
- deps.Settings,
- deps.Activity,
- ),
- deps: deps,
+// New wires system routes around an existing service.
+func New(service *SystemService, dockerService *docker.DockerClientService, upgradeService *SystemUpgradeService, environmentService *environment.EnvironmentService, cfg *config.Config, activityService *activity.ActivityService) *Module {
+ return &Module{service: service, dockerService: dockerService, upgradeService: upgradeService, environmentService: environmentService, cfg: cfg, activityService: activityService}
+}
+
+// Service exposes the system service to collaborators.
+func (m *Module) Service() *SystemService {
+ if m == nil {
+ return nil
}
+ return m.service
}
// RegisterRoutes mounts the system endpoints. A nil module still registers, so
@@ -65,5 +41,5 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
RegisterSystem(api, nil, nil, nil, nil, nil, nil, appCtx)
return
}
- RegisterSystem(api, m.deps.Docker, m.Service, m.deps.SystemUpgrade, m.deps.Environment, m.deps.Config, m.deps.Activity, appCtx)
+ RegisterSystem(api, m.dockerService, m.service, m.upgradeService, m.environmentService, m.cfg, m.activityService, appCtx)
}
diff --git a/backend/internal/system/prune_recovery.go b/backend/internal/system/prune_recovery.go
new file mode 100644
index 000000000..0095b3d45
--- /dev/null
+++ b/backend/internal/system/prune_recovery.go
@@ -0,0 +1,518 @@
+package system
+
+import (
+ "context"
+ "encoding/json/v2"
+ "fmt"
+ "regexp"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/containerd/errdefs"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
+ "github.com/moby/moby/client"
+ "go.getarcane.app/docker/compat"
+)
+
+type pruneResourceInternal struct {
+ Kind string `json:"kind"`
+ ID string `json:"id"`
+ Size int64 `json:"size"`
+ Names []string `json:"names,omitempty"`
+}
+type prunePlanInternal struct {
+ Request systemtypes.PruneAllRequest `json:"request"`
+ Resources []pruneResourceInternal `json:"resources"`
+}
+
+// PruneScheduled freezes exact targets before applying destructive work.
+func (s *SystemService) PruneScheduled(ctx context.Context, environmentID string, req systemtypes.PruneAllRequest) (*systemtypes.PruneAllResult, bool, error) {
+ prune := s.beginSystemPruneInternal(ctx, environmentID, req)
+ result := &systemtypes.PruneAllResult{Success: true}
+ if prune.activityID != "" {
+ result.ActivityID = &prune.activityID
+ }
+ if prune.started.IsAbsent() {
+ return result, false, nil
+ }
+ defer s.finishSystemPruneInternal(environmentID)
+ fail := func(err error) (*systemtypes.PruneAllResult, bool, error) {
+ result.Success = false
+ result.Errors = append(result.Errors, err.Error())
+ s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)
+ return result, true, err
+ }
+
+ plan, err := s.preparePrunePlanInternal(ctx, req)
+ if err != nil {
+ return fail(err)
+ }
+ payload, err := json.Marshal(plan)
+ if err != nil {
+ return fail(err)
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "prune_plan", ID: "prune-plan", Status: schedulertypes.Succeeded, RecoveryData: payload, ActivityID: prune.activityID}); err != nil {
+ return fail(err)
+ }
+ previous, _ := jobcontext.Run(ctx)
+ err = s.executePrunePlanInternal(ctx, plan, previous, result, false)
+ s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)
+ if err == nil && result.Success {
+ err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "scheduled-prune", Status: schedulertypes.Succeeded, ActivityID: prune.activityID})
+ }
+ return result, true, err
+}
+
+func (s *SystemService) preparePrunePlanInternal(ctx context.Context, req systemtypes.PruneAllRequest) (prunePlanInternal, error) {
+ plan := prunePlanInternal{Request: req}
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return plan, err
+ }
+ if req.Volumes != nil {
+ s.volumeService.CleanupHelperContainers(ctx)
+ }
+ usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{Containers: true, Images: true, Volumes: true, BuildCache: req.BuildCache != nil, Verbose: true})
+ if err != nil {
+ return plan, err
+ }
+ if err := s.selectPruneContainersInternal(&plan, usage); err != nil {
+ return plan, err
+ }
+ selectedContainers, imageRefs, volumeRefs := pruneSelectedReferencesInternal(plan.Resources, usage)
+ if err := s.selectPruneImagesInternal(&plan, usage, imageRefs); err != nil {
+ return plan, err
+ }
+ selectPruneVolumesInternal(&plan, usage, volumeRefs)
+ if err := s.selectPruneNetworksInternal(ctx, dockerClient, &plan, selectedContainers); err != nil {
+ return plan, err
+ }
+ if err := s.selectPruneBuildCacheInternal(&plan, usage); err != nil {
+ return plan, err
+ }
+ return plan, nil
+}
+
+func (s *SystemService) selectPruneContainersInternal(plan *prunePlanInternal, usage client.DiskUsageResult) error {
+ req := plan.Request
+ var err error
+ if req.Containers == nil || req.Containers.Mode == systemtypes.PruneContainerModeNone {
+ return nil
+ }
+ until := time.Time{}
+ if req.Containers.Mode == systemtypes.PruneContainerModeOlderThan {
+ until, err = s.pruneCutoffInternal(req.Containers.Until)
+ if err != nil {
+ return err
+ }
+ }
+ for _, item := range usage.Containers.Items {
+ if item.State == "running" || item.State == "paused" || item.State == "restarting" || (!until.IsZero() && !time.Unix(item.Created, 0).Before(until)) {
+ continue
+ }
+ plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "container", ID: item.ID, Size: item.SizeRw})
+ }
+ return nil
+}
+
+func (s *SystemService) selectPruneImagesInternal(plan *prunePlanInternal, usage client.DiskUsageResult, imageRefs map[string]int) error {
+ req := plan.Request
+ var err error
+ if req.Images == nil || req.Images.Mode == systemtypes.PruneImageModeNone {
+ return nil
+ }
+ until := time.Time{}
+ if req.Images.Mode == systemtypes.PruneImageModeOlderThan {
+ until, err = s.pruneCutoffInternal(req.Images.Until)
+ if err != nil {
+ return err
+ }
+ }
+ for _, item := range usage.Images.Items {
+ if item.Containers > int64(imageRefs[item.ID]) || (req.Images.Mode == systemtypes.PruneImageModeDangling && len(item.RepoTags) > 0) || (!until.IsZero() && !time.Unix(item.Created, 0).Before(until)) {
+ continue
+ }
+ plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "image", ID: item.ID, Size: item.Size, Names: slices.Clone(item.RepoTags)})
+ }
+ return nil
+}
+
+func selectPruneVolumesInternal(plan *prunePlanInternal, usage client.DiskUsageResult, volumeRefs map[string]int) {
+ req := plan.Request
+ if req.Volumes != nil && req.Volumes.Mode != systemtypes.PruneVolumeModeNone {
+ for _, item := range usage.Volumes.Items {
+ if item.UsageData == nil || item.UsageData.RefCount > int64(volumeRefs[item.Name]) || strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") {
+ continue
+ }
+ if req.Volumes.Mode == systemtypes.PruneVolumeModeAnonymous {
+ if _, anonymous := item.Labels["com.docker.volume.anonymous"]; !anonymous {
+ continue
+ }
+ }
+ plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "volume", ID: item.Name, Size: item.UsageData.Size})
+ }
+ }
+}
+
+func (s *SystemService) selectPruneNetworksInternal(ctx context.Context, dockerClient *client.Client, plan *prunePlanInternal, selectedContainers map[string]bool) error {
+ req := plan.Request
+ var err error
+ if req.Networks == nil || req.Networks.Mode == systemtypes.PruneNetworkModeNone {
+ return nil
+ }
+ until := time.Time{}
+ if req.Networks.Mode == systemtypes.PruneNetworkModeOlderThan {
+ until, err = s.pruneCutoffInternal(req.Networks.Until)
+ if err != nil {
+ return err
+ }
+ }
+ networks, err := dockerClient.NetworkList(ctx, client.NetworkListOptions{})
+ if err != nil {
+ return err
+ }
+ for _, item := range networks.Items {
+ if item.Name == "bridge" || item.Name == "host" || item.Name == "none" || item.Ingress || item.Scope != "local" || (!until.IsZero() && !item.Created.Before(until)) {
+ continue
+ }
+ inspect, err := compat.NetworkInspectWithCompatibility(ctx, dockerClient, item.ID, client.NetworkInspectOptions{})
+ if err != nil {
+ return err
+ }
+ eligible := true
+ for id := range inspect.Network.Containers {
+ if !selectedContainers[id] {
+ eligible = false
+ break
+ }
+ }
+ if !eligible {
+ continue
+ }
+ plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "network", ID: item.ID})
+ }
+ return nil
+}
+
+func (s *SystemService) selectPruneBuildCacheInternal(plan *prunePlanInternal, usage client.DiskUsageResult) error {
+ req := plan.Request
+ var err error
+ if req.BuildCache == nil || req.BuildCache.Mode == systemtypes.PruneBuildCacheModeNone {
+ return nil
+ }
+ until := time.Time{}
+ if req.BuildCache.Mode == systemtypes.PruneBuildCacheModeOlderThan {
+ until, err = s.pruneCutoffInternal(req.BuildCache.Until)
+ if err != nil {
+ return err
+ }
+ }
+ for _, item := range usage.BuildCache.Items {
+ if item.InUse || (!until.IsZero() && (item.LastUsedAt == nil || !item.LastUsedAt.Before(until))) {
+ continue
+ }
+ plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "build_cache", ID: item.ID, Size: item.Size})
+ }
+ return nil
+}
+
+func pruneSelectedReferencesInternal(resources []pruneResourceInternal, usage client.DiskUsageResult) (map[string]bool, map[string]int, map[string]int) {
+ selectedContainers := make(map[string]bool)
+ imageRefs := make(map[string]int)
+ volumeRefs := make(map[string]int)
+ for _, resource := range resources {
+ if resource.Kind == "container" {
+ selectedContainers[resource.ID] = true
+ }
+ }
+ for _, item := range usage.Containers.Items {
+ if !selectedContainers[item.ID] {
+ continue
+ }
+ imageRefs[item.ImageID]++
+ mounted := make(map[string]bool)
+ for _, m := range item.Mounts {
+ if m.Type == "volume" && !mounted[m.Name] {
+ mounted[m.Name] = true
+ volumeRefs[m.Name]++
+ }
+ }
+ }
+ return selectedContainers, imageRefs, volumeRefs
+}
+
+func (s *SystemService) pruneCutoffInternal(raw string) (time.Time, error) {
+ now := time.Now()
+ location := s.location
+ if location == nil {
+ location = time.UTC
+ }
+ if duration, err := time.ParseDuration(raw); err == nil && duration >= 0 {
+ return now.Add(-duration), nil
+ }
+ for _, format := range []string{time.RFC3339Nano, "2006-01-02T15:04:05", "2006-01-02"} {
+ if value, err := time.ParseInLocation(format, raw, location); err == nil {
+ return value, nil
+ }
+ }
+ if seconds, err := strconv.ParseFloat(raw, 64); err == nil {
+ return time.Unix(int64(seconds), int64((seconds-float64(int64(seconds)))*1e9)), nil
+ }
+ return time.Time{}, fmt.Errorf("unsupported prune cutoff %q", raw)
+}
+
+func (s *SystemService) executePrunePlanInternal(ctx context.Context, plan prunePlanInternal, previous schedulertypes.Run, result *systemtypes.PruneAllResult, recovery bool) error {
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ for _, resource := range plan.Resources {
+ if resource.Kind == "build_cache" {
+ continue
+ }
+ target := schedulertypes.TargetOutcome{ResourceType: resource.Kind, ID: "prune:" + resource.Kind + ":" + resource.ID, Status: schedulertypes.Running}
+ if pruneTargetCompletedInternal(previous, target.ID) {
+ continue
+ }
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return err
+ }
+ eligible, exists, err := s.pruneResourceEligibleInternal(ctx, dockerClient, plan, resource)
+ if err != nil {
+ return err
+ }
+ switch {
+ case !exists:
+ target.Status = schedulertypes.Succeeded
+ case !eligible:
+ target.Status = schedulertypes.Skipped
+ default:
+ err = s.removePruneResourceInternal(ctx, dockerClient, resource)
+ if err != nil && !errdefs.IsNotFound(err) {
+ target.Status = schedulertypes.Failed
+ target.Message = err.Error()
+ result.Success = false
+ result.Errors = append(result.Errors, err.Error())
+ } else {
+ target.Status = schedulertypes.Succeeded
+ recordPrunedResourceInternal(resource, result)
+ }
+ }
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return err
+ }
+ }
+ if len(result.ImagesDeleted) > 0 {
+ if err := s.imageUpdateService.DeleteRecordsForImages(ctx, result.ImagesDeleted); err != nil {
+ return err
+ }
+ }
+ return s.executePruneCacheInternal(ctx, dockerClient, plan, result, recovery)
+}
+
+func pruneTargetCompletedInternal(previous schedulertypes.Run, targetID string) bool {
+ for _, target := range previous.Outcome.Targets {
+ if target.ID == targetID && (target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped) {
+ return true
+ }
+ }
+ return false
+}
+
+func (s *SystemService) removePruneResourceInternal(ctx context.Context, dockerClient *client.Client, resource pruneResourceInternal) error {
+ var err error
+ switch resource.Kind {
+ case "container":
+ _, err = dockerClient.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})
+ case "image":
+ err = removePruneImageInternal(ctx, dockerClient, resource)
+ case "volume":
+ err = s.volumeService.DeleteVolume(ctx, resource.ID, false, common.SystemUser)
+ case "network":
+ err = s.networkService.RemoveNetwork(ctx, resource.ID, common.SystemUser)
+ default:
+ return fmt.Errorf("unknown prune target %q", resource.Kind)
+ }
+ return err
+}
+
+func removePruneImageInternal(ctx context.Context, dockerClient *client.Client, resource pruneResourceInternal) error {
+ var err error
+ for _, name := range resource.Names {
+ inspected, inspectErr := dockerClient.ImageInspect(ctx, name)
+ if errdefs.IsNotFound(inspectErr) {
+ continue
+ }
+ if inspectErr != nil {
+ return inspectErr
+ }
+ if inspected.ID != resource.ID {
+ continue
+ }
+ _, err = dockerClient.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false})
+ if err != nil {
+ return err
+ }
+ }
+ _, err = dockerClient.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})
+ return err
+}
+
+func recordPrunedResourceInternal(resource pruneResourceInternal, result *systemtypes.PruneAllResult) {
+ switch resource.Kind {
+ case "container":
+ result.ContainersPruned = append(result.ContainersPruned, resource.ID)
+ case "image":
+ result.ImagesDeleted = append(result.ImagesDeleted, resource.ID)
+ // ImageRemove does not report reclaimed bytes; Size includes shared layers.
+ return
+ case "volume":
+ result.VolumesDeleted = append(result.VolumesDeleted, resource.ID)
+ case "network":
+ result.NetworksDeleted = append(result.NetworksDeleted, resource.ID)
+ }
+ if resource.Size <= 0 {
+ return
+ }
+ result.SpaceReclaimed += uint64(resource.Size)
+}
+
+func (s *SystemService) executePruneCacheInternal(ctx context.Context, dockerClient *client.Client, plan prunePlanInternal, result *systemtypes.PruneAllResult, recovery bool) error {
+ request := plan.Request.BuildCache
+ if request == nil || request.Mode == systemtypes.PruneBuildCacheModeNone {
+ return nil
+ }
+ if recovery {
+ usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{BuildCache: true, Verbose: true})
+ if err != nil {
+ return err
+ }
+ selected := make(map[string]bool)
+ for _, resource := range plan.Resources {
+ if resource.Kind == "build_cache" {
+ selected[resource.ID] = true
+ }
+ }
+ for _, current := range usage.BuildCache.Items {
+ if selected[current.ID] {
+ return errors.New("build cache prune outcome is unconfirmed")
+ }
+ }
+ return nil
+ }
+ for _, resource := range plan.Resources {
+ if resource.Kind != "build_cache" {
+ continue
+ }
+ // Docker interprets the ID filter as a regular expression.
+ filters := make(client.Filters).Add("id", "^"+regexp.QuoteMeta(resource.ID)+"$")
+ if request.Mode == systemtypes.PruneBuildCacheModeOlderThan {
+ filters = filters.Add("until", request.Until)
+ }
+ report, err := dockerClient.BuildCachePrune(ctx, client.BuildCachePruneOptions{
+ All: request.Mode == systemtypes.PruneBuildCacheModeAll,
+ Filters: filters,
+ })
+ if err != nil {
+ return err
+ }
+ result.SpaceReclaimed += report.Report.SpaceReclaimed
+ result.BuildCacheSpaceReclaimed += report.Report.SpaceReclaimed
+ }
+ return nil
+}
+
+func (s *SystemService) pruneResourceEligibleInternal(ctx context.Context, dockerClient *client.Client, plan prunePlanInternal, resource pruneResourceInternal) (eligible, exists bool, err error) {
+ if resource.Kind == "network" {
+ inspected, err := compat.NetworkInspectWithCompatibility(ctx, dockerClient, resource.ID, client.NetworkInspectOptions{})
+ if errdefs.IsNotFound(err) {
+ return false, false, nil
+ }
+ if err != nil {
+ return false, false, err
+ }
+ return len(inspected.Network.Containers) == 0 && !inspected.Network.Ingress, true, nil
+ }
+ usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{Containers: resource.Kind == "container", Images: resource.Kind == "image", Volumes: resource.Kind == "volume", Verbose: true})
+ if err != nil {
+ return false, false, err
+ }
+ switch resource.Kind {
+ case "container":
+ for _, item := range usage.Containers.Items {
+ if item.ID == resource.ID {
+ return item.State != "running" && item.State != "paused" && item.State != "restarting", true, nil
+ }
+ }
+ case "image":
+ return pruneImageEligibleInternal(usage, plan.Request.Images, resource)
+ case "volume":
+ for _, item := range usage.Volumes.Items {
+ if item.Name == resource.ID {
+ return item.UsageData != nil && item.UsageData.RefCount == 0 && !strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true"), true, nil
+ }
+ }
+ }
+ return false, false, nil
+}
+
+func pruneImageEligibleInternal(usage client.DiskUsageResult, request *systemtypes.PruneImagesOptions, resource pruneResourceInternal) (eligible, exists bool, err error) {
+ for _, item := range usage.Images.Items {
+ if item.ID == resource.ID {
+ for _, tag := range item.RepoTags {
+ if !slices.Contains(resource.Names, tag) {
+ return false, true, nil
+ }
+ }
+ return item.Containers == 0 && (request.Mode != systemtypes.PruneImageModeDangling || len(item.RepoTags) == 0), true, nil
+ }
+ }
+ return false, false, nil
+}
+
+func (s *SystemService) ReconcileScheduledPrune(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ outcome := jobcontext.ConfirmedTarget(previous, "scheduled-prune")
+ if outcome.Status == schedulertypes.Succeeded {
+ return outcome, nil
+ }
+ for _, target := range previous.Outcome.Targets {
+ if target.ID != "prune-plan" || len(target.RecoveryData) == 0 {
+ continue
+ }
+ var plan prunePlanInternal
+ if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {
+ return outcome, err
+ }
+ prune := s.beginSystemPruneInternal(ctx, previous.EnvironmentID, plan.Request)
+ if prune.started.IsAbsent() {
+ return outcome, nil
+ }
+ defer s.finishSystemPruneInternal(previous.EnvironmentID)
+ result := &systemtypes.PruneAllResult{Success: true}
+ err := s.executePrunePlanInternal(ctx, plan, previous, result, true)
+ if err != nil {
+ result.Success = false
+ result.Errors = append(result.Errors, err.Error())
+ }
+ s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)
+ if err != nil {
+ outcome.Message = err.Error()
+ return outcome, nil
+ }
+ if !result.Success {
+ return schedulertypes.Outcome{Status: schedulertypes.Partial, Targets: previous.Outcome.Targets}, nil
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "scheduled-prune", Status: schedulertypes.Succeeded}); err != nil {
+ return outcome, err
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
+ }
+ return outcome, nil
+}
diff --git a/backend/internal/system/system.go b/backend/internal/system/system.go
index 7f6aae3c1..38b787b4d 100644
--- a/backend/internal/system/system.go
+++ b/backend/internal/system/system.go
@@ -7,10 +7,12 @@ import (
"log/slog"
"strings"
"sync"
+ "time"
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -35,6 +37,7 @@ import (
)
type SystemService struct {
+ location *time.Location
db *database.DB
dockerService *docker.DockerClientService
containerService *container.ContainerService
@@ -59,8 +62,14 @@ func NewSystemService(
networkService *network.NetworkService,
settingsService *settings.SettingsService,
activityService *activity.ActivityService,
+ cfg *config.Config,
) *SystemService {
+ location := time.UTC
+ if cfg != nil {
+ location = cfg.GetLocation()
+ }
return &SystemService{
+ location: location,
db: db,
dockerService: dockerService,
containerService: containerService,
diff --git a/backend/internal/systembackup/background.go b/backend/internal/systembackup/background.go
index 0c44cf705..33b118a1f 100644
--- a/backend/internal/systembackup/background.go
+++ b/backend/internal/systembackup/background.go
@@ -2,6 +2,7 @@ package systembackup
import (
"context"
+ "encoding/json/v2"
"fmt"
"log/slog"
@@ -9,10 +10,16 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/moby/moby/client"
+ "go.getarcane.app/sys/crypto"
)
// StartBackup prepares a manual backup before submitting its snapshot work.
@@ -26,13 +33,13 @@ func (s *SystemBackupService) StartBackup(ctx context.Context, user common.User,
}
prepared, err := s.prepareBackupInternal(ctx, SystemBackupTriggerManual, request)
if err != nil {
- lease.Release()
+ lease.Release(ctx)
return nil, err
}
activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "arcane", "Arcane", &user,
"Creating system backup", "Creating Arcane system backup", database.JSON{"action": "create_system_backup", "backupId": prepared.run.ID, "destination": prepared.run.Destination, "s3DestinationId": prepared.run.S3DestinationID, "policyId": prepared.run.PolicyID}, false)
finish := func(runErr error) {
- defer lease.Release()
+ defer lease.Release(ctx)
if runErr != nil {
if saveErr := s.db.WithContext(context.WithoutCancel(workCtx)).Model(&SystemBackupRun{}).Where("id = ?", prepared.run.ID).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": runErr.Error()}).Error; saveErr != nil {
runErr = errors.Combine(runErr, errors.WrapIf(saveErr, "save backup failure"))
@@ -47,20 +54,31 @@ func (s *SystemBackupService) StartBackup(ctx context.Context, user common.User,
}
dto := prepared.run.ToDTO()
dto.ActivityID = activityID
- if err = s.engine.SubmitRun(workCtx, prepared.run.ID, func(runCtx context.Context) error {
- _, runErr := s.executeBackupInternal(runCtx, prepared)
- return runErr
- }, finish); err != nil {
+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ encryptedKey, keyErr := crypto.Encrypt(prepared.recoveryKey)
+ if keyErr != nil {
+ finish(keyErr)
+ return nil, keyErr
+ }
+ payload, err := json.Marshal(manualSystemBackupInternal{Checkpoint: systemBackupRecoveryInternal{BackupID: prepared.run.ID, LocalEnabled: prepared.localEnabled, S3Enabled: prepared.s3Enabled}, EncryptedKey: encryptedKey, ActivityID: activityID, UserID: user.ID})
+ if err == nil {
+ err = s.engine.SubmitDurableRun(workCtx, backuptypes.DurableRunCommand{Kind: "system", RunID: prepared.run.ID, ActivityID: activityID, Payload: payload, UserID: user.ID, EnvironmentID: "0", Permission: authz.PermSystemBackupsManage, RequestedWithKey: keyID}, lease)
+ }
+ if err != nil {
finish(err)
return nil, err
}
+
return &dto, nil
}
// ReconcileInterruptedBackups marks work interrupted by the previous process.
-func (s *SystemBackupService) ReconcileInterruptedBackups(ctx context.Context) error {
- return s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("status = ?", SystemBackupStatusRunning).
- Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error
+func (s *SystemBackupService) ReconcileInterruptedBackups(ctx context.Context, protectedIDs ...string) error {
+ query := s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("status = ?", SystemBackupStatusRunning)
+ if len(protectedIDs) > 0 {
+ query = query.Where("id NOT IN ?", protectedIDs)
+ }
+ return query.Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error
}
// StartSystemVolumeBackups freezes the selected policy and volumes before returning.
@@ -78,23 +96,19 @@ func (s *SystemBackupService) StartSystemVolumeBackups(ctx context.Context, user
activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "volumes", "Volumes", &user,
"Backing up volumes", "Creating system-managed volume backups", database.JSON{"action": "run_system_volume_backups", "policyId": policy.ID, "volumeNames": names, "matched": len(candidates), "succeeded": 0, "failed": 0, "skipped": 0, "failures": []backuptypes.SystemVolumeBackupFailure{}}, false)
if activityID == "" {
- lease.Release()
+ lease.Release(ctx)
return nil, errors.New("failed to create system-managed volume backup activity")
}
finish := func(runErr error) {
- defer lease.Release()
+ defer lease.Release(ctx)
activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "System-managed volume backups completed", runErr)
}
- err = s.engine.SubmitRun(workCtx, activityID, func(runCtx context.Context) error {
- result, runErr := s.executeSystemVolumeBackupsInternal(runCtx, policy, manualPolicy, candidates, volume.VolumeBackupTriggerManual, activityID)
- if runErr != nil {
- return runErr
- }
- if result.Failed > 0 {
- return fmt.Errorf("%d volume backups failed", result.Failed)
- }
- return nil
- }, finish)
+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ payload, err := json.Marshal(manualSystemVolumesInternal{Policy: policy, ManualPolicy: manualPolicy, Candidates: candidates, ActivityID: activityID, UserID: user.ID})
+ if err == nil {
+ err = s.engine.SubmitDurableRun(workCtx, backuptypes.DurableRunCommand{Kind: "system-volumes", RunID: activityID, ActivityID: activityID, Payload: payload, UserID: user.ID, EnvironmentID: "0", Permission: authz.PermSystemBackupsManage, RequestedWithKey: keyID}, lease)
+ }
+
if err != nil {
finish(err)
return nil, err
@@ -122,3 +136,314 @@ func (s *SystemBackupService) updateSystemVolumeProgressInternal(ctx context.Con
slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", policyID, "error", err)
}
}
+
+func (s *SystemBackupService) reconcileBackupInternal(ctx context.Context, previous schedulertypes.Run, policyID string, suppliedKeys ...string) (schedulertypes.Outcome, error) {
+ outcome := jobcontext.ConfirmedTarget(previous, policyID)
+ if outcome.Status == schedulertypes.Succeeded {
+ return outcome, nil
+ }
+ for _, target := range previous.Outcome.Targets {
+ if target.ID != policyID || len(target.RecoveryData) == 0 {
+ continue
+ }
+ var checkpoint systemBackupRecoveryInternal
+ if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
+ return outcome, err
+ }
+ var run SystemBackupRun
+ if err := s.db.WithContext(ctx).Where("id = ?", checkpoint.BackupID).First(&run).Error; err != nil {
+ return outcome, err
+ }
+ if run.Status != SystemBackupStatusSucceeded {
+ lease, admitted, err := s.engine.AcquireDurableRun(ctx, previous.ID, backup.SystemAdmissionScope, systemAdmissionID)
+ if err != nil {
+ return outcome, err
+ }
+ if !admitted {
+ return outcome, nil
+ }
+ defer lease.Release(ctx)
+ key := ""
+ if len(suppliedKeys) > 0 {
+ key = suppliedKeys[0]
+ }
+ if err := s.resumeBackupInternal(ctx, previous, &run, checkpoint, key); err != nil {
+ return outcome, err
+ }
+ }
+ target.Status = schedulertypes.Succeeded
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return outcome, err
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: []schedulertypes.TargetOutcome{target}}, nil
+ }
+ return outcome, nil
+}
+
+func systemBackupStageEvidenceInternal(previous schedulertypes.Run, run *SystemBackupRun) (string, bool) {
+ stageID, attempted := run.LocalSnapshotID, false
+ for _, evidence := range previous.Outcome.Targets {
+ if evidence.ID != run.ID+":stage" {
+ continue
+ }
+ attempted = true
+ if stageID == "" && evidence.Status == schedulertypes.Succeeded {
+ stageID = evidence.Message
+ }
+ }
+ return stageID, attempted
+}
+
+func systemBackupRemoteAttemptedInternal(previous schedulertypes.Run, runID string) bool {
+ for _, evidence := range previous.Outcome.Targets {
+ if evidence.ID == runID+":remote" {
+ return true
+ }
+ }
+ return false
+}
+
+func (s *SystemBackupService) observeBackupSnapshotsInternal(ctx context.Context, dockerClient *client.Client, previous schedulertypes.Run, run *SystemBackupRun, checkpoint systemBackupRecoveryInternal, key string) (backup.Snapshot, backup.Repository, backup.Repository, error) {
+ var staged backup.Snapshot
+ var remote backup.Repository
+ local, err := s.localRepositoryInternal(ctx, dockerClient, false)
+ if err != nil && checkpoint.LocalEnabled {
+ return staged, local, remote, err
+ }
+ stageID, stageAttempted := systemBackupStageEvidenceInternal(previous, run)
+ if err == nil && (stageAttempted || stageID != "") {
+ snapshot, found, findErr := s.engine.FindRunSnapshot(ctx, dockerClient, local, key, run.ID, stageID)
+ if findErr == nil && found {
+ staged = snapshot
+ }
+ if checkpoint.LocalEnabled && findErr != nil {
+ return staged, local, remote, findErr
+ }
+ }
+ if !checkpoint.S3Enabled {
+ return staged, local, remote, nil
+ }
+ remote, err = s.remoteRepositoryInternal(ctx, run.S3DestinationID)
+ if err != nil {
+ return staged, local, remote, err
+ }
+ attempted := (stageAttempted && staged.ID == "") || systemBackupRemoteAttemptedInternal(previous, run.ID)
+ if attempted || run.RemoteSnapshotID != "" {
+ snapshot, found, err := s.engine.FindRunSnapshot(ctx, dockerClient, remote, key, run.ID, run.RemoteSnapshotID)
+ if err != nil {
+ return staged, local, remote, err
+ }
+ if found {
+ run.RemoteSnapshotID = snapshot.ID
+ if run.Size == 0 {
+ run.Size = snapshot.Size
+ }
+ }
+ }
+ return staged, local, remote, nil
+}
+
+func (s *SystemBackupService) finishRecoveredDestinationsInternal(ctx context.Context, dockerClient *client.Client, run *SystemBackupRun, checkpoint systemBackupRecoveryInternal, key string, staged backup.Snapshot, local, remote backup.Repository) error {
+ if checkpoint.LocalEnabled && staged.ID == "" && run.RemoteSnapshotID != "" {
+ var err error
+ staged, err = s.engine.Replicate(ctx, dockerClient, remote, run.RemoteSnapshotID, local, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
+ if err != nil {
+ return err
+ }
+ }
+ if staged.ID == "" && run.RemoteSnapshotID == "" {
+ _, err := s.executeBackupInternal(ctx, &preparedSystemBackupInternal{run: run, recoveryKey: key, localEnabled: checkpoint.LocalEnabled, s3Enabled: checkpoint.S3Enabled})
+ return err
+ }
+ if checkpoint.LocalEnabled {
+ run.LocalSnapshotID, run.Size = staged.ID, staged.Size
+ }
+ if checkpoint.S3Enabled && run.RemoteSnapshotID == "" {
+ if staged.ID == "" {
+ return errors.New("System backup has no snapshot to replicate")
+ }
+ snapshot, err := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
+ if err != nil {
+ return err
+ }
+ run.RemoteSnapshotID = snapshot.ID
+ }
+ if !checkpoint.LocalEnabled && staged.ID != "" {
+ return s.engine.ForgetSnapshots(ctx, dockerClient, local, key, []string{staged.ID})
+ }
+ return nil
+}
+
+func (s *SystemBackupService) resumeBackupInternal(ctx context.Context, previous schedulertypes.Run, run *SystemBackupRun, checkpoint systemBackupRecoveryInternal, suppliedKey string) error {
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ key, err := s.recoveryKeyInternal(ctx, suppliedKey)
+ if err != nil {
+ return err
+ }
+ staged, local, remote, err := s.observeBackupSnapshotsInternal(ctx, dockerClient, previous, run, checkpoint, key)
+ if err != nil {
+ return err
+ }
+ if err := s.finishRecoveredDestinationsInternal(ctx, dockerClient, run, checkpoint, key, staged, local, remote); err != nil {
+ return err
+ }
+ run.Status, run.Error = SystemBackupStatusSucceeded, ""
+ return s.db.WithContext(ctx).Save(run).Error
+}
+
+type manualSystemBackupInternal struct {
+ Checkpoint systemBackupRecoveryInternal `json:"checkpoint"`
+ EncryptedKey string `json:"encryptedKey"`
+ ActivityID string `json:"activityId"`
+ UserID string `json:"userId"`
+}
+
+type manualSystemVolumesInternal struct {
+ Policy backuptypes.SystemVolumeBackupPolicy `json:"policy"`
+ ManualPolicy bool `json:"manualPolicy"`
+ Candidates []backuptypes.SystemVolumeBackupOption `json:"candidates"`
+ ActivityID string `json:"activityId"`
+ UserID string `json:"userId"`
+}
+
+func (s *SystemBackupService) executeDurableBackupInternal(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {
+ var command manualSystemBackupInternal
+ if err = json.Unmarshal(payload, &command); err != nil {
+ return err
+ }
+ defer func() {
+ if ctx.Err() == nil {
+ if err != nil {
+ saveErr := s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("id = ?", command.Checkpoint.BackupID).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": err.Error()}).Error
+ err = errors.Combine(err, saveErr)
+ }
+ activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Arcane system backup created successfully", err)
+ }
+ }()
+ var run SystemBackupRun
+ if err = s.db.WithContext(ctx).Where("id = ?", command.Checkpoint.BackupID).First(&run).Error; err != nil {
+ return err
+ }
+ if run.Status == SystemBackupStatusSucceeded {
+ return nil
+ }
+ key, err := crypto.Decrypt(command.EncryptedKey)
+ if err != nil {
+ return err
+ }
+ if interrupted {
+ previous, _ := jobcontext.Run(ctx)
+ outcome, recoveryErr := s.reconcileBackupInternal(ctx, previous, runID, key)
+ if recoveryErr != nil {
+ return recoveryErr
+ }
+ if outcome.Status != schedulertypes.Succeeded {
+ return errors.New(outcome.Message)
+ }
+ return nil
+ }
+ lease, admitted, err := s.engine.AcquireDurableRun(ctx, runID, backup.SystemAdmissionScope, systemAdmissionID)
+ if err != nil {
+ return err
+ }
+ if !admitted {
+ return ErrSystemBackupAlreadyRunning
+ }
+ defer lease.Release(ctx)
+ checkpoint, err := json.Marshal(command.Checkpoint)
+ if err != nil {
+ return err
+ }
+ if err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "system_backup", ID: runID, Status: schedulertypes.Running, RecoveryData: checkpoint, ActivityID: command.ActivityID}); err != nil {
+ return err
+ }
+ _, err = s.executeBackupInternal(ctx, &preparedSystemBackupInternal{run: &run, recoveryKey: key, localEnabled: command.Checkpoint.LocalEnabled, s3Enabled: command.Checkpoint.S3Enabled})
+ return err
+}
+
+func (s *SystemBackupService) executeDurableVolumeBackupsInternal(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {
+ var command manualSystemVolumesInternal
+ if err = json.Unmarshal(payload, &command); err != nil {
+ return err
+ }
+ defer func() {
+ if ctx.Err() == nil {
+ activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "System-managed volume backups completed", err)
+ }
+ }()
+ lease, admitted, err := s.engine.AcquireDurableRun(ctx, runID, backup.SystemAdmissionScope, systemAdmissionID)
+ if err != nil {
+ return err
+ }
+ if !admitted {
+ return ErrSystemBackupAlreadyRunning
+ }
+ defer lease.Release(ctx)
+ if interrupted {
+ ctx, err = s.reconcileVolumeCandidatesInternal(ctx, command.Candidates)
+ if err != nil {
+ return err
+ }
+ }
+ result, err := s.executeSystemVolumeBackupsInternal(ctx, command.Policy, command.ManualPolicy, command.Candidates, volume.VolumeBackupTriggerManual, command.ActivityID)
+ if err != nil {
+ return err
+ }
+ if result.Failed > 0 {
+ return fmt.Errorf("%d volume backups failed", result.Failed)
+ }
+ return nil
+}
+
+func (s *SystemBackupService) reconcileVolumeCandidatesInternal(ctx context.Context, candidates []backuptypes.SystemVolumeBackupOption) (context.Context, error) {
+ previous, _ := jobcontext.Run(ctx)
+ for _, candidate := range candidates {
+ pending := false
+ for _, target := range previous.Outcome.Targets {
+ if target.ID == candidate.Name && target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {
+ pending = true
+ break
+ }
+ }
+ if !pending {
+ continue
+ }
+ outcome, recoveryErr := s.volumeService.ReconcileBackup(ctx, previous, candidate.Name)
+ if recoveryErr != nil {
+ return ctx, recoveryErr
+ }
+ if outcome.Status != schedulertypes.Succeeded {
+ return ctx, errors.New(outcome.Message)
+ }
+ for i := range previous.Outcome.Targets {
+ if previous.Outcome.Targets[i].ID == candidate.Name {
+ previous.Outcome.Targets[i].Status = schedulertypes.Succeeded
+ }
+ }
+ }
+ progressCtx := ctx
+ ctx = jobcontext.WithExecution(ctx, previous, func(target schedulertypes.TargetOutcome) error { return jobcontext.Progress(progressCtx, target) })
+ return ctx, nil
+}
+
+func (s *SystemBackupService) failDurableBackupInternal(ctx context.Context, _ string, payload []byte, runErr error) error {
+ var command manualSystemBackupInternal
+ if err := json.Unmarshal(payload, &command); err != nil {
+ return err
+ }
+ err := s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("id = ?", command.Checkpoint.BackupID).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": runErr.Error()}).Error
+ activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Arcane system backup created successfully", runErr)
+ return err
+}
+
+func (s *SystemBackupService) failDurableVolumeBackupsInternal(ctx context.Context, _ string, payload []byte, runErr error) error {
+ var command manualSystemVolumesInternal
+ if err := json.Unmarshal(payload, &command); err != nil {
+ return err
+ }
+ activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "System-managed volume backups completed", runErr)
+ return nil
+}
diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.go
index 5c566fc5f..ffb74ee75 100644
--- a/backend/internal/systembackup/service.go
+++ b/backend/internal/systembackup/service.go
@@ -1,6 +1,7 @@
package systembackup
import (
+ "cmp"
"context"
"encoding/json/jsontext"
"encoding/json/v2"
@@ -14,10 +15,10 @@ import (
"slices"
"strings"
"time"
+ "uuid"
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -34,13 +35,13 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
recoverytypes "github.com/getarcaneapp/arcane/types/v2/recovery"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/google/uuid"
containertypes "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
@@ -82,12 +83,17 @@ type SystemBackupService struct {
}
func NewSystemBackupService(db *database.DB, dockerService *docker.DockerClientService, volumeService *volume.VolumeService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config, recoveryKeys *backup.RecoveryKeyStore) *SystemBackupService {
- return &SystemBackupService{
+ service := &SystemBackupService{
db: db, dockerService: dockerService, volumeService: volumeService, engine: engine,
s3Destinations: s3Destinations, activityService: activityService, settingsService: settingsService, config: cfg,
recoveryKeys: recoveryKeys,
jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
}
+ if engine != nil {
+ engine.RegisterRunKind("system", service.executeDurableBackupInternal, service.failDurableBackupInternal)
+ engine.RegisterRunKind("system-volumes", service.executeDurableVolumeBackupsInternal, service.failDurableVolumeBackupsInternal)
+ }
+ return service
}
// SupportedDatabaseProvider reports whether system recovery works on the
@@ -221,10 +227,6 @@ func (s *SystemBackupService) remoteRepositoryInternal(ctx context.Context, dest
}, nil
}
-func (s *SystemBackupService) createSnapshotInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey string, layout backupSourceLayoutInternal) (backup.Snapshot, error) {
- return s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", backup.CreateSnapshotInput{Mounts: layout.mounts, Sources: layout.sources})
-}
-
func (s *SystemBackupService) recoveryKeyInternal(ctx context.Context, supplied string) (string, error) {
if strings.TrimSpace(supplied) != "" {
if err := backup.ValidateRecoveryKey(supplied); err != nil {
@@ -290,7 +292,7 @@ func (s *SystemBackupService) CreateBackup(ctx context.Context, user common.User
if !admitted {
return nil, ErrSystemBackupAlreadyRunning
}
- defer lease.Release()
+ defer lease.Release(ctx)
return s.createBackupInternal(ctx, trigger, request)
}
@@ -298,7 +300,7 @@ func (s *SystemBackupService) createStagedSystemRecoverySnapshotInternal(ctx con
localRepository, err := s.localRepositoryInternal(ctx, dockerClient, false)
var snapshot backup.Snapshot
if err == nil {
- snapshot, err = s.snapshotWithDatabaseLockInternal(ctx, dockerClient, localRepository, recoveryKey, backupID)
+ snapshot, err = s.snapshotWithStagedDatabaseInternal(ctx, dockerClient, localRepository, recoveryKey, backupID)
}
if err == nil {
return snapshot, localRepository, false, nil
@@ -307,20 +309,25 @@ func (s *SystemBackupService) createStagedSystemRecoverySnapshotInternal(ctx con
if localEnabled || !s3Enabled {
return backup.Snapshot{}, backup.Repository{}, false, localStagingErr
}
- // S3-only backups normally use local staging so the database lock does
- // not span an upload. If local storage is unusable, write directly to
+ // S3-only backups normally use local repository staging. If local storage is unusable, write directly to
// S3 so remote backups and restore safety snapshots remain available.
remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)
if repoErr != nil {
return backup.Snapshot{}, backup.Repository{}, false, errors.Combine(localStagingErr, repoErr)
}
- remoteSnapshot, snapshotErr := s.snapshotWithDatabaseLockInternal(ctx, dockerClient, remoteRepository, recoveryKey, backupID)
+ remoteSnapshot, snapshotErr := s.snapshotWithStagedDatabaseInternal(ctx, dockerClient, remoteRepository, recoveryKey, backupID)
if snapshotErr != nil {
return backup.Snapshot{}, backup.Repository{}, false, errors.Combine(localStagingErr, fmt.Errorf("failed to create S3 system recovery snapshot: %w", snapshotErr))
}
return remoteSnapshot, remoteRepository, true, nil
}
+type systemBackupRecoveryInternal struct {
+ BackupID string `json:"backupId"`
+ LocalEnabled bool `json:"localEnabled"`
+ S3Enabled bool `json:"s3Enabled"`
+}
+
type preparedSystemBackupInternal struct {
run *SystemBackupRun
recoveryKey string
@@ -349,10 +356,17 @@ func (s *SystemBackupService) prepareBackupInternal(ctx context.Context, trigger
return nil, err
}
run := &SystemBackupRun{CreatedAt: time.Now().UTC(), Status: SystemBackupStatusRunning, Trigger: trigger, Destination: destination, S3DestinationID: destinationID, PolicyID: request.PolicyID}
- run.ID = "system-" + uuid.NewString()
+ run.ID = "system-" + uuid.New().String()
if err := s.db.WithContext(ctx).Create(run).Error; err != nil {
return nil, err
}
+ checkpoint, err := json.Marshal(systemBackupRecoveryInternal{BackupID: run.ID, LocalEnabled: localEnabled, S3Enabled: s3Enabled})
+ if err == nil {
+ err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: schedulertypes.Running, RecoveryData: checkpoint})
+ }
+ if err != nil {
+ return nil, err
+ }
return &preparedSystemBackupInternal{run: run, recoveryKey: recoveryKey, localEnabled: localEnabled, s3Enabled: s3Enabled}, nil
}
@@ -360,38 +374,33 @@ func (s *SystemBackupService) executeBackupInternal(ctx context.Context, prepare
run, recoveryKey := prepared.run, prepared.recoveryKey
localEnabled, s3Enabled, destinationID := prepared.localEnabled, prepared.s3Enabled, run.S3DestinationID
- defer func() {
- if err == nil {
- err = ctx.Err()
- }
- if err != nil {
- run.Status, run.Error = SystemBackupStatusFailed, err.Error()
- } else {
- run.Status, run.Error = SystemBackupStatusSucceeded, ""
- }
- if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(run).Error; saveErr != nil {
- err = errors.Combine(err, fmt.Errorf("failed to save system backup result: %w", saveErr))
- }
- }()
+ defer func() { err = s.completeBackupInternal(ctx, run, err) }()
defer utils.RecoverToError(&err, "system backup")
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return run, err
}
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":stage", Status: schedulertypes.Running}); err != nil {
+ return run, err
+ }
stagedSnapshot, stagingRepository, directRemote, err := s.createStagedSystemRecoverySnapshotInternal(
ctx, dockerClient, recoveryKey, run.ID, destinationID, localEnabled, s3Enabled,
)
if err != nil {
return run, err
}
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":stage", Status: schedulertypes.Succeeded, Message: stagedSnapshot.ID}); err != nil {
+ return run, err
+ }
if directRemote {
run.RemoteSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size
+ if err := s.db.WithContext(ctx).Save(run).Error; err != nil {
+ return run, err
+ }
return run, nil
}
localRepository := stagingRepository
- // The database is locked only while the local staging snapshot is taken.
- // An S3 copy replicates that immutable snapshot afterwards, so the
- // exclusive lock never spans a network upload and the source is read once.
+ // Both destinations use the same staged database copy.
// The staged snapshot is kept only for local-enabled runs. Forgetting it in
// a defer covers the failure paths too: a failed replication otherwise
// orphans one unreferenced snapshot in the staging repository per run.
@@ -406,56 +415,63 @@ func (s *SystemBackupService) executeBackupInternal(ctx context.Context, prepare
}()
if localEnabled {
run.LocalSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size
- }
- if s3Enabled {
- remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)
- if repoErr != nil {
- return run, repoErr
+ if err := s.db.WithContext(ctx).Save(run).Error; err != nil {
+ return run, err
}
- remoteSnapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, localRepository, stagedSnapshot.ID, remoteRepository, recoveryKey, "arcane-system-recovery")
- if replicateErr != nil {
- return run, fmt.Errorf("failed to create S3 system recovery snapshot: %w", replicateErr)
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":local", Status: schedulertypes.Succeeded, Message: stagedSnapshot.ID}); err != nil {
+ return run, err
}
- run.RemoteSnapshotID = remoteSnapshot.ID
- if run.Size == 0 {
- run.Size = remoteSnapshot.Size
+ }
+ if s3Enabled {
+ if err := s.replicateBackupInternal(ctx, dockerClient, prepared, localRepository, stagedSnapshot); err != nil {
+ return run, err
}
}
return run, nil
}
-// snapshotWithDatabaseLockInternal takes a snapshot while the SQLite database
-// is checkpointed and exclusively locked, and releases the lock before returning.
-func (s *SystemBackupService) snapshotWithDatabaseLockInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey, backupID string) (backup.Snapshot, error) {
- layout, err := s.backupSourceLayoutInternal(ctx, dockerClient)
+func (s *SystemBackupService) completeBackupInternal(ctx context.Context, run *SystemBackupRun, err error) error {
+ if err == nil {
+ err = ctx.Err()
+ }
if err != nil {
- return backup.Snapshot{}, err
+ run.Status, run.Error = SystemBackupStatusFailed, err.Error()
+ } else {
+ run.Status, run.Error = SystemBackupStatusSucceeded, ""
}
- if err := s.writeManifestInternal(ctx, backupID, layout); err != nil {
- return backup.Snapshot{}, err
+ if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(run).Error; saveErr != nil {
+ err = errors.Combine(err, fmt.Errorf("failed to save system backup result: %w", saveErr))
}
- defer func() { _ = os.Remove(layout.manifestPathInternal()) }()
- sqlDB, err := s.db.SQLDB()
- if err != nil {
- return backup.Snapshot{}, err
+ if err == nil {
+ err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: schedulertypes.Succeeded})
}
- if _, err := sqlDB.ExecContext(ctx, "PRAGMA wal_checkpoint(FULL)"); err != nil {
- return backup.Snapshot{}, fmt.Errorf("failed to checkpoint Arcane database: %w", err)
+ return err
+}
+
+func (s *SystemBackupService) replicateBackupInternal(ctx context.Context, dockerClient *client.Client, prepared *preparedSystemBackupInternal, localRepository backup.Repository, stagedSnapshot backup.Snapshot) error {
+ run := prepared.run
+ remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, run.S3DestinationID)
+ if repoErr != nil {
+ return repoErr
}
- connection, err := sqlDB.Conn(ctx)
- if err != nil {
- return backup.Snapshot{}, fmt.Errorf("failed to reserve Arcane database connection: %w", err)
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":remote", Status: schedulertypes.Running}); err != nil {
+ return err
}
- defer func() { _ = connection.Close() }()
- if _, err := connection.ExecContext(ctx, "BEGIN EXCLUSIVE"); err != nil {
- return backup.Snapshot{}, fmt.Errorf("failed to lock Arcane database for backup: %w", err)
+ remoteSnapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, localRepository, stagedSnapshot.ID, remoteRepository, prepared.recoveryKey, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))
+ if replicateErr != nil {
+ return fmt.Errorf("failed to create S3 system recovery snapshot: %w", replicateErr)
}
- defer func() { _, _ = connection.ExecContext(context.WithoutCancel(ctx), "ROLLBACK") }()
- snapshot, err := s.createSnapshotInternal(ctx, dockerClient, repository, recoveryKey, layout)
- if err != nil {
- return backup.Snapshot{}, fmt.Errorf("failed to create system recovery snapshot: %w", err)
+ run.RemoteSnapshotID = remoteSnapshot.ID
+ if run.Size == 0 {
+ run.Size = remoteSnapshot.Size
}
- return snapshot, nil
+ if err := s.db.WithContext(ctx).Save(run).Error; err != nil {
+ return err
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":remote", Status: schedulertypes.Succeeded, Message: remoteSnapshot.ID}); err != nil {
+ return err
+ }
+ return nil
}
func (s *SystemBackupService) ListBackups(ctx context.Context, params pagination.QueryParams) ([]backuptypes.SystemBackupRun, pagination.Response, error) {
@@ -856,7 +872,7 @@ func (s *SystemBackupService) RestoreBackupFiles(ctx context.Context, id string,
if !admitted {
return ErrSystemBackupAlreadyRunning
}
- defer lease.Release()
+ defer lease.Release(ctx)
run, err := s.backupInternal(ctx, id)
if err != nil {
return err
@@ -925,7 +941,7 @@ func (s *SystemBackupService) DeleteBackup(ctx context.Context, id, recoveryKey
if !admitted {
return ErrSystemBackupAlreadyRunning
}
- defer lease.Release()
+ defer lease.Release(ctx)
run, err := s.backupInternal(ctx, id)
if err != nil {
return err
@@ -1076,7 +1092,7 @@ func (s *SystemBackupService) UploadBackup(ctx context.Context, id string, reque
if !admitted {
return nil, ErrSystemBackupAlreadyRunning
}
- defer lease.Release()
+ defer lease.Release(ctx)
run, err := s.backupInternal(ctx, id)
if err != nil {
return nil, err
@@ -1424,7 +1440,7 @@ func (s *SystemBackupService) UpdatePolicies(ctx context.Context, updates []back
// SetScheduler injects the dynamic scheduler and admission gate for per-policy
// system backup jobs. Agent mode leaves them unset.
-func (s *SystemBackupService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {
+func (s *SystemBackupService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *runs.Admission) error {
return s.jobs.SetScheduler(ctx, scheduler, admissionGate)
}
@@ -1514,19 +1530,19 @@ func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.C
return s.runScheduledBackupInternal(ctx, policyID)
},
func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
- outcome := jobcontext.ConfirmedTarget(previous, policy.ID)
- if outcome.Status == schedulertypes.Succeeded {
- current, err := s.loadPolicyInternal(ctx, policyID)
- if err != nil {
+ outcome, err := s.reconcileBackupInternal(ctx, previous, policyID)
+ if err != nil || outcome.Status != schedulertypes.Succeeded {
+ return outcome, err
+ }
+ current, err := s.loadPolicyInternal(ctx, policyID)
+ if err != nil {
+ return outcome, err
+ }
+ if current != nil && current.RetentionCount > 0 {
+ if err := s.applyRetentionInternal(ctx, current.ID, current.RetentionCount, current.S3Enabled); err != nil {
+ outcome.Status = schedulertypes.Partial
return outcome, err
}
- if current != nil && current.RetentionCount > 0 {
- if err := s.applyRetentionInternal(ctx, policyID, current.RetentionCount, current.S3Enabled); err != nil {
- outcome.Status = schedulertypes.Partial
- outcome.Message = "Backup completed but retention failed"
- return outcome, err
- }
- }
}
return outcome, nil
},
@@ -1567,7 +1583,7 @@ func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policy
if !admitted {
return ErrSystemBackupAlreadyRunning
}
- defer lease.Release()
+ defer lease.Release(ctx)
var runs []*SystemBackupRun
if err := s.db.WithContext(ctx).Where("id IN ?", expired).Find(&runs).Error; err != nil {
return err
@@ -1608,6 +1624,7 @@ type backupSourceLayoutInternal struct {
projectsPath string
mounts []mount.Mount
sources []string
+ excludes []string
}
func (layout backupSourceLayoutInternal) manifestPathInternal() string {
diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.go
index 409bcf962..cbc36bd08 100644
--- a/backend/internal/systembackup/service_test.go
+++ b/backend/internal/systembackup/service_test.go
@@ -9,15 +9,15 @@ import (
"path/filepath"
"slices"
"testing"
- "time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
recoverytypes "github.com/getarcaneapp/arcane/types/v2/recovery"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -26,23 +26,15 @@ import (
mounttypes "github.com/moby/moby/api/types/mount"
"github.com/stretchr/testify/require"
"go.getarcane.app/sys/crypto"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
-func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {
+func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *runs.Admission {
t.Helper()
- fxLifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "system-backup-test-admission", t.Name())
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, fxLifecycle.Stop(stopCtx))
- })
+ runtime := francistest.New(t)
+ gate := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, gate.Register(runtime))
+ francistest.Start(t, runtime)
return gate
}
@@ -129,7 +121,7 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
}))
defer server.Close()
require.NoError(t, gormDB.AutoMigrate(&s3domain.S3Destination{}))
- service.s3Destinations = s3domain.NewS3DestinationService(service.db)
+ service.s3Destinations = s3domain.NewS3DestinationService(service.db, nil)
destination, err := service.s3Destinations.CreateS3Destination(t.Context(), backuptypes.CreateS3Destination{
Name: "Missing storage", Endpoint: server.URL, Bucket: "backups", AccessKeyID: "test", SecretAccessKey: "test", ForcePathStyle: true,
})
diff --git a/backend/internal/systembackup/snapshot.go b/backend/internal/systembackup/snapshot.go
new file mode 100644
index 000000000..162463c26
--- /dev/null
+++ b/backend/internal/systembackup/snapshot.go
@@ -0,0 +1,170 @@
+package systembackup
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "syscall"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/moby/moby/client"
+)
+
+// snapshotWithStagedDatabaseInternal archives a consistent database without blocking actor leases during the upload.
+func (s *SystemBackupService) snapshotWithStagedDatabaseInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey, backupID string) (backup.Snapshot, error) {
+ layout, err := s.backupSourceLayoutInternal(ctx, dockerClient)
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ stage, err := os.MkdirTemp(layout.dataDirectory, ".arcane-snapshot-")
+ if err != nil {
+ return backup.Snapshot{}, fmt.Errorf("create system backup staging directory: %w", err)
+ }
+ defer func() { _ = os.RemoveAll(stage) }()
+ sqlDB, err := s.db.SQLDB()
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ databasePath := filepath.Join(layout.dataDirectory, layout.databaseName)
+ stagedDatabase := filepath.Join(stage, layout.databaseName)
+ snapshotDatabase := filepath.Join(snapshotDataPath, layout.databaseName)
+ if err := s.writeManifestInternal(ctx, backupID, layout); err != nil {
+ return backup.Snapshot{}, err
+ }
+ defer func() { _ = os.Remove(layout.manifestPathInternal()) }()
+ layout.excludes = []string{
+ ".arcane-snapshot-*", systemRecoveryRequestName,
+ layout.databaseName + "-wal", layout.databaseName + "-shm", layout.databaseName + "-journal",
+ }
+ files, err := snapshotSourceFilesInternal(ctx, layout)
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ if err := stageSystemDatabaseInternal(ctx, sqlDB, databasePath, stagedDatabase); err != nil {
+ return backup.Snapshot{}, err
+ }
+ mounts, inContainer, err := currentMountsInternal(ctx, dockerClient)
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ stagedMounts, err := sourceMountsInternal(mounts, inContainer, stagedDatabase, snapshotDatabase)
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ layout.mounts = append(layout.mounts, stagedMounts...)
+ input := backup.CreateSnapshotInput{Mounts: layout.mounts, Sources: layout.sources}
+ for _, excluded := range layout.excludes {
+ input.Globs = append(input.Globs, "!"+snapshotDataPath+"/"+excluded)
+ }
+ snapshot, err := s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", input)
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ err = validateSnapshotSourcesInternal(ctx, layout, files)
+ if err == nil {
+ var confirmed backup.Snapshot
+ confirmed, err = s.engine.ConfirmRunSnapshot(ctx, dockerClient, repository, recoveryKey, backupID, snapshot.ID)
+ if err == nil {
+ return confirmed, nil
+ }
+ }
+ // Tagging changes the ID; only remove the original, unconfirmed snapshot.
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
+ defer cancel()
+ cleanupErr := s.engine.ForgetSnapshots(cleanupCtx, dockerClient, repository, recoveryKey, []string{snapshot.ID})
+ if cleanupErr != nil {
+ cleanupErr = fmt.Errorf("remove unconfirmed system snapshot %s: %w", snapshot.ID, cleanupErr)
+ }
+ return backup.Snapshot{}, errors.Combine(err, cleanupErr)
+}
+
+// snapshotSourceFilesInternal records inputs before taking the database snapshot.
+func snapshotSourceFilesInternal(ctx context.Context, layout backupSourceLayoutInternal) (map[string]os.FileInfo, error) {
+ roots := []string{layout.dataDirectory}
+ if layout.projectsPath == snapshotProjectsPath {
+ roots = append(roots, layout.projectsDirectory)
+ }
+ files := make(map[string]os.FileInfo)
+ visit := func(filePath string, entry fs.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ relative, err := filepath.Rel(layout.dataDirectory, filePath)
+ if err != nil {
+ return err
+ }
+ if relative == layout.databaseName {
+ return nil
+ }
+ for _, excluded := range layout.excludes {
+ matched, err := filepath.Match(excluded, relative)
+ if err != nil {
+ return err
+ }
+ if !matched {
+ continue
+ }
+ if entry.IsDir() {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ info, err := entry.Info()
+ if err != nil {
+ return err
+ }
+ files[filePath] = info
+ return nil
+ }
+ for _, root := range roots {
+ err := filepath.WalkDir(root, visit)
+ if err != nil {
+ return nil, fmt.Errorf("inspect system backup sources: %w", err)
+ }
+ }
+ return files, nil
+}
+
+func validateSnapshotSourcesInternal(ctx context.Context, layout backupSourceLayoutInternal, before map[string]os.FileInfo) error {
+ after, err := snapshotSourceFilesInternal(ctx, layout)
+ if err != nil {
+ return err
+ }
+ if len(before) != len(after) {
+ return errors.New("system backup source files changed during capture; retry when file updates finish")
+ }
+ for filePath, original := range before {
+ current, exists := after[filePath]
+ if !exists || !os.SameFile(original, current) || original.Size() != current.Size() || original.Mode() != current.Mode() || !original.ModTime().Equal(current.ModTime()) {
+ return fmt.Errorf("system backup source %q changed during capture; retry when file updates finish", filePath)
+ }
+ }
+ return nil
+}
+
+func stageSystemDatabaseInternal(ctx context.Context, db *sql.DB, databasePath, stagedDatabase string) error {
+ info, err := os.Stat(databasePath)
+ if err != nil {
+ return err
+ }
+ if _, err := db.ExecContext(ctx, "VACUUM INTO ?", stagedDatabase); err != nil {
+ return fmt.Errorf("stage Arcane database: %w", err)
+ }
+ if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+ if err := os.Chown(stagedDatabase, int(stat.Uid), int(stat.Gid)); err != nil {
+ return fmt.Errorf("preserve staged database ownership: %w", err)
+ }
+ }
+ if err := os.Chmod(stagedDatabase, info.Mode()); err != nil {
+ return fmt.Errorf("preserve staged database permissions: %w", err)
+ }
+ return nil
+}
diff --git a/backend/internal/systembackup/snapshot_test.go b/backend/internal/systembackup/snapshot_test.go
new file mode 100644
index 000000000..8cef1f8de
--- /dev/null
+++ b/backend/internal/systembackup/snapshot_test.go
@@ -0,0 +1,60 @@
+package systembackup
+
+import (
+ "database/sql"
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestStageSystemDatabaseExcludesLiveFilesInternal(t *testing.T) {
+ root := t.TempDir()
+ stage := filepath.Join(root, ".arcane-snapshot-stage")
+ require.NoError(t, os.Mkdir(stage, 0o700))
+ for _, name := range []string{systemRecoveryRequestName, "settings.txt"} {
+ require.NoError(t, os.WriteFile(filepath.Join(root, name), []byte("data"), 0o600))
+ }
+ require.NoError(t, os.Symlink("settings.txt", filepath.Join(root, "link")))
+ db, err := sql.Open("sqlite", filepath.Join(root, "arcane.db"))
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = db.Close() })
+ _, err = db.ExecContext(t.Context(), "PRAGMA journal_mode=WAL; CREATE TABLE evidence (id TEXT); INSERT INTO evidence VALUES ('committed')")
+ require.NoError(t, err)
+ require.NoError(t, os.Chmod(filepath.Join(root, "arcane.db"), 0o640))
+ projects := t.TempDir()
+ composePath := filepath.Join(projects, "compose.yaml")
+ require.NoError(t, os.WriteFile(composePath, []byte("services: {}"), 0o600))
+ layout := backupSourceLayoutInternal{
+ dataDirectory: root,
+ databaseName: "arcane.db",
+ projectsDirectory: projects,
+ projectsPath: snapshotProjectsPath,
+ excludes: []string{".arcane-snapshot-*", systemRecoveryRequestName, "arcane.db-wal", "arcane.db-shm", "arcane.db-journal"},
+ }
+ files, err := snapshotSourceFilesInternal(t.Context(), layout)
+ require.NoError(t, err)
+ require.NoError(t, stageSystemDatabaseInternal(t.Context(), db, filepath.Join(root, "arcane.db"), filepath.Join(stage, "arcane.db")))
+ require.NoError(t, validateSnapshotSourcesInternal(t.Context(), layout, files))
+ require.NoError(t, os.WriteFile(composePath, []byte("services: {app: {image: nginx}}"), 0o600))
+ require.ErrorContains(t, validateSnapshotSourcesInternal(t.Context(), layout, files), "changed during capture")
+ entries, err := os.ReadDir(stage)
+ require.NoError(t, err)
+ require.Len(t, entries, 1)
+ require.Equal(t, "arcane.db", entries[0].Name())
+ info, err := entries[0].Info()
+ require.NoError(t, err)
+ require.Equal(t, os.FileMode(0o640), info.Mode().Perm())
+ original, err := os.Stat(filepath.Join(root, "arcane.db"))
+ require.NoError(t, err)
+ require.Equal(t, original.Sys().(*syscall.Stat_t).Uid, info.Sys().(*syscall.Stat_t).Uid)
+ require.Equal(t, original.Sys().(*syscall.Stat_t).Gid, info.Sys().(*syscall.Stat_t).Gid)
+ snapshot, err := sql.Open("sqlite", filepath.Join(stage, "arcane.db"))
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = snapshot.Close() })
+ var id string
+ require.NoError(t, snapshot.QueryRowContext(t.Context(), "SELECT id FROM evidence").Scan(&id))
+ require.Equal(t, "committed", id)
+}
diff --git a/backend/internal/systembackup/system_volume.go b/backend/internal/systembackup/system_volume.go
index e1cd66b90..290c7da68 100644
--- a/backend/internal/systembackup/system_volume.go
+++ b/backend/internal/systembackup/system_volume.go
@@ -7,19 +7,19 @@ import (
"log/slog"
"slices"
"strings"
+ "uuid"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/google/uuid"
kit "go.getarcane.app/kit/pkg"
"gorm.io/gorm"
)
@@ -142,7 +142,7 @@ func (s *SystemBackupService) UpdateSystemVolumeBackupConfig(ctx context.Context
ID: func(policy *backuptypes.SystemVolumeBackupPolicy) string { return policy.ID },
UpdateID: func(update backuptypes.UpdateSystemVolumeBackupPolicy) string { return update.ID },
New: func() backuptypes.SystemVolumeBackupPolicy {
- return backuptypes.SystemVolumeBackupPolicy{ID: uuid.NewString()}
+ return backuptypes.SystemVolumeBackupPolicy{ID: uuid.New().String()}
},
Build: func(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy, update backuptypes.UpdateSystemVolumeBackupPolicy) error {
normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdateInternal(ctx, update)
@@ -267,7 +267,7 @@ type preparedSystemVolumeBackupInternal struct {
policy backuptypes.SystemVolumeBackupPolicy
manualPolicy bool
candidates []backuptypes.SystemVolumeBackupOption
- lease *actors.Lease[actors.AdmissionKey]
+ lease *runs.Lease
}
func (s *SystemBackupService) runSystemVolumeBackupsInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest, trigger volume.VolumeBackupTrigger) (*backuptypes.SystemVolumeBackupRunResult, error) {
@@ -275,7 +275,14 @@ func (s *SystemBackupService) runSystemVolumeBackupsInternal(ctx context.Context
if err != nil {
return nil, err
}
- defer prepared.lease.Release()
+ defer prepared.lease.Release(ctx)
+ frozen, err := json.Marshal(manualSystemVolumesInternal{Policy: prepared.policy, ManualPolicy: prepared.manualPolicy, Candidates: prepared.candidates})
+ if err != nil {
+ return nil, err
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_plan", ID: "system-volume-plan", Status: schedulertypes.Succeeded, RecoveryData: frozen}); err != nil {
+ return nil, err
+ }
return s.executeSystemVolumeBackupsInternal(ctx, prepared.policy, prepared.manualPolicy, prepared.candidates, trigger, "")
}
@@ -296,7 +303,7 @@ func (s *SystemBackupService) prepareSystemVolumeBackupsInternal(ctx context.Con
}
options, err := s.volumeService.ListBackupVolumeOptions(ctx)
if err != nil {
- lease.Release()
+ lease.Release(ctx)
return nil, err
}
candidates := selectSystemVolumeBackupCandidatesInternal(policyConfig, options)
@@ -316,18 +323,14 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Con
S3DestinationID: policyConfig.S3DestinationID,
}
for _, candidate := range candidates {
- if previous, ok := jobcontext.Run(ctx); ok {
- completed := false
- for _, target := range previous.Outcome.Targets {
- if target.ID == candidate.Name && target.Status == schedulertypes.Succeeded {
- completed = true
- break
- }
- }
- if completed {
- result.Succeeded++
- continue
- }
+ status := completedVolumeBackupStatusInternal(ctx, candidate.Name)
+ if status == schedulertypes.Succeeded {
+ result.Succeeded++
+ continue
+ }
+ if status == schedulertypes.Skipped {
+ result.Skipped++
+ continue
}
if err := ctx.Err(); err != nil {
return result, err
@@ -340,6 +343,9 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Con
continue
}
if overridden {
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: candidate.Name, Status: schedulertypes.Skipped}); err != nil {
+ return result, err
+ }
result.Skipped++
continue
}
@@ -347,9 +353,6 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Con
if manualPolicy {
seriesID = systemVolumeManualPolicyIDInternal(candidate.Name)
}
- if progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: candidate.Name, Status: schedulertypes.Running}); progressErr != nil {
- return result, progressErr
- }
_, backupErr := s.volumeService.CreateSystemManagedBackup(ctx, candidate.Name, common.SystemUser, trigger, seriesID, policy)
if errors.Is(backupErr, volume.ErrVolumeBackupAlreadyRunning) {
result.Skipped++
@@ -368,6 +371,19 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Con
return result, nil
}
+func completedVolumeBackupStatusInternal(ctx context.Context, volumeName string) schedulertypes.RunStatus {
+ previous, ok := jobcontext.Run(ctx)
+ if !ok {
+ return ""
+ }
+ for _, target := range previous.Outcome.Targets {
+ if target.ID == volumeName && (target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped) {
+ return target.Status
+ }
+ }
+ return ""
+}
+
func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {
policy, err := s.systemVolumeBackupPolicyInternal(policyID)
if err != nil {
@@ -426,11 +442,15 @@ func (s *SystemBackupService) rescheduleSystemVolumeBackupInternal(ctx context.C
return s.runScheduledSystemVolumeBackupInternal(ctx, policyID)
}, func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
for _, target := range previous.Outcome.Targets {
- if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Targets: previous.Outcome.Targets, Message: "A volume backup has an unconfirmed outcome"}, nil
+ if target.ID != "system-volume-plan" || len(target.RecoveryData) == 0 {
+ continue
+ }
+ if err := s.executeDurableVolumeBackupsInternal(ctx, previous.ID, target.RecoveryData, true); err != nil {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: err.Error()}, err
}
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
- return s.runScheduledSystemVolumeBackupInternal(jobcontext.WithExecution(ctx, previous, nil), policyID)
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The interrupted volume backup has no frozen selection"}, nil
})
}
diff --git a/backend/internal/systembackup/system_volume_test.go b/backend/internal/systembackup/system_volume_test.go
index b21702e5e..192c7bd02 100644
--- a/backend/internal/systembackup/system_volume_test.go
+++ b/backend/internal/systembackup/system_volume_test.go
@@ -105,7 +105,7 @@ func TestHistoryDestinationDecorationInternal(t *testing.T) {
require.NoError(t, db.AutoMigrate(&s3domain.S3Destination{}))
destination := s3domain.S3Destination{Name: "Offsite", Bucket: "backups"}
require.NoError(t, db.Create(&destination).Error)
- service := s3domain.NewS3DestinationService(&database.DB{DB: db})
+ service := s3domain.NewS3DestinationService(&database.DB{DB: db}, nil)
history := []backuptypes.HistoryEntry{{S3DestinationID: destination.ID}, {S3DestinationID: "missing"}}
decorateHistoryDestinationsInternal(t.Context(), service, history)
require.Equal(t, "Offsite", history[0].S3DestinationName)
diff --git a/backend/internal/template/module.go b/backend/internal/template/module.go
index 4f78117eb..d20fa086d 100644
--- a/backend/internal/template/module.go
+++ b/backend/internal/template/module.go
@@ -2,27 +2,15 @@
package template
import (
- "context"
- "net/http"
-
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-type Dependencies struct {
- Context context.Context
- DB *database.DB
- HTTPClient *http.Client
- Settings *settings.SettingsService
-}
-
type Module struct {
service *TemplateService
}
-func New(deps Dependencies) *Module {
- return &Module{service: NewTemplateService(deps.Context, deps.DB, deps.HTTPClient, deps.Settings)}
+func New(service *TemplateService) *Module {
+ return &Module{service: service}
}
func (m *Module) Service() *TemplateService {
diff --git a/backend/internal/template/service_test.go b/backend/internal/template/service_test.go
index 3af106202..20a60bc65 100644
--- a/backend/internal/template/service_test.go
+++ b/backend/internal/template/service_test.go
@@ -14,7 +14,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
@@ -24,7 +23,6 @@ import (
"github.com/libtnb/sqlite"
"github.com/samber/mo"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -426,19 +424,9 @@ func minimalSettingsServiceForTest(t *testing.T) *settings.SettingsService {
func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "template-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "template-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, executor, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
diff --git a/backend/internal/updater/actors.go b/backend/internal/updater/actors.go
new file mode 100644
index 000000000..564d318a1
--- /dev/null
+++ b/backend/internal/updater/actors.go
@@ -0,0 +1,355 @@
+package updater
+
+import (
+ "context"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/host/local"
+)
+
+const (
+ singleUpdateTypeInternal = "single-container-update"
+ singleUpdateStateTypeInternal = "single-container-update-state"
+)
+
+type singleUpdateActorInternal struct{ service *UpdaterService }
+
+func (s *UpdaterService) RegisterActors(runtime *francis.Runtime) error {
+ s.singleUpdates = runtime.Service()
+ return runtime.RegisterActor(singleUpdateTypeInternal, func(_ string, _ *actor.Service) actor.Actor { return &singleUpdateActorInternal{service: s} }, local.WithCapacityGroup("jobs", 4), local.WithCompletedJobRetention(7*24*time.Hour))
+}
+
+// Start repairs persisted single-container delivery intents until shutdown.
+func (s *UpdaterService) Start(ctx context.Context) error {
+ s.singleMu.Lock()
+ defer s.singleMu.Unlock()
+ if s.singleContext != nil {
+ return nil
+ }
+ s.singleContext, s.singleCancel = context.WithCancel(ctx)
+ s.singleDone = make(chan struct{})
+ go func() {
+ defer close(s.singleDone)
+ ticker := time.NewTicker(5 * time.Second)
+ defer ticker.Stop()
+ for {
+ if s.coordinator == nil || s.coordinator.Active() {
+ if err := s.repairSinglesInternal(s.singleContext); err != nil && s.singleContext.Err() == nil {
+ s.loggerInternal().WarnContext(s.singleContext, "repair container update dispatch", "error", err)
+ }
+ }
+ select {
+ case <-s.singleContext.Done():
+ return
+ case <-ticker.C:
+ }
+ }
+ }()
+ return nil
+}
+
+// Stop cancels business execution before the actor host drains jobs.
+func (s *UpdaterService) Stop(ctx context.Context) error {
+ s.singleMu.Lock()
+ s.singleStopping = true
+ if s.singleCancel != nil {
+ s.singleCancel()
+ }
+ done := s.singleDone
+ s.singleMu.Unlock()
+ joined := make(chan struct{})
+ go func() {
+ if done != nil {
+ <-done
+ }
+ s.singleWorkers.Wait()
+ close(joined)
+ }()
+ select {
+ case <-joined:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+func (s *UpdaterService) dispatchSingleInternal(ctx context.Context, state arcaneupdater.SingleUpdateState) error {
+ if s.coordinator != nil && !s.coordinator.Active() {
+ return nil
+ }
+ id, _, err := s.singleUpdates.Dispatch(ctx, singleUpdateTypeInternal, "single-container", "update", state.Command, actor.WithIdempotencyKey(state.Command.ActivityID))
+ if err != nil {
+ return err
+ }
+ job, err := s.singleUpdates.GetJob(ctx, id)
+ if err != nil {
+ return err
+ }
+ if !job.Status.IsTerminal() {
+ return nil
+ }
+ // A terminal transport record cannot prove the durable business outcome.
+ // Its replacement reconciles the saved effect before doing any further work.
+ if err := s.singleUpdates.DeleteJob(ctx, singleUpdateTypeInternal, "single-container", id); err != nil {
+ return err
+ }
+ _, _, err = s.singleUpdates.Dispatch(ctx, singleUpdateTypeInternal, "single-container", "update", state.Command, actor.WithIdempotencyKey(state.Command.ActivityID))
+ return err
+}
+
+func (s *UpdaterService) repairSinglesInternal(ctx context.Context) error {
+ for cursor := ""; ; {
+ page, err := s.singleUpdates.ListStates(ctx, singleUpdateStateTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ if err != nil {
+ return err
+ }
+ for _, entry := range page.States {
+ var state arcaneupdater.SingleUpdateState
+ if err := entry.Data.Decode(&state); err != nil {
+ return err
+ }
+ if err := s.repairSingleInternal(ctx, state); err != nil {
+ return err
+ }
+ }
+ cursor = page.AfterID()
+ if cursor == "" {
+ return nil
+ }
+ }
+}
+
+func (s *UpdaterService) repairSingleInternal(ctx context.Context, state arcaneupdater.SingleUpdateState) error {
+ if state.Status == "queued" || state.Status == "running" {
+ return s.dispatchSingleInternal(ctx, state)
+ }
+ if s.deps.Activity == nil {
+ return nil
+ }
+ detail, err := s.deps.Activity.GetActivityDetail(ctx, "0", state.Command.ActivityID, 1)
+ if err != nil {
+ return err
+ }
+ if detail.Activity.Status != activitytypes.StatusQueued && detail.Activity.Status != activitytypes.StatusRunning {
+ return nil
+ }
+ var outcomeErr error
+ if state.Failure != "" {
+ outcomeErr = errors.New(state.Failure)
+ }
+ s.finishSingleContainerUpdateInternal(ctx, state.Command.ActivityID, state.Result, outcomeErr)
+ return nil
+}
+
+func (a *singleUpdateActorInternal) Job(ctx context.Context, _ string, data actor.Envelope) error {
+ s := a.service
+ if s.coordinator != nil && !s.coordinator.Active() {
+ return actor.ErrJobRejected
+ }
+ s.singleMu.Lock()
+ if s.singleStopping {
+ s.singleMu.Unlock()
+ return actor.ErrJobRejected
+ }
+ s.singleWorkers.Add(1)
+ lifetime := s.singleContext
+ s.singleMu.Unlock()
+ defer s.singleWorkers.Done()
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ if lifetime != nil {
+ //nolint:contextcheck // The service lifetime cancels the inherited Francis job context before host drain.
+ stop := context.AfterFunc(lifetime, cancel)
+ defer stop()
+ }
+ ctx, release, err := s.acquireUpdateInternal(ctx)
+ if err != nil {
+ if errors.Is(err, errUpdateBusyInternal) {
+ return actor.ErrJobRejected
+ }
+ return err
+ }
+ defer release()
+ var command arcaneupdater.SingleUpdateCommand
+ if err := data.Decode(&command); err != nil {
+ return errors.Combine(err, actor.ErrJobPermanentFailure)
+ }
+ var state arcaneupdater.SingleUpdateState
+ if err := s.singleUpdates.GetState(ctx, singleUpdateStateTypeInternal, command.ActivityID, &state); err != nil {
+ return err
+ }
+ if state.Status == "completed" || state.Status == "needs_attention" {
+ return nil
+ }
+ if state.Command != command {
+ return actor.ErrJobPermanentFailure
+ }
+ if err := s.authorizeSingleInternal(ctx, command); err != nil {
+ return s.failSingleInternal(ctx, &state, err.Error())
+ }
+ ctx = s.trackActivityInternal(ctx, command.ActivityID)
+ canceled, err := s.awaitSingleActivityInternal(ctx, lifetime, &state)
+ if err != nil || canceled {
+ return err
+ }
+ completed, err := s.prepareSingleTargetInternal(ctx, &state)
+ if err != nil || completed {
+ return err
+ }
+ ctx = context.WithValue(ctx, frozenSingleKeyInternal{}, &frozenSingleInternal{Target: state.Target, Persist: func() error {
+ return s.singleUpdates.SetState(ctx, singleUpdateStateTypeInternal, command.ActivityID, state, nil)
+ }})
+ result, runErr := func() (out *arcaneupdater.Result, err error) {
+ defer utils.RecoverToError(&err, "single container update")
+ return s.runSingleContainerUpdateInternal(ctx, state.Target.ContainerID, command.ActivityID)
+ }()
+ return s.completeSingleInternal(ctx, &state, result, runErr)
+}
+
+func (s *UpdaterService) failSingleInternal(ctx context.Context, state *arcaneupdater.SingleUpdateState, reason string) error {
+ state.Status = "needs_attention"
+ state.Failure = reason
+ if err := s.singleUpdates.SetState(ctx, singleUpdateStateTypeInternal, state.Command.ActivityID, *state, nil); err != nil {
+ return err
+ }
+ s.finishSingleContainerUpdateInternal(ctx, state.Command.ActivityID, nil, errors.New(reason))
+ return actor.ErrJobPermanentFailure
+}
+
+func (s *UpdaterService) authorizeSingleInternal(ctx context.Context, command arcaneupdater.SingleUpdateCommand) error {
+ if command.UserID == "" {
+ return nil
+ }
+ if s.config != nil && s.config.AgentMode && command.UserID == "agent" && command.KeyID == "" {
+ return nil
+ }
+ if s.roles == nil {
+ return errors.New("requesting user permissions unavailable")
+ }
+ permissions, err := s.roles.ResolveExecutionPermissions(ctx, command.UserID, command.KeyID)
+ if err != nil || !permissions.Allows(authz.PermImageUpdatesCheck, "0") {
+ return errors.New("requesting user no longer has permission to update containers")
+ }
+ return nil
+}
+
+func (s *UpdaterService) awaitSingleActivityInternal(ctx, lifetime context.Context, state *arcaneupdater.SingleUpdateState) (bool, error) {
+ if s.deps.Activity == nil {
+ return false, nil
+ }
+ detail, err := s.deps.Activity.GetActivityDetail(ctx, "0", state.Command.ActivityID, 1)
+ if err != nil {
+ return false, err
+ }
+ if detail.Activity.Status == activitytypes.StatusCancelled {
+ state.Status = "needs_attention"
+ state.Failure = "Container update cancelled"
+ return true, s.singleUpdates.SetState(ctx, singleUpdateStateTypeInternal, state.Command.ActivityID, *state, nil)
+ }
+ err = s.deps.Activity.AwaitActivitySlotBounded(ctx, state.Command.ActivityID, "0")
+ if err == nil {
+ return false, nil
+ }
+ if ctx.Err() == nil {
+ return false, s.failSingleInternal(ctx, state, err.Error())
+ }
+ if lifetime != nil && lifetime.Err() != nil {
+ return false, ctx.Err()
+ }
+ state.Status = "needs_attention"
+ state.Failure = err.Error()
+ writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+ defer cancel()
+ return true, s.singleUpdates.SetState(writeCtx, singleUpdateStateTypeInternal, state.Command.ActivityID, *state, nil)
+}
+
+func (s *UpdaterService) prepareSingleTargetInternal(ctx context.Context, state *arcaneupdater.SingleUpdateState) (bool, error) {
+ if state.Status != "running" {
+ target, err := s.freezeContainerInternal(ctx, state.Command.ContainerID)
+ if err != nil {
+ return false, s.failSingleInternal(ctx, state, err.Error())
+ }
+ state.Target = target
+ state.Status = "running"
+ return false, s.singleUpdates.SetState(ctx, singleUpdateStateTypeInternal, state.Command.ActivityID, *state, nil)
+ }
+ if state.Target == nil {
+ return false, s.failSingleInternal(ctx, state, "Interrupted container update has no saved target evidence")
+ }
+ confirmed, unchanged, err := s.confirmFrozenTargetInternal(ctx, *state.Target)
+ if err != nil {
+ return false, err
+ }
+ if confirmed {
+ result := &arcaneupdater.Result{Updated: 1, Items: []arcaneupdater.ResourceResult{{ResourceID: state.Target.ContainerID, ResourceName: state.Target.ContainerName, ResourceType: "container", Status: arcaneupdater.StatusUpdated, UpdateApplied: true}}}
+ return true, s.completeSingleInternal(ctx, state, result, nil)
+ }
+ if !unchanged {
+ return false, s.failSingleInternal(ctx, state, "Interrupted container update effect cannot be confirmed")
+ }
+ return false, nil
+}
+
+func (s *UpdaterService) completeSingleInternal(ctx context.Context, state *arcaneupdater.SingleUpdateState, result *arcaneupdater.Result, runErr error) error {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if runErr != nil {
+ return s.failSingleInternal(ctx, state, runErr.Error())
+ }
+ if state.Target.DesiredImageRef != "" && result != nil && result.Updated > 0 {
+ confirmed, _, err := s.confirmFrozenTargetInternal(ctx, *state.Target)
+ if err != nil {
+ return err
+ }
+ if !confirmed {
+ return s.failSingleInternal(ctx, state, "Updater result does not confirm the frozen desired image")
+ }
+ }
+ state.Status = "completed"
+ state.Result = result
+ if err := s.singleUpdates.SetState(ctx, singleUpdateStateTypeInternal, state.Command.ActivityID, *state, nil); err != nil {
+ return err
+ }
+ s.finishSingleContainerUpdateInternal(ctx, state.Command.ActivityID, result, nil)
+ return nil
+}
+
+func singleUpdateCommandInternal(ctx context.Context, containerID, activityID string) arcaneupdater.SingleUpdateCommand {
+ command := arcaneupdater.SingleUpdateCommand{ContainerID: containerID, ActivityID: activityID}
+ if user, ok := common.CurrentUserFromContext(ctx); ok && user != nil {
+ command.UserID = user.ID
+ }
+ command.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ return command
+}
+
+func (s *UpdaterService) ActiveUpdateActivityIDs(ctx context.Context) ([]string, error) {
+ ids := []string{}
+ for cursor := ""; ; {
+ page, err := s.singleUpdates.ListStates(ctx, singleUpdateStateTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ if err != nil {
+ return nil, err
+ }
+ for _, entry := range page.States {
+ var state arcaneupdater.SingleUpdateState
+ if err := entry.Data.Decode(&state); err != nil {
+ return nil, err
+ }
+ ids = append(ids, state.Command.ActivityID)
+ }
+ cursor = page.AfterID()
+ if cursor == "" {
+ return ids, nil
+ }
+ }
+}
diff --git a/backend/internal/updater/admission.go b/backend/internal/updater/admission.go
new file mode 100644
index 000000000..2b6620101
--- /dev/null
+++ b/backend/internal/updater/admission.go
@@ -0,0 +1,26 @@
+package updater
+
+import (
+ "context"
+
+ "emperror.dev/errors"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+type updateAdmissionKeyInternal struct{}
+
+var errUpdateBusyInternal = errors.New("another container update is running")
+
+func (s *UpdaterService) acquireUpdateInternal(ctx context.Context) (context.Context, func(), error) {
+ if s.admission == nil || ctx.Value(updateAdmissionKeyInternal{}) == s {
+ return ctx, func() {}, nil
+ }
+ lease, admitted, err := s.admission.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: "updater"})
+ if err != nil {
+ return ctx, nil, err
+ }
+ if !admitted {
+ return ctx, nil, errUpdateBusyInternal
+ }
+ return context.WithValue(ctx, updateAdmissionKeyInternal{}, s), func() { lease.Release(ctx) }, nil
+}
diff --git a/backend/internal/updater/module.go b/backend/internal/updater/module.go
index cd31feada..8d2548935 100644
--- a/backend/internal/updater/module.go
+++ b/backend/internal/updater/module.go
@@ -4,61 +4,17 @@ package updater
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
- "github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/image"
- "github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
- "github.com/getarcaneapp/arcane/backend/v2/internal/notification"
- "github.com/getarcaneapp/arcane/backend/v2/internal/project"
- "github.com/getarcaneapp/arcane/backend/v2/internal/registry"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-// Dependencies are the collaborators the updater domain needs.
-type Dependencies struct {
- DB *database.DB
- Settings *settings.SettingsService
- Docker *docker.DockerClientService
- Project *project.ProjectService
- ImageUpdate *imageupdate.ImageUpdateService
- Registry *registry.ContainerRegistryService
- Event *event.EventService
- Image *image.ImageService
- Notification *notification.NotificationService
- SelfUpgrade selfUpgradeServiceInternal
- Activity *activity.ActivityService
- SingleUpdates *actors.Executor
-}
-
// Module wires the updater domain and mounts its routes.
type Module struct {
service *UpdaterService
}
-// New builds the updater domain from its dependencies.
-func New(deps Dependencies) (*Module, error) {
- service, err := NewUpdaterService(
- deps.DB,
- deps.Settings,
- deps.Docker,
- deps.Project,
- deps.ImageUpdate,
- deps.Registry,
- deps.Event,
- deps.Image,
- deps.Notification,
- deps.SelfUpgrade,
- deps.Activity,
- )
- if err != nil {
- return nil, err
- }
- service.singleUpdates = deps.SingleUpdates
- return &Module{service: service}, nil
+// New assembles updater routes around the provided service.
+func New(service *UpdaterService) *Module {
+ return &Module{service: service}
}
// Service exposes the updater service to collaborators that trigger updates
diff --git a/backend/internal/updater/recovery.go b/backend/internal/updater/recovery.go
index 1b024185d..caedf5927 100644
--- a/backend/internal/updater/recovery.go
+++ b/backend/internal/updater/recovery.go
@@ -3,6 +3,7 @@ package updater
import (
"cmp"
"context"
+ "encoding/json/v2"
"log/slog"
"strings"
"sync"
@@ -13,7 +14,10 @@ import (
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
+ "github.com/moby/moby/client"
+ "go.getarcane.app/docker/compat"
"go.getarcane.app/updater"
+ "go.getarcane.app/updater/refs"
)
type (
@@ -62,8 +66,19 @@ func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.Res
slog.DebugContext(ctx, "failed to append update result activity message", "activityId", activityID, "resource", name, "error", appendErr)
}
}
- progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: result.ResourceID, ResourceType: string(result.ResourceType), Status: status, Message: result.Error, ActivityID: activityIDFromContextInternal(ctx)})
- err := errors.Combine(recordErr, progressErr)
+ var evidenceErr error
+ if status == schedulertypes.Succeeded {
+ evidenceErr = s.verifyFrozenResultInternal(ctx, result.ResourceID)
+ }
+ if evidenceErr != nil {
+ status = schedulertypes.NeedsAttention
+ }
+ progressMessage := result.Error
+ if evidenceErr != nil {
+ progressMessage = evidenceErr.Error()
+ }
+ progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: result.ResourceID, ResourceType: string(result.ResourceType), Status: status, Message: progressMessage, ActivityID: activityIDFromContextInternal(ctx)})
+ err := errors.Combine(recordErr, progressErr, evidenceErr)
if progress, ok := ctx.Value(updateProgressKeyInternal{}).(*updateProgressInternal); ok && err != nil {
progress.mu.Lock()
progress.err = errors.Combine(progress.err, err)
@@ -72,6 +87,27 @@ func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.Res
return err
}
+func (s *UpdaterService) verifyFrozenResultInternal(ctx context.Context, resourceID string) error {
+ plan, ok := ctx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal)
+ if !ok {
+ return nil
+ }
+ for _, target := range plan.Targets {
+ if target.ContainerID != resourceID {
+ continue
+ }
+ confirmed, _, err := s.confirmFrozenTargetInternal(ctx, target)
+ if err != nil {
+ return err
+ }
+ if !confirmed {
+ return errors.New("updater result does not confirm the frozen desired image")
+ }
+ return nil
+ }
+ return nil
+}
+
func (p *updateProgressInternal) completeBatchInternal(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result, batchCompleted bool, err error) error {
activityID := activityIDFromContextInternal(ctx)
p.mu.Lock()
@@ -128,3 +164,339 @@ func updateBatchTypeInternal(ctx context.Context, options arcaneupdater.Options,
}
return batchType
}
+
+type (
+ frozenPendingKeyInternal struct{}
+ frozenSingleKeyInternal struct{}
+ frozenUpdatePlanInternal struct {
+ Records []updater.ImageUpdateRecord
+ Targets []arcaneupdater.FrozenUpdateTarget
+ }
+)
+
+type frozenSingleInternal struct {
+ Target *arcaneupdater.FrozenUpdateTarget
+ Persist func() error
+}
+
+// freezePendingInternal persists identities and desired images before any pull.
+func (s *UpdaterService) freezePendingInternal(ctx context.Context) (context.Context, error) {
+ if _, ok := ctx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal); ok {
+ return ctx, nil
+ }
+ run, ok := jobcontext.Run(ctx)
+ if !ok {
+ return ctx, nil
+ }
+ for _, target := range run.Outcome.Targets {
+ if target.ID == "auto-update" && len(target.RecoveryData) > 0 {
+ var plan frozenUpdatePlanInternal
+ if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {
+ return ctx, err
+ }
+ return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), nil
+ }
+ }
+ records, err := s.PendingImageUpdates(ctx)
+ if err != nil {
+ return ctx, err
+ }
+ plan, err := s.buildFrozenPlanInternal(ctx, records)
+ if err != nil {
+ return ctx, err
+ }
+ raw, err := json.Marshal(plan)
+ if err != nil {
+ return ctx, err
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "auto-update", ResourceType: "update-batch", Status: schedulertypes.Running, RecoveryData: raw, ActivityID: activityIDFromContextInternal(ctx)}); err != nil {
+ return ctx, err
+ }
+ for _, target := range plan.Targets {
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: target.ContainerID, ResourceType: "container", Status: schedulertypes.Queued}); err != nil {
+ return ctx, err
+ }
+ }
+ return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), nil
+}
+
+func (s *UpdaterService) buildFrozenPlanInternal(ctx context.Context, records []updater.ImageUpdateRecord) (frozenUpdatePlanInternal, error) {
+ plan := frozenUpdatePlanInternal{Records: make([]updater.ImageUpdateRecord, 0, len(records))}
+ if len(records) == 0 {
+ return plan, nil
+ }
+ dockerClient, err := s.DockerClient(ctx)
+ if err != nil {
+ return plan, err
+ }
+ listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
+ if err != nil {
+ return plan, err
+ }
+ for _, record := range records {
+ if !record.HasUpdate {
+ continue
+ }
+ for _, candidate := range listed.Items {
+ if record.ContainerID != "" && record.ContainerID != candidate.ID {
+ continue
+ }
+ if record.ContainerID == "" && refs.NormalizeImageUpdateRef(candidate.Image) != refs.NormalizeImageUpdateRef(record.ImageRef()) {
+ continue
+ }
+ target, selected, err := s.freezeRecordTargetInternal(ctx, record, candidate.ID)
+ if err != nil {
+ return plan, err
+ }
+ plan.Records = append(plan.Records, selected)
+ plan.Targets = append(plan.Targets, *target)
+ }
+ }
+ return plan, nil
+}
+
+func (s *UpdaterService) freezeRecordTargetInternal(ctx context.Context, record updater.ImageUpdateRecord, containerID string) (*arcaneupdater.FrozenUpdateTarget, updater.ImageUpdateRecord, error) {
+ target, err := s.freezeContainerInternal(ctx, containerID)
+ if err != nil {
+ return nil, record, err
+ }
+ target.DesiredImageRef = record.NewImageRef()
+ if record.LatestDigest != nil && !record.IsTagUpdate() {
+ target.DesiredDigest = *record.LatestDigest
+ }
+ if target.DesiredDigest == "" {
+ resolver := s.registryDigestResolverInternal()
+ if resolver == nil {
+ return nil, record, errors.New("cannot freeze update without registry digest resolver")
+ }
+ target.DesiredDigest, err = resolver.ImageDigest(ctx, target.DesiredImageRef)
+ if err != nil {
+ return nil, record, err
+ }
+ }
+ if target.DesiredDigest == "" {
+ return nil, record, errors.New("registry returned no desired digest")
+ }
+ record.ContainerID = containerID
+ digest := target.DesiredDigest
+ record.LatestDigest = &digest
+ return target, record, nil
+}
+
+func (s *UpdaterService) freezeContainerInternal(ctx context.Context, id string) (*arcaneupdater.FrozenUpdateTarget, error) {
+ dockerClient, err := s.DockerClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ result, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, id, client.ContainerInspectOptions{})
+ if err != nil {
+ return nil, err
+ }
+ inspected := result.Container
+ target := &arcaneupdater.FrozenUpdateTarget{ContainerID: inspected.ID, ContainerName: strings.TrimPrefix(inspected.Name, "/"), BaselineImageID: inspected.Image, BaselineRestartCount: inspected.RestartCount}
+ if inspected.State != nil {
+ target.BaselineStartedAt = inspected.State.StartedAt
+ }
+ if inspected.Config != nil {
+ target.ComposeProject = inspected.Config.Labels["com.docker.compose.project"]
+ target.ComposeService = inspected.Config.Labels["com.docker.compose.service"]
+ target.ComposeNumber = inspected.Config.Labels["com.docker.compose.container-number"]
+ }
+ return target, nil
+}
+
+func immutableImageInternal(target arcaneupdater.FrozenUpdateTarget) (string, error) {
+ reference, err := refs.NormalizeReference(target.DesiredImageRef)
+ if err != nil {
+ return "", err
+ }
+ if target.DesiredDigest == "" {
+ return "", errors.New("desired update digest is unavailable")
+ }
+ return reference.RegistryHost + "/" + reference.Repository + "@" + target.DesiredDigest, nil
+}
+
+// prepareFrozenPullInternal commits the exact image before the engine changes Docker.
+func (s *UpdaterService) prepareFrozenPullInternal(ctx context.Context, imageRef string) (string, error) {
+ if single, ok := ctx.Value(frozenSingleKeyInternal{}).(*frozenSingleInternal); ok {
+ return s.prepareSinglePullInternal(ctx, imageRef, single)
+ }
+ if plan, ok := ctx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal); ok {
+ return prepareBatchPullInternal(ctx, imageRef, plan)
+ }
+ return imageRef, nil
+}
+
+func (s *UpdaterService) prepareSinglePullInternal(ctx context.Context, imageRef string, single *frozenSingleInternal) (string, error) {
+ target := single.Target
+ if target.DesiredImageRef == "" {
+ target.DesiredImageRef = imageRef
+ resolver := s.registryDigestResolverInternal()
+ if resolver == nil {
+ return "", errors.New("cannot freeze update without registry digest resolver")
+ }
+ digest, err := resolver.ImageDigest(ctx, imageRef)
+ if err != nil {
+ return "", err
+ }
+ target.DesiredDigest = digest
+ if err := single.Persist(); err != nil {
+ return "", err
+ }
+ }
+ if refs.NormalizeImageUpdateRef(target.DesiredImageRef) != refs.NormalizeImageUpdateRef(imageRef) {
+ return "", errors.New("selected image changed after the update was accepted")
+ }
+ return immutableImageInternal(*target)
+}
+
+func prepareBatchPullInternal(ctx context.Context, imageRef string, plan *frozenUpdatePlanInternal) (string, error) {
+ immutable := ""
+ for _, target := range plan.Targets {
+ if refs.NormalizeImageUpdateRef(target.DesiredImageRef) != refs.NormalizeImageUpdateRef(imageRef) {
+ continue
+ }
+ selected, err := immutableImageInternal(target)
+ if err != nil {
+ return "", err
+ }
+ if immutable != "" && immutable != selected {
+ return "", errors.New("conflicting frozen digests for one image reference")
+ }
+ immutable = selected
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: target.ContainerID, ResourceType: "container", Status: schedulertypes.Running}); err != nil {
+ return "", err
+ }
+ }
+ if immutable == "" {
+ return "", errors.New("updater attempted to pull an image outside its frozen plan")
+ }
+ return immutable, nil
+}
+
+// confirmFrozenTargetInternal returns whether the desired effect is confirmed,
+// or whether the exact original container is unchanged and safe to resume.
+func (s *UpdaterService) confirmFrozenTargetInternal(ctx context.Context, target arcaneupdater.FrozenUpdateTarget) (bool, bool, error) {
+ dockerClient, err := s.DockerClient(ctx)
+ if err != nil {
+ return false, false, err
+ }
+ listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ if err != nil {
+ return false, false, err
+ }
+ immutable, imageErr := immutableImageInternal(target)
+ desiredID := ""
+ if imageErr == nil {
+ image, err := dockerClient.ImageInspect(ctx, immutable)
+ if err == nil {
+ desiredID = image.ID
+ }
+ }
+ for _, candidate := range listed.Items {
+ named := false
+ for _, name := range candidate.Names {
+ if strings.TrimPrefix(name, "/") == target.ContainerName {
+ named = true
+ break
+ }
+ }
+ if !named {
+ continue
+ }
+ result, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, candidate.ID, client.ContainerInspectOptions{})
+ if err != nil {
+ return false, false, err
+ }
+ inspected := result.Container
+ if inspected.Config == nil {
+ return false, false, nil
+ }
+ labels := inspected.Config.Labels
+ if labels["com.docker.compose.project"] != target.ComposeProject || labels["com.docker.compose.service"] != target.ComposeService || labels["com.docker.compose.container-number"] != target.ComposeNumber {
+ return false, false, nil
+ }
+ if desiredID != "" && inspected.Image == desiredID {
+ return true, false, nil
+ }
+ unchanged := inspected.ID == target.ContainerID && inspected.Image == target.BaselineImageID && inspected.RestartCount == target.BaselineRestartCount && inspected.State != nil && inspected.State.StartedAt == target.BaselineStartedAt
+ return false, unchanged, nil
+ }
+ return false, false, nil
+}
+
+// ReconcilePending resumes only frozen targets whose original container is unchanged.
+func (s *UpdaterService) ReconcilePending(ctx context.Context, run schedulertypes.Run) (schedulertypes.Outcome, error) {
+ ctx, release, err := s.acquireUpdateInternal(ctx)
+ if err != nil {
+ return schedulertypes.Outcome{Status: schedulertypes.Waiting}, err
+ }
+ defer release()
+ var plan frozenUpdatePlanInternal
+ found := false
+ for _, target := range run.Outcome.Targets {
+ if target.ID == "auto-update" && len(target.RecoveryData) > 0 {
+ if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention}, err
+ }
+ found = true
+ }
+ }
+ if !found {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil
+ }
+ remaining := frozenUpdatePlanInternal{}
+ unresolved := false
+ for _, target := range plan.Targets {
+ if frozenTargetSettledInternal(run, target.ContainerID) {
+ continue
+ }
+ confirmed, unchanged, err := s.confirmFrozenTargetInternal(ctx, target)
+ if err != nil {
+ return schedulertypes.Outcome{Status: schedulertypes.Waiting}, err
+ }
+ if confirmed {
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: target.ContainerID, ResourceType: "container", Status: schedulertypes.Succeeded, Message: "Frozen desired image confirmed after restart"}); err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ continue
+ }
+ if !unchanged {
+ unresolved = true
+ continue
+ }
+ remaining.Targets = append(remaining.Targets, target)
+ remaining.Records = appendFrozenRecordInternal(remaining.Records, plan.Records, target.ContainerID)
+ }
+ if len(remaining.Records) > 0 {
+ result, err := s.ApplyPending(context.WithValue(ctx, frozenPendingKeyInternal{}, &remaining), arcaneupdater.Options{})
+ if err != nil || result == nil || result.Failed > 0 {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Frozen update recovery requires review"}, err
+ }
+ }
+ status := schedulertypes.Succeeded
+ message := "Frozen update targets confirmed"
+ if unresolved {
+ status = schedulertypes.NeedsAttention
+ message = "Unconfirmed update effects require review"
+ }
+ return schedulertypes.Outcome{Status: status, Message: message}, nil
+}
+
+func frozenTargetSettledInternal(run schedulertypes.Run, id string) bool {
+ for _, target := range run.Outcome.Targets {
+ if target.ID == id && (target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped) {
+ return true
+ }
+ }
+ return false
+}
+
+func appendFrozenRecordInternal(remaining, records []updater.ImageUpdateRecord, id string) []updater.ImageUpdateRecord {
+ for _, record := range records {
+ if record.ContainerID == id {
+ return append(remaining, record)
+ }
+ }
+ return remaining
+}
diff --git a/backend/internal/updater/recovery_test.go b/backend/internal/updater/recovery_test.go
new file mode 100644
index 000000000..58b1b70a8
--- /dev/null
+++ b/backend/internal/updater/recovery_test.go
@@ -0,0 +1,161 @@
+package updater
+
+import (
+ "context"
+ "encoding/json/v2"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
+ "github.com/moby/moby/api/types/container"
+ image "github.com/moby/moby/api/types/image"
+ "github.com/stretchr/testify/require"
+)
+
+func TestFrozenTargetsPreserveAllSelectedContainers(t *testing.T) {
+ db := setupProjectTestDBInternal(t)
+ digest := "sha256:desired"
+ require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{ID: "shared-image", Repository: "registry.example.com/app", Tag: "latest", HasUpdate: true, LatestDigest: &digest}).Error)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if strings.HasSuffix(r.URL.Path, "/containers/json") {
+ require.NoError(t, json.MarshalWrite(w, []container.Summary{{ID: "a", Names: []string{"/a"}, Image: "registry.example.com/app:latest"}, {ID: "b", Names: []string{"/b"}, Image: "registry.example.com/app:latest"}}))
+ return
+ }
+ for _, id := range []string{"a", "b"} {
+ if strings.HasSuffix(r.URL.Path, "/containers/"+id+"/json") {
+ require.NoError(t, json.MarshalWrite(w, container.InspectResponse{ID: id, Name: "/" + id, Image: "sha256:old", Config: &container.Config{Image: "registry.example.com/app:latest"}, State: &container.State{StartedAt: "baseline"}}))
+ return
+ }
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+ svc, err := NewUpdaterService(db, nil, (&docker.DockerClientService{}).WithClient(newTestDockerClientInternal(t, server)), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ require.NoError(t, err)
+ var evidence []byte
+ ctx := jobcontext.WithExecution(t.Context(), schedulertypes.Run{ID: "run"}, func(target schedulertypes.TargetOutcome) error {
+ if target.ID == "auto-update" {
+ evidence = target.RecoveryData
+ }
+ return nil
+ })
+ frozenCtx, err := svc.freezePendingInternal(ctx)
+ require.NoError(t, err)
+ plan := frozenCtx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal)
+ require.Len(t, plan.Records, 2)
+ require.Equal(t, "a", plan.Records[0].ContainerID)
+ require.Equal(t, "b", plan.Records[1].ContainerID)
+ require.NotEmpty(t, evidence)
+ require.Equal(t, digest, plan.Targets[1].DesiredDigest)
+}
+
+func TestFrozenTargetConfirmsReplacementAndRejectsUnknownEffect(t *testing.T) {
+ for _, test := range []struct {
+ name, id, image string
+ confirmed, unchanged bool
+ }{
+ {"replacement", "new", "sha256:desired", true, false},
+ {"unchanged", "old", "sha256:baseline", false, true},
+ {"unknown replacement", "new", "sha256:other", false, false},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/containers/json"):
+ require.NoError(t, json.MarshalWrite(w, []container.Summary{{ID: test.id, Names: []string{"/app"}}}))
+ case strings.Contains(r.URL.Path, "/images/"):
+ require.NoError(t, json.MarshalWrite(w, image.InspectResponse{ID: "sha256:desired"}))
+ case strings.Contains(r.URL.Path, "/containers/"):
+ require.NoError(t, json.MarshalWrite(w, container.InspectResponse{ID: test.id, Name: "/app", Image: test.image, Config: &container.Config{}, State: &container.State{StartedAt: "baseline"}}))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ svc, err := NewUpdaterService(nil, nil, (&docker.DockerClientService{}).WithClient(newTestDockerClientInternal(t, server)), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ require.NoError(t, err)
+ confirmed, unchanged, err := svc.confirmFrozenTargetInternal(t.Context(), arcaneupdater.FrozenUpdateTarget{ContainerID: "old", ContainerName: "app", BaselineImageID: "sha256:baseline", BaselineStartedAt: "baseline", DesiredImageRef: "registry.example.com/app:latest", DesiredDigest: "sha256:desired"})
+ require.NoError(t, err)
+ require.Equal(t, test.confirmed, confirmed)
+ require.Equal(t, test.unchanged, unchanged)
+ })
+ }
+}
+
+func TestSingleUpdateRepairsAcceptedIntentBeforeDispatch(t *testing.T) {
+ svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ require.NoError(t, err)
+ runtime := francistest.New(t)
+ require.NoError(t, svc.RegisterActors(runtime))
+ francistest.Start(t, runtime)
+ command := arcaneupdater.SingleUpdateCommand{ContainerID: "missing", ActivityID: "accepted"}
+ require.NoError(t, runtime.Service().SetState(t.Context(), singleUpdateStateTypeInternal, command.ActivityID, arcaneupdater.SingleUpdateState{Command: command, Status: "queued"}, nil))
+ require.NoError(t, svc.Start(t.Context()))
+ t.Cleanup(func() {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, svc.Stop(ctx))
+ })
+ require.Eventually(t, func() bool {
+ var state arcaneupdater.SingleUpdateState
+ return runtime.Service().GetState(t.Context(), singleUpdateStateTypeInternal, command.ActivityID, &state) == nil && state.Status == "needs_attention"
+ }, 3*time.Second, 10*time.Millisecond)
+}
+
+func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {
+ db := setupProjectTestDBInternal(t)
+ require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}, &AutoUpdateRecord{}))
+ pool, err := db.DB.DB()
+ require.NoError(t, err)
+ pool.SetMaxOpenConns(1)
+ immutableRef := "registry.example.com/app@sha256:" + strings.Repeat("a", 64)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/containers/json"):
+ require.NoError(t, json.MarshalWrite(w, []container.Summary{{ID: "app", Names: []string{"/app"}, Image: immutableRef, State: "running"}}))
+ case strings.HasSuffix(r.URL.Path, "/containers/app/json"):
+ require.NoError(t, json.MarshalWrite(w, container.InspectResponse{ID: "app", Name: "/app", Image: "sha256:baseline", Config: &container.Config{Image: immutableRef}, State: &container.State{StartedAt: "baseline", Running: true}}))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ activityService := activity.NewActivityService(db, nil)
+ svc, err := NewUpdaterService(db, nil, (&docker.DockerClientService{}).WithClient(newTestDockerClientInternal(t, server)), nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
+ require.NoError(t, err)
+ runtime := francistest.New(t)
+ require.NoError(t, svc.RegisterActors(runtime))
+ francistest.Start(t, runtime)
+ require.NoError(t, svc.Start(t.Context()))
+ t.Cleanup(func() {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, svc.Stop(ctx))
+ })
+ accepted, err := svc.AcceptSingleContainerUpdate(t.Context(), "app")
+ require.NoError(t, err)
+ require.Eventually(t, func() bool {
+ var state arcaneupdater.SingleUpdateState
+ if runtime.Service().GetState(t.Context(), singleUpdateStateTypeInternal, accepted.ID, &state) != nil || state.Status != "completed" {
+ return false
+ }
+ detail, err := activityService.GetActivityDetail(t.Context(), "0", accepted.ID, 1)
+ return err == nil && detail.Activity.Status == activitytypes.StatusSuccess
+ }, 3*time.Second, 10*time.Millisecond)
+ detail, err := activityService.GetActivityDetail(t.Context(), "0", accepted.ID, 1)
+ require.NoError(t, err)
+ require.Equal(t, activitytypes.StatusSuccess, detail.Activity.Status)
+}
diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.go
index 434b87ff9..7ac68754a 100644
--- a/backend/internal/updater/service.go
+++ b/backend/internal/updater/service.go
@@ -14,8 +14,8 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -24,17 +24,20 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
projectpkg "github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
projectspkg "github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
+ "github.com/italypaleale/francis/actor"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
"github.com/samber/mo"
@@ -48,9 +51,19 @@ import (
// UpdaterService is Arcane's handler-facing service for the standalone updater engine.
type UpdaterService struct {
- deps updaterDependenciesInternal
- engine *updater.Service
- singleUpdates *actors.Executor
+ admission *runs.Admission
+ config *config.Config
+ coordinator *runs.Coordinator
+ roles *role.RoleService
+ deps updaterDependenciesInternal
+ engine *updater.Service
+ singleUpdates *actor.Service
+ singleMu sync.Mutex
+ singleContext context.Context
+ singleCancel context.CancelFunc
+ singleWorkers sync.WaitGroup
+ singleDone chan struct{}
+ singleStopping bool
// updateMu serializes per-container updates. docker compose's recreate
// pipeline is not concurrency-safe for sibling containers sharing a
// namespace. ponytail: global lock ceiling — all updates serialize; fine
@@ -93,8 +106,13 @@ func NewUpdaterService(
notifications *notification.NotificationService,
upgrade selfUpgradeServiceInternal,
activityService *activity.ActivityService,
+ cfg *config.Config,
+ coordinator *runs.Coordinator,
+ admission *runs.Admission,
+ roles *role.RoleService,
) (*UpdaterService, error) {
service := &UpdaterService{
+ config: cfg, coordinator: coordinator, admission: admission, roles: roles,
deps: updaterDependenciesInternal{
DB: db,
Docker: docker,
@@ -171,6 +189,12 @@ func (s *UpdaterService) registryDigestResolverInternal() updater.RegistryDigest
// to concrete containers and go through the engine's single-container path
// instead — same activity, events, and cleanup either way.
func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater.Options) (out *arcaneupdater.Result, err error) {
+ var release func()
+ ctx, release, err = s.acquireUpdateInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ defer release()
start := time.Now()
batchCompleted := false
activityID := s.startAutoUpdateActivityInternal(ctx, options.DryRun)
@@ -195,7 +219,8 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater
s.completeAutoUpdateActivityInternal(ctx, activityID, out, err)
}()
- if err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "auto-update", Status: schedulertypes.Running, ActivityID: activityID}); err != nil {
+ ctx, err = s.freezePendingInternal(ctx)
+ if err != nil {
return out, err
}
@@ -431,7 +456,7 @@ func (s *UpdaterService) containerIDsForImagesInternal(ctx context.Context, imag
// UpdateSingleContainer updates a single container by ID to the latest available image.
func (s *UpdaterService) UpdateSingleContainer(ctx context.Context, containerID string) (out *arcaneupdater.Result, err error) {
- activity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID)
+ activity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID, true)
if err != nil {
return nil, err
}
@@ -451,24 +476,29 @@ func (s *UpdaterService) AcceptSingleContainerUpdate(ctx context.Context, contai
if s.deps.Activity == nil || s.singleUpdates == nil {
return nil, errors.New("asynchronous container updates unavailable")
}
- activity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID)
+ activity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID, false)
if err != nil {
return nil, err
}
- _, err = s.singleUpdates.Submit(workCtx, "update container", func(actorCtx context.Context) (*arcaneupdater.Result, error) {
- runCtx := utils.ActivityRuntimeContext(workCtx, actorCtx)
- return s.runSingleContainerUpdateInternal(runCtx, containerID, activity.ID)
- }, func(result *arcaneupdater.Result, runErr error) {
- s.finishSingleContainerUpdateInternal(workCtx, activity.ID, result, runErr)
- })
- if err != nil {
+ command := singleUpdateCommandInternal(ctx, containerID, activity.ID)
+ state := arcaneupdater.SingleUpdateState{Command: command, Status: "queued"}
+ if err := s.singleUpdates.SetState(workCtx, singleUpdateStateTypeInternal, activity.ID, state, nil); err != nil {
s.finishSingleContainerUpdateInternal(workCtx, activity.ID, nil, err)
- return nil, errors.WrapIf(err, "submit container update")
+ return nil, errors.WrapIf(err, "persist container update")
+ }
+ if err := s.dispatchSingleInternal(workCtx, state); err != nil {
+ s.loggerInternal().WarnContext(workCtx, "container update dispatch deferred", "activityId", activity.ID, "error", err)
}
return activity, nil
}
func (s *UpdaterService) runSingleContainerUpdateInternal(ctx context.Context, containerID, activityID string) (out *arcaneupdater.Result, err error) {
+ var release func()
+ ctx, release, err = s.acquireUpdateInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ defer release()
start := time.Now()
out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}
ctx = contextWithActivityIDInternal(ctx, activityID)
@@ -615,6 +645,10 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres
if s == nil || s.deps.ImagePuller == nil {
return common.Classify(common.ErrUnavailable, errors.New("image service unavailable"))
}
+ pulledRef, err := s.prepareFrozenPullInternal(ctx, imageRef)
+ if err != nil {
+ return err
+ }
activityID := activityIDFromContextInternal(ctx)
writer := activitylib.NewWriter(ctx, s.deps.Activity, activityID, progress, "Pulling updated images")
defer activitylib.FlushWriter(writer)
@@ -631,14 +665,23 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres
if err != nil {
return errors.WrapIf(err, "resolve registry credentials")
}
- return s.deps.ImagePuller.PullImage(pullCtx, imageRef, writer, s.deps.SystemUser, resolved)
+ if err := s.deps.ImagePuller.PullImage(pullCtx, pulledRef, writer, s.deps.SystemUser, resolved); err != nil {
+ return err
+ }
+ return s.tagFrozenPullInternal(pullCtx, pulledRef, imageRef)
}
- return s.deps.ImagePuller.PullImage(pullCtx, imageRef, writer, s.deps.SystemUser, nil)
+ if err := s.deps.ImagePuller.PullImage(pullCtx, pulledRef, writer, s.deps.SystemUser, nil); err != nil {
+ return err
+ }
+ return s.tagFrozenPullInternal(pullCtx, pulledRef, imageRef)
}
// PendingImageUpdates returns pending image update records from Arcane's database.
func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.ImageUpdateRecord, error) {
+ if frozen, ok := ctx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal); ok {
+ return slices.Clone(frozen.Records), nil
+ }
if s == nil || s.deps.DB == nil {
return nil, common.Classify(common.ErrUnavailable, errors.New("database unavailable"))
}
@@ -712,6 +755,13 @@ func (s *UpdaterService) UpdateServices(ctx context.Context, projectID string, s
// TriggerSelfUpdate runs Arcane's CLI-backed self-update hook.
func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.SelfUpdateTarget) error {
+ if target.NewImageRef != "" {
+ pinned, err := s.prepareFrozenPullInternal(ctx, target.NewImageRef)
+ if err != nil {
+ return err
+ }
+ target.NewImageRef = pinned
+ }
if s == nil || s.deps.SelfUpgrade == nil {
instanceType := cmp.Or(strings.TrimSpace(target.InstanceType), "server")
return errors.Errorf("%s self-update requires CLI upgrade service", instanceType)
@@ -902,7 +952,7 @@ func (s *UpdaterService) startAutoUpdateActivityInternal(ctx context.Context, dr
return activity.ID
}
-func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.Context, containerID string) (*activitytypes.Activity, context.Context, error) {
+func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.Context, containerID string, tracked bool) (*activitytypes.Activity, context.Context, error) {
if s.deps.Activity == nil {
return nil, ctx, nil
}
@@ -924,7 +974,7 @@ func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.
user.DisplayName = &initiator.DisplayName
}
}
- activity, workCtx, err := s.deps.Activity.StartTrackedActivity(ctx, activitylib.StartRequest{
+ request := activitylib.StartRequest{
EnvironmentID: "0",
Type: activitytypes.TypeAutoUpdate,
Queue: true,
@@ -936,11 +986,19 @@ func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.
Step: "Updating container",
LatestMessage: "Container update started",
Metadata: database.JSON{"containerID": containerID},
- })
+ }
+ var item *activitytypes.Activity
+ workCtx := ctx
+ var err error
+ if tracked {
+ item, workCtx, err = s.deps.Activity.StartTrackedActivity(ctx, request)
+ } else {
+ item, err = s.deps.Activity.StartActivity(ctx, request)
+ }
if err != nil {
return nil, nil, errors.WrapIf(err, "start container update activity")
}
- return activity, workCtx, nil
+ return item, workCtx, nil
}
func (s *UpdaterService) finishSingleContainerUpdateInternal(ctx context.Context, activityID string, result *arcaneupdater.Result, runErr error) {
@@ -1454,3 +1512,15 @@ func addNormalizedImageUpdateRefInternal(ctx context.Context, out map[string]str
}
slog.Debug(logMessage, args...)
}
+
+func (s *UpdaterService) tagFrozenPullInternal(ctx context.Context, pulledRef, imageRef string) error {
+ if pulledRef == imageRef {
+ return nil
+ }
+ dockerClient, err := s.DockerClient(ctx)
+ if err != nil {
+ return err
+ }
+ _, err = dockerClient.ImageTag(ctx, client.ImageTagOptions{Source: pulledRef, Target: imageRef})
+ return err
+}
diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.go
index 9ce57735c..17e22245c 100644
--- a/backend/internal/updater/service_test.go
+++ b/backend/internal/updater/service_test.go
@@ -3,7 +3,7 @@ package updater
import (
"bytes"
"context"
- "encoding/json"
+ "encoding/json/v2"
"errors"
"fmt"
"io"
@@ -20,7 +20,6 @@ import (
composetypes "github.com/compose-spec/compose-go/v2/types"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -38,11 +37,13 @@ import (
projectspkg "github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
projecttypes "github.com/getarcaneapp/arcane/types/v2/project"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
+ "github.com/italypaleale/francis/actor"
"github.com/libtnb/sqlite"
dockerauthconfig "github.com/moby/moby/api/pkg/authconfig"
"github.com/moby/moby/api/types/container"
@@ -57,7 +58,6 @@ import (
"go.getarcane.app/updater/labels"
"go.getarcane.app/updater/refs"
updatertypes "go.getarcane.app/updater/types"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -156,14 +156,14 @@ func newUpdaterApplyPendingDockerServerInternal(
_, _ = io.WriteString(w, "OK")
case strings.HasSuffix(r.URL.Path, "/version"):
w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(map[string]string{
+ _ = json.MarshalWrite(w, map[string]string{
"ApiVersion": "1.41",
"MinAPIVersion": "1.24",
"Version": "24.0.0",
})
case strings.HasSuffix(r.URL.Path, "/images/json"):
w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode([]dockertypesimage.Summary{})
+ _ = json.MarshalWrite(w, []dockertypesimage.Summary{})
case strings.Contains(r.URL.Path, "/images/") && strings.HasSuffix(r.URL.Path, "/json"):
encodedRef := strings.TrimSuffix(r.URL.Path[strings.LastIndex(r.URL.Path, "/images/")+len("/images/"):], "/json")
imageRef, err := url.PathUnescape(encodedRef)
@@ -176,7 +176,7 @@ func newUpdaterApplyPendingDockerServerInternal(
return
}
w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(inspect)
+ _ = json.MarshalWrite(w, inspect)
case strings.HasSuffix(r.URL.Path, "/containers/json"):
response := containers
if filters := strings.TrimSpace(r.URL.Query().Get("filters")); filters != "" {
@@ -203,7 +203,7 @@ func newUpdaterApplyPendingDockerServerInternal(
}
}
w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(response)
+ _ = json.MarshalWrite(w, response)
case strings.Contains(r.URL.Path, "/containers/") && strings.HasSuffix(r.URL.Path, "/json"):
containerID := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path[strings.LastIndex(r.URL.Path, "/containers/"):], "/containers/"), "/json")
inspect, ok := inspectByID[containerID]
@@ -212,7 +212,7 @@ func newUpdaterApplyPendingDockerServerInternal(
return
}
w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(inspect)
+ _ = json.MarshalWrite(w, inspect)
default:
http.NotFound(w, r)
}
@@ -239,7 +239,7 @@ func (m *mockSystemUpgradeServiceInternal) TriggerUpgradeViaCLI(_ context.Contex
func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {
ctx := context.Background()
db := setupProjectTestDBInternal(t)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
var checkpoints []schedulertypes.TargetOutcome
@@ -315,7 +315,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
t.Run("server label triggers upgrade with system user", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeServiceInternal{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{
@@ -334,7 +334,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
t.Run("agent label triggers upgrade", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeServiceInternal{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane-agent", map[string]string{
@@ -347,7 +347,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
t.Run("non Arcane labels fail without triggering upgrade", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeServiceInternal{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "demo", map[string]string{
@@ -360,7 +360,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
})
t.Run("missing upgrade service reports required hook", func(t *testing.T) {
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{
@@ -373,7 +373,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
t.Run("upgrade errors are wrapped", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeServiceInternal{triggerError: errors.New("upgrade failed")}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{
@@ -386,7 +386,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {
}
func TestUpdaterService_StatusTrackingInternal(t *testing.T) {
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
stopContainer := svc.BeginContainerUpdate("container-1")
@@ -412,7 +412,7 @@ func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {
ctx := context.Background()
t.Run("active engine updates correlate before the stop event is recorded", func(t *testing.T) {
events := event.NewEventService(nil, nil, nil)
- svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
end := svc.engineInternal().BeginContainerUpdate("updating-container")
require.True(t, events.ShouldSuppressDaemonEvent("container", "updating-container", "", ""))
@@ -423,7 +423,7 @@ func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {
})
t.Run("missing docker service returns unavailable error", func(t *testing.T) {
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
cli, err := svc.DockerClient(ctx)
@@ -437,7 +437,7 @@ func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {
server := newImagePullServerWithObserverInternal(t, nil, nil)
wantClient := newTestDockerClientInternal(t, server)
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(wantClient)
- svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
gotClient, err := svc.DockerClient(ctx)
@@ -451,7 +451,7 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {
ctx := context.Background()
t.Run("missing image service returns unavailable error", func(t *testing.T) {
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.PullImage(ctx, "registry.example.com/app:1.2.3", nil)
@@ -465,7 +465,7 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {
server := newImagePullServerWithObserverInternal(t, nil, nil)
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
imageSvc := image.NewImageService(db, dockerSvc, nil, nil, nil, event.NewEventService(db, nil, nil))
- svc, svcErr := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, imageSvc, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
var progress bytes.Buffer
@@ -492,12 +492,12 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {
}
})
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
- registrySvc := registry.NewContainerRegistryService(db, nil, kv.NewKVService(db))
+ registrySvc := registry.NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)
imageSvc := image.NewImageService(db, dockerSvc, registrySvc, nil, nil, event.NewEventService(db, nil, nil))
envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)
- projectSvc := project.NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, nil)
+ projectSvc := project.NewProjectService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
projectSvc.RegistryCredentialsProvider = envSvc.GetEnabledRegistryCredentials
- svc, svcErr := NewUpdaterService(db, nil, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
var progress bytes.Buffer
@@ -541,7 +541,7 @@ func TestUpdaterService_PendingImageUpdatesAdapterInternal(t *testing.T) {
UpdateType: imageupdate.UpdateTypeDigest,
CheckTime: checkTime,
}).Error)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
records, err := svc.PendingImageUpdates(ctx)
@@ -568,7 +568,7 @@ func newEmptyContainerListDockerServiceInternal(t *testing.T) *docker.DockerClie
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/containers/json") {
- require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{}))
+ require.NoError(t, json.MarshalWrite(w, []container.Summary{}))
return
}
http.NotFound(w, r)
@@ -585,7 +585,7 @@ func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnre
require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, &notification.NotificationSettings{}))
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, notif, nil)
- svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil)
+ svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{ID: "sha256:pending", Repository: "test/repo", Tag: "latest", HasUpdate: true}).Error)
@@ -624,7 +624,7 @@ func TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal(t
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerServiceInternal(t), nil, notif, nil)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{
@@ -655,7 +655,7 @@ func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotifiedInternal(t
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerServiceInternal(t), nil, notif, nil)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{
@@ -679,7 +679,7 @@ func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {
ctx := context.Background()
db := setupProjectTestDBInternal(t)
require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}))
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.RecordUpdateRun(ctx, updater.ResourceResult{
@@ -721,7 +721,7 @@ func TestUpdaterService_RecordUpdateRunAppendsActivityMessageInternal(t *testing
db := setupProjectTestDBInternal(t)
require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}, &activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
require.NoError(t, svcErr)
for _, tt := range []struct {
@@ -780,31 +780,22 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivityInternal(t
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
activityService := activity.NewActivityService(db, nil)
- svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService)
+ svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
require.NoError(t, err)
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "updater-test", t.Name(), 3)
- require.NoError(t, err)
- svc.singleUpdates = executor
+ runtime := francistest.New(t)
+ svc.singleUpdates = runtime.Service()
release := make(chan struct{})
+ started := make(chan struct{})
var releaseOnce sync.Once
defer releaseOnce.Do(func() { close(release) })
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- started := make(chan struct{})
- _, err = executor.Submit(t.Context(), "hold executor", func(context.Context) (int, error) {
- close(started)
- <-release
- return 0, nil
- }, nil)
- require.NoError(t, err)
+ require.NoError(t, runtime.RegisterActor(singleUpdateTypeInternal, func(_ string, _ *actor.Service) actor.Actor {
+ return &blockingSingleUpdateActorInternal{singleUpdateActorInternal: singleUpdateActorInternal{service: svc}, started: started, release: release}
+ }))
+ francistest.Start(t, runtime)
+ go func() {
+ _, _ = runtime.Service().Invoke(t.Context(), singleUpdateTypeInternal, "single-container", "hold", nil)
+ }()
<-started
requestCtx, cancelRequest := context.WithCancel(context.Background())
@@ -847,26 +838,15 @@ func TestUpdaterService_AcceptSingleContainerUpdateFailureFinalizesInternal(t *t
db := setupProjectTestDBInternal(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
- svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService)
- require.NoError(t, err)
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "updater-test", t.Name(), 3)
- require.NoError(t, err)
- svc.singleUpdates = executor
- stopCtx, cancelStop := context.WithTimeout(context.Background(), time.Second)
- require.NoError(t, executor.Stop(stopCtx))
- cancelStop()
- t.Cleanup(func() {
- cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, lifecycle.Stop(cleanupCtx))
- })
+ runtime := francistest.New(t)
+ require.NoError(t, svc.RegisterActors(runtime))
+ // A service whose host has not started rejects durable dispatch.
accepted, err := svc.AcceptSingleContainerUpdate(context.Background(), "container-1")
require.Nil(t, accepted)
- require.ErrorContains(t, err, "submit container update")
+ require.ErrorContains(t, err, "persist container update")
var saved activity.Activity
require.NoError(t, db.First(&saved, "resource_id = ?", "container-1").Error)
require.Equal(t, activitytypes.StatusFailed, saved.Status)
@@ -895,7 +875,7 @@ func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing
db := setupProjectTestDBInternal(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
- svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService)
+ svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
require.NoError(t, err)
started, err := activityService.StartActivity(t.Context(), activitylib.StartRequest{EnvironmentID: "0", Type: activitytypes.TypeAutoUpdate, Metadata: database.JSON{"containerID": "old-id"}})
require.NoError(t, err)
@@ -1009,7 +989,7 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInte
},
}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
engine, engineErr := updater.New(updater.Config{
DockerClientProvider: dockerProvider,
@@ -1142,7 +1122,7 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI
}
mockUpgrade := &mockSystemUpgradeServiceInternal{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
engine, engineErr := updater.New(updater.Config{
DockerClientProvider: dockerProvider,
@@ -1170,6 +1150,7 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI
})
svc.engine = engine
+ ctx = context.WithValue(ctx, frozenPendingKeyInternal{}, &frozenUpdatePlanInternal{Targets: []arcaneupdater.FrozenUpdateTarget{{ContainerID: "arcane-container", DesiredImageRef: newRef, DesiredDigest: "sha256:desired"}}})
ctx = jobcontext.WithExecution(ctx, schedulertypes.Run{AttemptCount: 1}, nil)
result, err := svc.ApplyPending(ctx, arcaneupdater.Options{})
require.ErrorContains(t, err, "self-update was triggered")
@@ -1290,7 +1271,7 @@ func newImagePullServerWithObserverInternal(t *testing.T, inspectByRef map[strin
}
w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(map[string]string{"status": "Pulled", "id": fullRef})
+ _ = json.MarshalWrite(w, map[string]string{"status": "Pulled", "id": fullRef})
return
case strings.Contains(r.URL.Path, "/images/") && strings.HasSuffix(r.URL.Path, "/json"):
path := r.URL.Path
@@ -1311,7 +1292,7 @@ func newImagePullServerWithObserverInternal(t *testing.T, inspectByRef map[strin
}
w.Header().Set("Content-Type", "application/json")
- if !assert.NoError(t, json.NewEncoder(w).Encode(inspect)) {
+ if !assert.NoError(t, json.MarshalWrite(w, inspect)) {
return
}
return
@@ -1351,7 +1332,7 @@ func TestUpdaterService_ScopedPendingRecordsIsolationInternal(t *testing.T) {
}
server := newUpdaterApplyPendingDockerServerInternal(t, containers, nil, nil, nil)
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
- svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
records, err := svc.PendingImageUpdates(t.Context())
require.NoError(t, err)
@@ -1397,7 +1378,7 @@ func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {
}
server := newUpdaterApplyPendingDockerServerInternal(t, containers, nil, nil, map[string]dockertypesimage.InspectResponse{imageRef: {ID: "new-image"}})
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
- svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
converted := imageUpdateRecordToModuleInternal(record)
converted.ContainerID = "updated"
@@ -1423,7 +1404,7 @@ func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.Contains(t, r.URL.Path, "/images/", "project checks must not need containers")
w.Header().Set("Content-Type", "application/json")
- require.NoError(t, json.NewEncoder(w).Encode(dockertypesimage.InspectResponse{ID: "sha256:" + strings.Repeat("a", 64), RepoDigests: []string{"alpine@sha256:" + strings.Repeat("a", 64)}}))
+ require.NoError(t, json.MarshalWrite(w, dockertypesimage.InspectResponse{ID: "sha256:" + strings.Repeat("a", 64), RepoDigests: []string{"alpine@sha256:" + strings.Repeat("a", 64)}}))
}))
defer server.Close()
puller := &fakeImagePullerInternal{}
@@ -1455,3 +1436,19 @@ func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {
}
require.Empty(t, puller.pulled)
}
+
+type blockingSingleUpdateActorInternal struct {
+ singleUpdateActorInternal
+ started chan struct{}
+ release chan struct{}
+}
+
+func (a *blockingSingleUpdateActorInternal) Invoke(ctx context.Context, _ string, _ actor.Envelope) (any, error) {
+ close(a.started)
+ select {
+ case <-a.release:
+ return nil, nil
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+}
diff --git a/backend/internal/user/module.go b/backend/internal/user/module.go
index 7f11e098f..15921f17a 100644
--- a/backend/internal/user/module.go
+++ b/backend/internal/user/module.go
@@ -7,25 +7,21 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-type Dependencies struct {
- Service *UserService
- InvalidateUserTokenCache func(string)
- Settings *settings.SettingsService
-}
-
type Module struct {
- deps Dependencies
+ service *UserService
+ invalidateUserTokenCache func(string)
+ settings *settings.SettingsService
}
-func New(deps Dependencies) *Module {
- return &Module{deps: deps}
+func New(service *UserService, invalidateUserTokenCache func(string), settingsService *settings.SettingsService) *Module {
+ return &Module{service: service, invalidateUserTokenCache: invalidateUserTokenCache, settings: settingsService}
}
func (m *Module) Service() *UserService {
if m == nil {
return nil
}
- return m.deps.Service
+ return m.service
}
func (m *Module) RegisterRoutes(api huma.API) {
@@ -33,5 +29,5 @@ func (m *Module) RegisterRoutes(api huma.API) {
RegisterUsers(api, nil, nil, nil)
return
}
- RegisterUsers(api, m.deps.Service, m.deps.InvalidateUserTokenCache, m.deps.Settings)
+ RegisterUsers(api, m.service, m.invalidateUserTokenCache, m.settings)
}
diff --git a/backend/internal/user/service.go b/backend/internal/user/service.go
index f589f987a..1fb76f43f 100644
--- a/backend/internal/user/service.go
+++ b/backend/internal/user/service.go
@@ -62,21 +62,14 @@ const (
ErrInsufficientPrivilege = errors.Sentinel("insufficient privilege to modify this user")
)
-func NewUserService(db *database.DB) *UserService {
+func NewUserService(db *database.DB, roleService *role.RoleService) *UserService {
return &UserService{
db: db,
+ roleService: roleService,
argon2Params: DefaultArgon2Params(),
}
}
-// WithRoleService wires the role.RoleService dependency. Separated from the
-// constructor so the bootstrap can construct UserService first (role.RoleService
-// itself has no UserService dependency).
-func (s *UserService) WithRoleService(roleService *role.RoleService) *UserService {
- s.roleService = roleService
- return s
-}
-
func (s *UserService) HashPassword(password string) (string, error) {
salt := make([]byte, s.argon2Params.saltLength)
_, err := rand.Read(salt)
diff --git a/backend/internal/user/service_test.go b/backend/internal/user/service_test.go
index 51e0e6ee7..219295aa2 100644
--- a/backend/internal/user/service_test.go
+++ b/backend/internal/user/service_test.go
@@ -48,7 +48,7 @@ func setupUserAndRoleServices(t *testing.T) (*UserService, *role.RoleService) {
db := setupAuthServiceTestDBInternal(t)
role := role.NewRoleService(db)
require.NoError(t, role.EnsureBuiltInRoles(context.Background()))
- userRecord := NewUserService(db).WithRoleService(role)
+ userRecord := NewUserService(db, role)
return userRecord, role
}
@@ -441,7 +441,7 @@ func TestCreateDefaultAdminRecoversArcaneUserWhenNoGlobalAdminExists(t *testing.
}
func TestUserDisplayNameNormalization(t *testing.T) {
- svc := NewUserService(setupAuthServiceTestDBInternal(t))
+ svc := NewUserService(setupAuthServiceTestDBInternal(t), nil)
ctx := context.Background()
created, err := svc.CreateUser(ctx, &common.User{ID: "normalized", Username: " Jose\u0301 ", Email: new(" Jose\[email protected] "), DisplayName: new(" Jose\u0301 ")})
require.NoError(t, err)
diff --git a/backend/internal/variable/service_test.go b/backend/internal/variable/service_test.go
index 87ec675ec..1937e140d 100644
--- a/backend/internal/variable/service_test.go
+++ b/backend/internal/variable/service_test.go
@@ -13,10 +13,8 @@ import (
"strings"
"sync"
"testing"
- "time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
@@ -29,27 +27,16 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.getarcane.app/sys/crypto"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
func newSettingsServiceForVariableTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- writes, err := actors.NewExecutor(t.Context(), runtime, "variable-settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "variable-settings-effects-test", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, writes.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return settings.NewSettingsService(ctx, db, writes, effects)
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
func setupVariableServiceTest(t *testing.T) (*VariableService, *database.DB, string) {
diff --git a/backend/internal/version/service_test.go b/backend/internal/version/service_test.go
index 7d7094b19..c0721e0ca 100644
--- a/backend/internal/version/service_test.go
+++ b/backend/internal/version/service_test.go
@@ -308,7 +308,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *te
})}
dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- registrySvc := registry.NewContainerRegistryService(nil, nil, nil)
+ registrySvc := registry.NewContainerRegistryService(nil, nil, nil, nil)
svc := NewVersionService(httpClient, false, "2.8.0-next.65", "revision", registrySvc, dockerService, imageUpdateService, nil)
info := svc.GetAppVersionInfo(ctx)
@@ -352,7 +352,7 @@ func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDiges
dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- registrySvc := registry.NewContainerRegistryService(nil, nil, nil)
+ registrySvc := registry.NewContainerRegistryService(nil, nil, nil, nil)
svc := NewVersionService(nil, false, "2.8.0-next.65", "revision", registrySvc, dockerService, imageUpdateService, nil)
info := svc.GetAppVersionInfo(ctx)
diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.go
index ce89459e7..cab55f998 100644
--- a/backend/internal/volume/backup.go
+++ b/backend/internal/volume/backup.go
@@ -5,6 +5,7 @@ import (
"cmp"
"compress/gzip"
"context"
+ "encoding/json/v2"
"fmt"
"io"
"log/slog"
@@ -16,7 +17,6 @@ import (
"emperror.dev/errors"
cerrdefs "github.com/containerd/errdefs"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -28,6 +28,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"
s3utils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
@@ -668,13 +669,21 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str
if err != nil {
return nil, err
}
- defer lease.Release()
+ defer lease.Release(ctx)
err = s.executeBackupInternal(ctx, entry, user, plan)
err = s.completeBackupInternal(ctx, entry, err)
return entry, err
}
-func (s *VolumeService) prepareBackupInternal(ctx context.Context, volumeName string, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (*VolumeBackup, *actors.Lease[actors.AdmissionKey], error) {
+type volumeBackupRecoveryInternal struct {
+ BackupID string `json:"backupId"`
+ LocalEnabled bool `json:"localEnabled"`
+ S3Enabled bool `json:"s3Enabled"`
+ S3DestinationID string `json:"s3DestinationId"`
+ Policy *VolumeBackupPolicy `json:"policy,omitempty"`
+}
+
+func (s *VolumeService) prepareBackupInternal(ctx context.Context, volumeName string, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (*VolumeBackup, *runs.Lease, error) {
lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)
if err != nil {
return nil, nil, err
@@ -694,7 +703,15 @@ func (s *VolumeService) prepareBackupInternal(ctx context.Context, volumeName st
}
entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])
if err := s.db.WithContext(ctx).Create(entry).Error; err != nil {
- lease.Release()
+ lease.Release(ctx)
+ return nil, nil, err
+ }
+ checkpoint, err := json.Marshal(volumeBackupRecoveryInternal{BackupID: entry.ID, LocalEnabled: plan.localEnabled, S3Enabled: plan.s3Enabled, S3DestinationID: plan.s3DestinationID, Policy: plan.policy})
+ if err == nil {
+ err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "volume_backup", ID: volumeName, Status: schedulertypes.Running, RecoveryData: checkpoint})
+ }
+ if err != nil {
+ lease.Release(ctx)
return nil, nil, err
}
return entry, lease, nil
@@ -712,10 +729,12 @@ func (s *VolumeService) completeBackupInternal(ctx context.Context, entry *Volum
if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(entry).Error; saveErr != nil {
return errors.Combine(err, fmt.Errorf("failed to save volume backup result: %w", saveErr))
}
+ if err == nil {
+ err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "volume_backup", ID: entry.VolumeName, Status: schedulertypes.Succeeded})
+ }
return err
}
-//nolint:gocognit // backup execution keeps container recovery with snapshot work
func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *VolumeBackup, user common.User, plan backupPlanInternal) (err error) {
defer utils.RecoverToError(&err, "volume backup")
volumeName, trigger := entry.VolumeName, entry.Trigger
@@ -745,50 +764,14 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volume
return err
}
}
- // The live volume is read once. A local+S3 backup replicates the local
- // snapshot to S3 instead of scanning the source a second time.
- var localSnapshot backup.Snapshot
- if plan.localEnabled {
- repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
- if repoErr != nil {
- return repoErr
- }
- password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
- if passwordErr != nil {
- return passwordErr
- }
- localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, password, volumeName, backup.RootSnapshotInput(volumeSourceMountInternal(volumeName)))
- if err != nil {
- return fmt.Errorf("failed to create local Rustic snapshot: %w", err)
+ if plan.localEnabled && entry.LocalSnapshotID == "" {
+ if err := s.createLocalBackupSnapshotInternal(ctx, dockerClient, entry); err != nil {
+ return err
}
- entry.LocalSnapshotID = localSnapshot.ID
- entry.Size = localSnapshot.Size
}
- if plan.s3Enabled {
- remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)
- if repoErr != nil {
- return repoErr
- }
- password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remoteRepository)
- if passwordErr != nil {
- return passwordErr
- }
- var remoteSnapshot backup.Snapshot
- if plan.localEnabled {
- localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)
- if localErr != nil {
- return localErr
- }
- remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, password, volumeName)
- } else {
- remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, password, volumeName, backup.RootSnapshotInput(volumeSourceMountInternal(volumeName)))
- }
- if err != nil {
- return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)
- }
- entry.RemoteSnapshotID = remoteSnapshot.ID
- if entry.Size == 0 {
- entry.Size = remoteSnapshot.Size
+ if plan.s3Enabled && entry.RemoteSnapshotID == "" {
+ if err := s.createRemoteBackupSnapshotInternal(ctx, dockerClient, entry, plan); err != nil {
+ return err
}
}
if containersStopped {
@@ -813,6 +796,72 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volume
return ctx.Err()
}
+func (s *VolumeService) createLocalBackupSnapshotInternal(ctx context.Context, dockerClient *client.Client, entry *VolumeBackup) error {
+ repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
+ if repoErr != nil {
+ return repoErr
+ }
+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if passwordErr != nil {
+ return passwordErr
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: schedulertypes.Running}); err != nil {
+ return err
+ }
+ localSnapshot, err := s.engine.CreateSnapshot(ctx, dockerClient, repository, password, entry.VolumeName, backup.RootSnapshotInput(volumeSourceMountInternal(entry.VolumeName), backup.RunSnapshotTag(entry.ID)))
+ if err != nil {
+ return fmt.Errorf("failed to create local Rustic snapshot: %w", err)
+ }
+ entry.LocalSnapshotID = localSnapshot.ID
+ entry.Size = localSnapshot.Size
+ if err := s.db.WithContext(ctx).Save(entry).Error; err != nil {
+ return err
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: schedulertypes.Succeeded, Message: localSnapshot.ID}); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *VolumeService) createRemoteBackupSnapshotInternal(ctx context.Context, dockerClient *client.Client, entry *VolumeBackup, plan backupPlanInternal) error {
+ var err error
+ remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)
+ if repoErr != nil {
+ return repoErr
+ }
+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remoteRepository)
+ if passwordErr != nil {
+ return passwordErr
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":remote", Status: schedulertypes.Running}); err != nil {
+ return err
+ }
+ var remoteSnapshot backup.Snapshot
+ if plan.localEnabled {
+ localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ if localErr != nil {
+ return localErr
+ }
+ remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))
+ } else {
+ remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, password, entry.VolumeName, backup.RootSnapshotInput(volumeSourceMountInternal(entry.VolumeName), backup.RunSnapshotTag(entry.ID)))
+ }
+ if err != nil {
+ return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)
+ }
+ entry.RemoteSnapshotID = remoteSnapshot.ID
+ if entry.Size == 0 {
+ entry.Size = remoteSnapshot.Size
+ }
+ if err := s.db.WithContext(ctx).Save(entry).Error; err != nil {
+ return err
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":remote", Status: schedulertypes.Succeeded, Message: remoteSnapshot.ID}); err != nil {
+ return err
+ }
+ return nil
+}
+
// HasEnabledBackupPolicy reports whether a volume-level schedule takes precedence over centralized backups.
func (s *VolumeService) HasEnabledBackupPolicy(ctx context.Context, volumeName string) (bool, error) {
var count int64
@@ -835,7 +884,7 @@ func (s *VolumeService) UploadBackup(ctx context.Context, backupID, s3Destinatio
if !admitted {
return nil, ErrVolumeBackupAlreadyRunning
}
- defer lease.Release()
+ defer lease.Release(ctx)
// Reload under the lease: a delete may have raced the first read.
if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
return nil, err
@@ -899,7 +948,7 @@ func (s *VolumeService) DeleteBackup(ctx context.Context, backupID string, user
if !admitted {
return ErrVolumeBackupAlreadyRunning
}
- defer lease.Release()
+ defer lease.Release(ctx)
return s.deleteBackupInternal(ctx, backupID, user)
}
@@ -1955,8 +2004,8 @@ func (s *VolumeService) rescheduleVolumeBackupPolicyInternal(ctx context.Context
func(ctx context.Context) (schedulertypes.Outcome, error) {
return s.runScheduledBackupInternal(ctx, policyID)
},
- func(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
- return jobcontext.ConfirmedTarget(previous, policy.VolumeName), nil
+ func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ return s.ReconcileBackup(ctx, previous, policy.VolumeName)
},
)
}
diff --git a/backend/internal/volume/backup_run.go b/backend/internal/volume/backup_run.go
index 934289298..725905c03 100644
--- a/backend/internal/volume/backup_run.go
+++ b/backend/internal/volume/backup_run.go
@@ -2,13 +2,21 @@ package volume
import (
"context"
+ "encoding/json/v2"
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
+ "github.com/moby/moby/client"
)
// StartBackup persists a running backup and submits application-owned work.
@@ -25,27 +33,240 @@ func (s *VolumeService) StartBackup(ctx context.Context, environmentID, volumeNa
"volume", volumeName, volumeName, &user, "Creating backup", "Creating volume backup",
database.JSON{"action": "create_volume_backup", "backupId": entry.ID, "destination": entry.Destination, "policyId": request.PolicyID, "s3DestinationId": entry.S3DestinationID}, false)
if activityID == "" {
- defer lease.Release()
+ defer lease.Release(ctx)
return volumetypes.BackupEntry{}, s.completeBackupInternal(ctx, entry, errors.New("failed to start backup activity"))
}
entry.ActivityID = &activityID
accepted := entry.ToDTO()
- complete := func(runErr error) {
- defer lease.Release()
- runErr = s.completeBackupInternal(workCtx, entry, runErr)
- activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Volume backup created successfully", runErr)
- }
- if err := s.engine.SubmitRun(workCtx, entry.ID, func(workerCtx context.Context) error {
- return s.executeBackupInternal(workerCtx, entry, user, plan)
- }, complete); err != nil {
- complete(err)
+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ payload, err := json.Marshal(manualVolumeBackupInternal{Checkpoint: volumeBackupRecoveryInternal{BackupID: entry.ID, LocalEnabled: plan.localEnabled, S3Enabled: plan.s3Enabled, S3DestinationID: plan.s3DestinationID, Policy: plan.policy}, VolumeName: volumeName, UserID: user.ID, ActivityID: activityID})
+ if err == nil {
+ err = s.engine.SubmitDurableRun(workCtx, backuptypes.DurableRunCommand{Kind: "volume", RunID: entry.ID, ActivityID: activityID, Payload: payload, UserID: user.ID, EnvironmentID: environmentID, Permission: authz.PermVolumesBackup, RequestedWithKey: keyID}, lease)
+ }
+ if err != nil {
+ lease.Release(ctx)
+ err = s.completeBackupInternal(workCtx, entry, err)
+ activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Volume backup created successfully", err)
return volumetypes.BackupEntry{}, err
}
+
return accepted, nil
}
// ReconcileInterruptedBackups runs before this process can accept backup work.
-func (s *VolumeService) ReconcileInterruptedBackups(ctx context.Context) error {
- return s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("status = ?", VolumeBackupStatusRunning).
- Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error
+func (s *VolumeService) ReconcileInterruptedBackups(ctx context.Context, protectedIDs ...string) error {
+ query := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("status = ?", VolumeBackupStatusRunning)
+ if len(protectedIDs) > 0 {
+ query = query.Where("id NOT IN ?", protectedIDs)
+ }
+ return query.Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error
+}
+
+func (s *VolumeService) ReconcileBackup(ctx context.Context, previous schedulertypes.Run, volumeName string) (schedulertypes.Outcome, error) {
+ outcome := jobcontext.ConfirmedTarget(previous, volumeName)
+ if outcome.Status == schedulertypes.Succeeded {
+ return outcome, nil
+ }
+ for _, target := range previous.Outcome.Targets {
+ if target.ID != volumeName || len(target.RecoveryData) == 0 {
+ continue
+ }
+ var checkpoint volumeBackupRecoveryInternal
+ if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {
+ return outcome, err
+ }
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ? AND volume_name = ?", checkpoint.BackupID, volumeName).First(&entry).Error; err != nil {
+ return outcome, err
+ }
+ if entry.Status != VolumeBackupStatusSucceeded {
+ if checkpoint.Policy != nil && checkpoint.Policy.StopContainers {
+ return outcome, nil
+ }
+ lease, admitted, err := s.engine.AcquireDurableRun(ctx, previous.ID, backup.VolumeAdmissionScope, volumeName)
+ if err != nil {
+ return outcome, err
+ }
+ if !admitted {
+ return outcome, nil
+ }
+ defer lease.Release(ctx)
+ if err := s.resumeBackupInternal(ctx, previous, &entry, checkpoint); err != nil {
+ return outcome, err
+ }
+ }
+ target.Status = schedulertypes.Succeeded
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return outcome, err
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: []schedulertypes.TargetOutcome{target}}, nil
+ }
+ return outcome, nil
+}
+
+func backupDestinationAttemptedInternal(previous schedulertypes.Run, backupID, destination string) bool {
+ for _, evidence := range previous.Outcome.Targets {
+ if evidence.ID == backupID+":"+destination {
+ return true
+ }
+ }
+ return false
+}
+
+func (s *VolumeService) recoverSnapshotInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, entry *VolumeBackup, snapshotID string) (backup.Snapshot, bool, error) {
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if err != nil {
+ return backup.Snapshot{}, false, err
+ }
+ return s.engine.FindRunSnapshot(ctx, dockerClient, repository, password, entry.ID, snapshotID)
+}
+
+func (s *VolumeService) observeBackupDestinationsInternal(ctx context.Context, dockerClient *client.Client, previous schedulertypes.Run, entry *VolumeBackup, checkpoint volumeBackupRecoveryInternal) (backup.Repository, backup.Repository, error) {
+ var local, remote backup.Repository
+ var err error
+ if checkpoint.LocalEnabled {
+ local, err = s.localRusticRepositoryInternal(ctx, dockerClient, false)
+ if err != nil {
+ return local, remote, err
+ }
+ if entry.LocalSnapshotID != "" || backupDestinationAttemptedInternal(previous, entry.ID, "local") {
+ snapshot, found, err := s.recoverSnapshotInternal(ctx, dockerClient, local, entry, entry.LocalSnapshotID)
+ if err != nil {
+ return local, remote, err
+ }
+ if found {
+ entry.LocalSnapshotID, entry.Size = snapshot.ID, snapshot.Size
+ }
+ }
+ }
+ if checkpoint.S3Enabled {
+ remote, err = s.remoteRusticRepositoryInternal(ctx, checkpoint.S3DestinationID)
+ if err != nil {
+ return local, remote, err
+ }
+ if entry.RemoteSnapshotID != "" || backupDestinationAttemptedInternal(previous, entry.ID, "remote") {
+ snapshot, found, err := s.recoverSnapshotInternal(ctx, dockerClient, remote, entry, entry.RemoteSnapshotID)
+ if err != nil {
+ return local, remote, err
+ }
+ if found {
+ entry.RemoteSnapshotID = snapshot.ID
+ if entry.Size == 0 {
+ entry.Size = snapshot.Size
+ }
+ }
+ }
+ }
+ return local, remote, nil
+}
+
+func (s *VolumeService) resumeBackupInternal(ctx context.Context, previous schedulertypes.Run, entry *VolumeBackup, checkpoint volumeBackupRecoveryInternal) error {
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ local, remote, err := s.observeBackupDestinationsInternal(ctx, dockerClient, previous, entry, checkpoint)
+ if err != nil {
+ return err
+ }
+ if checkpoint.LocalEnabled && entry.LocalSnapshotID == "" && entry.RemoteSnapshotID != "" {
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, remote)
+ if err != nil {
+ return err
+ }
+ snapshot, err := s.engine.Replicate(ctx, dockerClient, remote, entry.RemoteSnapshotID, local, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))
+ if err != nil {
+ return err
+ }
+ entry.LocalSnapshotID = snapshot.ID
+ }
+ if err := s.db.WithContext(ctx).Save(entry).Error; err != nil {
+ return err
+ }
+ plan := backupPlanInternal{localEnabled: checkpoint.LocalEnabled, s3Enabled: checkpoint.S3Enabled, s3DestinationID: checkpoint.S3DestinationID, policy: checkpoint.Policy, destination: entry.Destination}
+ err = s.executeBackupInternal(ctx, entry, common.SystemUser, plan)
+ return s.completeBackupInternal(ctx, entry, err)
+}
+
+type manualVolumeBackupInternal struct {
+ Checkpoint volumeBackupRecoveryInternal `json:"checkpoint"`
+ VolumeName string `json:"volumeName"`
+ UserID string `json:"userId"`
+ ActivityID string `json:"activityId"`
+}
+
+func (s *VolumeService) executeDurableBackupInternal(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {
+ var command manualVolumeBackupInternal
+ if err = json.Unmarshal(payload, &command); err != nil {
+ return err
+ }
+ defer func() {
+ if ctx.Err() == nil {
+ if err != nil {
+ saveErr := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("id = ?", command.Checkpoint.BackupID).Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": err.Error()}).Error
+ err = errors.Combine(err, saveErr)
+ }
+ activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Volume backup created successfully", err)
+ }
+ }()
+ var entry VolumeBackup
+ if err = s.db.WithContext(ctx).Where("id = ?", command.Checkpoint.BackupID).First(&entry).Error; err != nil {
+ return err
+ }
+
+ if entry.Status == VolumeBackupStatusSucceeded {
+ return nil
+ }
+ if interrupted {
+ previous, _ := jobcontext.Run(ctx)
+ outcome, recoveryErr := s.ReconcileBackup(ctx, previous, command.VolumeName)
+ if recoveryErr != nil {
+ return recoveryErr
+ }
+ if outcome.Status != schedulertypes.Succeeded {
+ return errors.New(outcome.Message)
+ }
+ return nil
+ }
+ lease, admitted, err := s.engine.AcquireDurableRun(ctx, runID, backup.VolumeAdmissionScope, command.VolumeName)
+ if err != nil {
+ return err
+ }
+ if !admitted {
+ return ErrVolumeBackupAlreadyRunning
+ }
+ defer lease.Release(ctx)
+ user := common.SystemUser
+ if command.UserID != "" && command.UserID != "agent" {
+ if err = s.db.WithContext(ctx).Where("id = ?", command.UserID).First(&user).Error; err != nil {
+ return err
+ }
+ }
+ checkpoint, err := json.Marshal(command.Checkpoint)
+ if err != nil {
+ return err
+ }
+ if err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "volume_backup", ID: command.VolumeName, Status: schedulertypes.Running, RecoveryData: checkpoint, ActivityID: command.ActivityID}); err != nil {
+ return err
+ }
+ plan := backupPlanInternal{localEnabled: command.Checkpoint.LocalEnabled, s3Enabled: command.Checkpoint.S3Enabled, s3DestinationID: command.Checkpoint.S3DestinationID, policy: command.Checkpoint.Policy, destination: entry.Destination}
+ err = s.executeBackupInternal(ctx, &entry, user, plan)
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ return s.completeBackupInternal(ctx, &entry, err)
+}
+
+func (s *VolumeService) failDurableBackupInternal(ctx context.Context, _ string, payload []byte, runErr error) error {
+ var command manualVolumeBackupInternal
+ if err := json.Unmarshal(payload, &command); err != nil {
+ return err
+ }
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", command.Checkpoint.BackupID).First(&entry).Error; err != nil {
+ return err
+ }
+ err := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("id = ?", entry.ID).Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": runErr.Error()}).Error
+ activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Volume backup created successfully", runErr)
+ return err
}
diff --git a/backend/internal/volume/module.go b/backend/internal/volume/module.go
index 14336e921..d627ed47c 100644
--- a/backend/internal/volume/module.go
+++ b/backend/internal/volume/module.go
@@ -5,49 +5,24 @@ package volume
import (
"github.com/danielgtaylor/huma/v2"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
- "github.com/getarcaneapp/arcane/backend/v2/internal/config"
- "github.com/getarcaneapp/arcane/backend/v2/internal/container"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
- "github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/image"
- s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/upload"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
-// Dependencies are the collaborators the volume domain needs.
-type Dependencies struct {
- DB *database.DB
- Docker *docker.DockerClientService
- Event *event.EventService
- Settings *settings.SettingsService
- Image *image.ImageService
- Activity *activity.ActivityService
- Environment *environment.EnvironmentService
- Container *container.ContainerService
- Engine *backup.Engine
- S3 *s3domain.S3DestinationService
- Config *config.Config
- Upload *upload.UploadService
- RecoveryKeys *backup.RecoveryKeyStore
-}
-
// Module wires the volume domain and mounts its routes.
type Module struct {
- service *VolumeService
- deps Dependencies
+ service *VolumeService
+ dockerService *docker.DockerClientService
+ activityService *activity.ActivityService
+ environmentService *environment.EnvironmentService
+ uploadService *upload.UploadService
}
-// New builds the volume domain from its dependencies.
-func New(deps Dependencies) *Module {
- return &Module{
- service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Activity, deps.Settings, deps.Container, deps.Image, deps.Engine, deps.S3, deps.Config, deps.RecoveryKeys),
- deps: deps,
- }
+// New wires volume routes around an existing service.
+func New(service *VolumeService, dockerService *docker.DockerClientService, activityService *activity.ActivityService, environmentService *environment.EnvironmentService, uploadService *upload.UploadService) *Module {
+ return &Module{service: service, dockerService: dockerService, activityService: activityService, environmentService: environmentService, uploadService: uploadService}
}
// Service exposes the volume service to collaborators that use it directly,
@@ -66,5 +41,5 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
RegisterVolumes(api, nil, nil, nil, nil, nil, appCtx)
return
}
- RegisterVolumes(api, m.deps.Docker, m.service, m.deps.Activity, m.deps.Environment, m.deps.Upload, appCtx)
+ RegisterVolumes(api, m.dockerService, m.service, m.activityService, m.environmentService, m.uploadService, appCtx)
}
diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.go
index 7963c879a..ec8056944 100644
--- a/backend/internal/volume/service.go
+++ b/backend/internal/volume/service.go
@@ -8,7 +8,6 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -23,6 +22,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
workspacepkg "github.com/getarcaneapp/arcane/backend/v2/pkg/workspace"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -61,7 +61,7 @@ type VolumeService struct {
// SetScheduler injects the dynamic scheduler and admission gate for per-policy
// backup jobs. Agent mode passes them too: agents run their own volume backups.
-func (s *VolumeService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {
+func (s *VolumeService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *runs.Admission) error {
return s.jobs.SetScheduler(ctx, scheduler, admissionGate)
}
@@ -91,7 +91,7 @@ func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService
if strings.TrimSpace(backupVolumeName) == "" {
backupVolumeName = "arcane-backups"
}
- return &VolumeService{
+ service := &VolumeService{
db: db,
dockerService: dockerService,
eventService: eventService,
@@ -110,6 +110,10 @@ func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService
helperByVolume: make(map[string]*volumeHelper),
jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),
}
+ if engine != nil {
+ engine.RegisterRunKind("volume", service.executeDurableBackupInternal, service.failDurableBackupInternal)
+ }
+ return service
}
func (s *VolumeService) GetVolumeByName(ctx context.Context, name string) (*volumetypes.Volume, error) {
diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.go
index 9a2beac2e..6d815a26c 100644
--- a/backend/internal/volume/service_test.go
+++ b/backend/internal/volume/service_test.go
@@ -11,7 +11,6 @@ import (
"time"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -24,6 +23,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
@@ -34,28 +35,26 @@ import (
"github.com/moby/moby/client"
"github.com/stretchr/testify/require"
"go.getarcane.app/sys/crypto"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
func newVolumeBackupEngineForTestInternal(t testing.TB) *backup.Engine {
t.Helper()
- fxLifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "volume-backup-test-admission", t.Name())
- require.NoError(t, err)
- engine := backup.NewEngine(t.Context(), runtime, gate, nil)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, engine.Stop(stopCtx))
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, fxLifecycle.Stop(stopCtx))
- })
+ gate := newVolumeAdmissionForTestInternal(t)
+ engine := backup.NewEngine(t.Context(), gate, nil)
+ t.Cleanup(func() { require.NoError(t, engine.Stop(context.WithoutCancel(t.Context()))) })
return engine
}
+func newVolumeAdmissionForTestInternal(t testing.TB) *runs.Admission {
+ t.Helper()
+ runtime := francistest.New(t)
+ gate := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, gate.Register(runtime))
+ francistest.Start(t, runtime)
+ return gate
+}
+
func newVolumeServiceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
dockerClient, err := client.New(
@@ -648,17 +647,7 @@ func TestVolumeBackupPolicy_UpdateRegistersIndependentJobsAndSettings(t *testing
db := &database.DB{DB: gormDB}
scheduler := &volumeBackupPolicySchedulerInternal{jobs: make(map[string]schedulertypes.Job)}
service := &VolumeService{db: db, jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope)}
- fxLifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
- require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "volume-policy-test-admission", t.Name())
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, fxLifecycle.Stop(stopCtx))
- })
+ gate := newVolumeAdmissionForTestInternal(t)
require.NoError(t, service.SetScheduler(context.Background(), scheduler, gate))
collection, err := service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{
@@ -772,7 +761,7 @@ func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {
lease, admitted, err := engine.TryAcquireRun(context.Background(), backup.VolumeAdmissionScope, policy.VolumeName)
require.NoError(t, err)
require.True(t, admitted)
- defer lease.Release()
+ defer lease.Release(t.Context())
service.runScheduledBackupInternal(context.Background(), policy.ID)
@@ -805,7 +794,7 @@ func TestVolumeBackupPolicy_UpdateUsesSelectedS3Destination(t *testing.T) {
require.NoError(t, gormDB.Create(destination).Error)
service := &VolumeService{
db: db,
- s3Destinations: s3domain.NewS3DestinationService(db),
+ s3Destinations: s3domain.NewS3DestinationService(db, nil),
jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),
}
collection, err := service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{{
@@ -852,7 +841,7 @@ func TestVolumeBackup_ListResolvesDestinationName(t *testing.T) {
S3DestinationID: destination.ID,
}).Error)
- service := &VolumeService{db: db, s3Destinations: s3domain.NewS3DestinationService(db)}
+ service := &VolumeService{db: db, s3Destinations: s3domain.NewS3DestinationService(db, nil)}
backups, _, err := service.ListBackupsPaginated(context.Background(), "app-data", pagination.QueryParams{})
require.NoError(t, err)
require.Len(t, backups, 1)
@@ -1029,7 +1018,7 @@ func TestBackupPolicyDestinationLookupFailureInternal(t *testing.T) {
require.NoError(t, db.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}, &s3domain.S3Destination{}))
policy := &VolumeBackupPolicy{VolumeName: "app-data", Schedule: "0 0 2 * * *", S3DestinationID: "missing"}
require.NoError(t, db.Create(policy).Error)
- service := &VolumeService{db: &database.DB{DB: db}, s3Destinations: s3domain.NewS3DestinationService(&database.DB{DB: db})}
+ service := &VolumeService{db: &database.DB{DB: db}, s3Destinations: s3domain.NewS3DestinationService(&database.DB{DB: db}, nil)}
for _, drop := range []bool{false, true} {
if drop {
require.NoError(t, db.Migrator().DropTable(&s3domain.S3Destination{}))
diff --git a/backend/internal/vulnerability/vulnerability_feature_test.go b/backend/internal/vulnerability/vulnerability_feature_test.go
index 22270dc34..08915579c 100644
--- a/backend/internal/vulnerability/vulnerability_feature_test.go
+++ b/backend/internal/vulnerability/vulnerability_feature_test.go
@@ -8,7 +8,6 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -19,7 +18,6 @@ import (
"github.com/libtnb/sqlite"
"github.com/moby/moby/client"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"gorm.io/gorm"
)
@@ -29,21 +27,10 @@ func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database
require.NoError(t, err)
require.NoError(t, gdb.AutoMigrate(&settings.SettingVariable{}, &VulnerabilityScanRecord{}, &VulnerabilityScanItem{}, &VulnerabilityReportRecord{}, &VulnerabilityIgnore{}))
db := &database.DB{DB: gdb}
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- writes, err := actors.NewExecutor(t.Context(), runtime, "vuln-feature-writes", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "vuln-feature-effects", t.Name(), 3)
- require.NoError(t, err)
- t.Cleanup(func() {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, writes.Stop(ctx))
- require.NoError(t, effects.Stop(ctx))
- require.NoError(t, lifecycle.Stop(ctx))
- })
- svc, err := settings.NewSettingsService(t.Context(), db, writes, effects)
+ svc, err := settings.NewSettingsService(t.Context(), db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
require.NoError(t, err)
return svc, db
}
diff --git a/backend/internal/webhook/module.go b/backend/internal/webhook/module.go
index 0bcc6e58c..14c7bde36 100644
--- a/backend/internal/webhook/module.go
+++ b/backend/internal/webhook/module.go
@@ -4,36 +4,16 @@ package webhook
import (
"github.com/danielgtaylor/huma/v2"
- "github.com/getarcaneapp/arcane/backend/v2/internal/container"
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
- "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
- "github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
- "github.com/getarcaneapp/arcane/backend/v2/internal/project"
- "github.com/getarcaneapp/arcane/backend/v2/internal/updater"
)
-// Dependencies are the collaborators the webhook domain needs.
-type Dependencies struct {
- DB *database.DB
- Container *container.ContainerService
- Updater *updater.UpdaterService
- Project *project.ProjectService
- GitOpsSync *gitops.GitOpsSyncService
- Event *event.EventService
- Environment *environment.EnvironmentService
-}
-
// Module wires the webhook domain and mounts its routes.
type Module struct {
service *WebhookService
}
-// New builds the webhook domain from its dependencies.
-func New(deps Dependencies) *Module {
- return &Module{
- service: NewWebhookService(deps.DB, deps.Container, deps.Updater, deps.Project, deps.GitOpsSync, deps.Event, deps.Environment),
- }
+// New assembles webhook routes around the provided service.
+func New(service *WebhookService) *Module {
+ return &Module{service: service}
}
// Service exposes the webhook service to collaborators that trigger webhooks
diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.go
index 915fc3932..abd059ea4 100644
--- a/backend/pkg/fswatch/watcher.go
+++ b/backend/pkg/fswatch/watcher.go
@@ -20,6 +20,7 @@ import (
)
type Watcher struct {
+ callbacks sync.WaitGroup
watcher *fsnotify.Watcher
watchedPath string
maxDepth int
@@ -111,7 +112,15 @@ func (fw *Watcher) Start(ctx context.Context) error {
return nil
}
+// Stop closes the watcher and waits for accepted callbacks.
func (fw *Watcher) Stop() error {
+ err := fw.StopWatching()
+ fw.callbacks.Wait()
+ return err
+}
+
+// StopWatching closes the watch loop before its owner joins callbacks.
+func (fw *Watcher) StopWatching() error {
fw.stopOnce.Do(func() {
fw.mu.Lock()
fw.stopped = true
@@ -213,12 +222,12 @@ func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bo
*lastGoroutineLog = time.Now()
}
if fw.onChange != nil {
- go fw.onChange(ctx)
+ fw.callbacks.Go(func() { fw.onChange(ctx) })
}
if fw.onChangePaths != nil {
paths := fw.drainPendingPathsInternal()
if len(paths) > 0 {
- go fw.onChangePaths(ctx, paths)
+ fw.callbacks.Go(func() { fw.onChangePaths(ctx, paths) })
}
}
return false
@@ -478,10 +487,10 @@ func (fw *Watcher) reconnectInternal(ctx context.Context) bool {
}
// Changes while disconnected need a full reconciliation.
if fw.onChange != nil {
- fw.onChange(ctx)
+ fw.callbacks.Go(func() { fw.onChange(ctx) })
}
if fw.onChangePaths != nil {
- fw.onChangePaths(ctx, []string{fw.watchedPath})
+ fw.callbacks.Go(func() { fw.onChangePaths(ctx, []string{fw.watchedPath}) })
}
return true
}
diff --git a/backend/pkg/libarcane/edge/client.go b/backend/pkg/libarcane/edge/client.go
index 4cf6ff429..2fa6e6bdf 100644
--- a/backend/pkg/libarcane/edge/client.go
+++ b/backend/pkg/libarcane/edge/client.go
@@ -16,9 +16,9 @@ import (
"emperror.dev/errors"
"github.com/cenkalti/backoff/v5"
"github.com/coder/websocket"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
wshub "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/ws"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/samber/mo"
kit "go.getarcane.app/kit/pkg"
@@ -1445,8 +1445,8 @@ func (r *streamingResponseRecorder) writeHeaderLocked(statusCode int) error {
return nil
}
-// StartTunnelClient starts the tunnel client on the shared actor runtime.
-func StartTunnelClient(ctx context.Context, runtime *actors.Runtime, cfg *Config, handler http.Handler) (func(context.Context) error, error) {
+// StartTunnelClient starts the tunnel client with an owned background worker.
+func StartTunnelClient(ctx context.Context, cfg *Config, handler http.Handler) (func(context.Context) error, error) {
if !cfg.EdgeAgent {
return nil, errors.New("edge tunnel disabled")
}
@@ -1477,12 +1477,8 @@ func StartTunnelClient(ctx context.Context, runtime *actors.Runtime, cfg *Config
}
client := NewTunnelClient(cfg, handler)
- runner, err := actors.NewRunner(ctx, runtime, "edge", "agent-client", "Edge tunnel client", 3, func(runCtx context.Context) error {
+ return concurrency.StartSupervised(ctx, "Edge tunnel client", func(runCtx context.Context) error {
client.StartWithErrorChan(runCtx, nil)
return nil
})
- if err != nil {
- return nil, err
- }
- return runner.Stop, nil
}
diff --git a/backend/pkg/libarcane/edge/client_test.go b/backend/pkg/libarcane/edge/client_test.go
index 853d5aa23..09585d0ef 100644
--- a/backend/pkg/libarcane/edge/client_test.go
+++ b/backend/pkg/libarcane/edge/client_test.go
@@ -996,13 +996,13 @@ func TestStartTunnelClient_GRPCValidation(t *testing.T) {
ctx := t.Context()
t.Run("edge mode required", func(t *testing.T) {
- _, err := StartTunnelClient(ctx, nil, &Config{}, http.NotFoundHandler())
+ _, err := StartTunnelClient(ctx, &Config{}, http.NotFoundHandler())
require.Error(t, err)
assert.Contains(t, err.Error(), "edge tunnel disabled")
})
t.Run("manager url required for grpc transport", func(t *testing.T) {
- _, err := StartTunnelClient(ctx, nil, &Config{
+ _, err := StartTunnelClient(ctx, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
AgentToken: "token",
@@ -1012,7 +1012,7 @@ func TestStartTunnelClient_GRPCValidation(t *testing.T) {
})
t.Run("agent token required", func(t *testing.T) {
- _, err := StartTunnelClient(ctx, nil, &Config{
+ _, err := StartTunnelClient(ctx, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
ManagerApiUrl: "https://manager.example.com/arcane/api",
@@ -1022,7 +1022,7 @@ func TestStartTunnelClient_GRPCValidation(t *testing.T) {
})
t.Run("mtls requires https manager url", func(t *testing.T) {
- _, err := StartTunnelClient(ctx, nil, &Config{
+ _, err := StartTunnelClient(ctx, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
ManagerApiUrl: "http://manager.example.com/api",
@@ -1034,7 +1034,7 @@ func TestStartTunnelClient_GRPCValidation(t *testing.T) {
})
t.Run("required mtls auto-enrollment failure surfaces", func(t *testing.T) {
- _, err := StartTunnelClient(ctx, nil, &Config{
+ _, err := StartTunnelClient(ctx, &Config{
EdgeAgent: true,
EdgeTransport: EdgeTransportGRPC,
ManagerApiUrl: "https://127.0.0.1:1/api",
diff --git a/backend/pkg/libarcane/edge/registry.go b/backend/pkg/libarcane/edge/registry.go
index ee97f2b57..dc72ed97a 100644
--- a/backend/pkg/libarcane/edge/registry.go
+++ b/backend/pkg/libarcane/edge/registry.go
@@ -7,7 +7,7 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"github.com/samber/mo"
)
@@ -97,19 +97,10 @@ func (t *AgentTunnel) CloseWithReason(reason string) error {
// NewTunnelRegistry creates a new tunnel registry
func NewTunnelRegistry() *TunnelRegistry {
return &TunnelRegistry{
- tunnels: actors.NewStateMap[string, *AgentTunnel](),
+ tunnels: concurrency.NewStateMap[string, *AgentTunnel](),
}
}
-// NewActorTunnelRegistry creates a registry owned by the shared actor runtime.
-func NewActorTunnelRegistry(ctx context.Context, runtime *actors.Runtime) (*TunnelRegistry, error) {
- tunnels, err := actors.NewActorStateMap[string, *AgentTunnel](ctx, runtime, "edge", "tunnel-registry", 3)
- if err != nil {
- return nil, err
- }
- return &TunnelRegistry{tunnels: tunnels}, nil
-}
-
// Get retrieves a tunnel by environment ID
func (r *TunnelRegistry) Get(envID string) mo.Option[*AgentTunnel] {
tunnel, ok := r.tunnels.Get(envID)
@@ -122,12 +113,19 @@ func (r *TunnelRegistry) Register(envID string, tunnel *AgentTunnel) {
slog.Error("Failed to register edge agent tunnel", "environment_id", envID, "error", "tunnel is required")
return
}
- previous, replaced, err := r.tunnels.Store(context.Background(), "register edge tunnel", envID, tunnel)
+ previous, err := r.tunnels.ApplyTyped(context.Background(), "register edge tunnel", func(tunnels map[string]*AgentTunnel) (*AgentTunnel, bool, error) {
+ if r.stopped.Load() {
+ return nil, false, errors.New("edge tunnel registry stopped")
+ }
+ previous := tunnels[envID]
+ tunnels[envID] = tunnel
+ return previous, true, nil
+ })
if err != nil {
slog.Error("Failed to register edge agent tunnel", "environment_id", envID, "error", err)
return
}
- if replaced && previous != tunnel {
+ if previous != nil && previous != tunnel {
slog.Info("Replacing existing edge tunnel")
_ = previous.CloseWithReason("")
}
@@ -154,6 +152,9 @@ func (r *TunnelRegistry) RegisterSession(ctx context.Context, tunnel *AgentTunne
result, err := r.tunnels.ApplyTyped(ctx, "register edge tunnel session", func(tunnels map[string]*AgentTunnel) (registerSessionResultInternal, bool, error) {
var result registerSessionResultInternal
+ if r.stopped.Load() {
+ return result, false, errors.New("edge tunnel registry stopped")
+ }
if existing := tunnels[envID]; existing != nil {
if existing == tunnel {
result.accepted = true
@@ -258,8 +259,9 @@ func (r *TunnelRegistry) CleanupStale(ctx context.Context, maxAge time.Duration)
return removed
}
-// Stop drains registry mutations, closes active tunnels, and joins its actor.
+// Stop drains registry mutations and closes active tunnels.
func (r *TunnelRegistry) Stop(ctx context.Context) error {
+ r.stopped.Store(true)
fallback := r.tunnels.Values()
removed, err := r.tunnels.Drain(ctx, "stop edge tunnel registry")
if err != nil {
@@ -270,7 +272,7 @@ func (r *TunnelRegistry) Stop(ctx context.Context) error {
err = errors.Combine(err, tunnel.CloseWithReason("edge tunnel registry stopped"))
}
}
- return errors.Combine(err, r.tunnels.Stop(ctx))
+ return err
}
var (
diff --git a/backend/pkg/libarcane/edge/registry_test.go b/backend/pkg/libarcane/edge/registry_test.go
index 8e34aa971..b1c098d12 100644
--- a/backend/pkg/libarcane/edge/registry_test.go
+++ b/backend/pkg/libarcane/edge/registry_test.go
@@ -9,10 +9,8 @@ import (
"time"
"github.com/coder/websocket"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
)
type controlledCloseTunnelConn struct {
@@ -185,12 +183,8 @@ func TestTunnelRegistry_RegisterSessionReplacesSameAgentInstance(t *testing.T) {
assert.Equal(t, tunnel2, got)
}
-func TestActorTunnelRegistryPublishesReplacementBeforeClosingPreviousInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- registry, err := NewActorTunnelRegistry(t.Context(), runtime)
- require.NoError(t, err)
+func TestTunnelRegistryPublishesReplacementBeforeClosingPreviousInternal(t *testing.T) {
+ registry := NewTunnelRegistry()
previousConn := &controlledCloseTunnelConn{
closeStarted: make(chan struct{}),
@@ -234,15 +228,10 @@ func TestActorTunnelRegistryPublishesReplacementBeforeClosingPreviousInternal(t
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
require.NoError(t, registry.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
}
-func TestActorTunnelRegistryReportsStoppedExecutorAsInfrastructureFailureInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- registry, err := NewActorTunnelRegistry(t.Context(), runtime)
- require.NoError(t, err)
+func TestTunnelRegistryRejectsRegistrationAfterStopInternal(t *testing.T) {
+ registry := NewTunnelRegistry()
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
@@ -258,7 +247,6 @@ func TestActorTunnelRegistryReportsStoppedExecutorAsInfrastructureFailureInterna
require.False(t, accepted)
require.False(t, drainPrevious)
require.Empty(t, reason)
- require.NoError(t, lifecycle.Stop(stopCtx))
}
func TestTunnelRegistry_CleanupStale(t *testing.T) {
diff --git a/backend/pkg/libarcane/edge/server.go b/backend/pkg/libarcane/edge/server.go
index f9856fe23..506ae91be 100644
--- a/backend/pkg/libarcane/edge/server.go
+++ b/backend/pkg/libarcane/edge/server.go
@@ -582,7 +582,7 @@ func (s *TunnelServer) handleEvent(ctx context.Context, tunnel *AgentTunnel, msg
// StartCleanupLoop periodically cleans up stale tunnels.
func (s *TunnelServer) StartCleanupLoop(ctx context.Context) {
- defer close(s.cleanupDone)
+ defer s.cleanupDoneOnce.Do(func() { close(s.cleanupDone) })
ticker := time.NewTicker(tunnelStaleSweepInterval)
defer ticker.Stop()
diff --git a/backend/pkg/libarcane/edge/server_test.go b/backend/pkg/libarcane/edge/server_test.go
index dd700e231..d593f9af0 100644
--- a/backend/pkg/libarcane/edge/server_test.go
+++ b/backend/pkg/libarcane/edge/server_test.go
@@ -16,11 +16,9 @@ import (
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/labstack/echo/v5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
@@ -648,11 +646,7 @@ func TestTunnelServer_ManageConnectedTunnel_RegistersBeforeSendingGRPCRegisterRe
}
func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCancellationInternal(t *testing.T) {
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- registry, err := NewActorTunnelRegistry(t.Context(), runtime)
- require.NoError(t, err)
+ registry := NewTunnelRegistry()
server := NewTunnelServerWithRegistry(registry, nil, nil)
tunnel := NewAgentTunnelWithConn("env-cancelled-disconnect", &registerResponseOrderConn{recvErr: context.Canceled})
@@ -666,5 +660,4 @@ func TestTunnelServer_ManageConnectedTunnel_UnregistersAfterConnectionContextCan
stopCtx, cancelStop := context.WithTimeout(context.Background(), time.Second)
defer cancelStop()
require.NoError(t, registry.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
}
diff --git a/backend/pkg/libarcane/edge/types.go b/backend/pkg/libarcane/edge/types.go
index 322d1a8f5..5bd8f8ac5 100644
--- a/backend/pkg/libarcane/edge/types.go
+++ b/backend/pkg/libarcane/edge/types.go
@@ -9,7 +9,7 @@ import (
"time"
"github.com/coder/websocket"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency"
"google.golang.org/grpc"
)
@@ -57,7 +57,7 @@ type TunnelClient struct {
preferWebSocketUntil time.Time
grpcFailureStreak int
agentInstanceID string
- registration actors.Snapshot[clientRegistrationInternal]
+ registration concurrency.Snapshot[clientRegistrationInternal]
}
type clientRegistrationInternal struct {
@@ -217,7 +217,8 @@ type AgentTunnel struct {
// TunnelRegistry manages active edge agent tunnel connections
type TunnelRegistry struct {
- tunnels *actors.StateMap[string, *AgentTunnel]
+ stopped atomic.Bool
+ tunnels *concurrency.StateMap[string, *AgentTunnel]
}
type internalTunnelRequestContextKey struct{}
@@ -231,6 +232,7 @@ type TunnelServer struct {
eventCallback EventCallback
enrollmentCallback EnrollmentCallback
cleanupDone chan struct{}
+ cleanupDoneOnce sync.Once
cfg *Config
statusMu sync.Mutex
}
diff --git a/backend/pkg/scheduler/actor_test.go b/backend/pkg/scheduler/actor_test.go
index e89aaba0c..0a81156db 100644
--- a/backend/pkg/scheduler/actor_test.go
+++ b/backend/pkg/scheduler/actor_test.go
@@ -6,89 +6,88 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
+ "gorm.io/gorm"
)
-func newTestAdmissionGateInternal(t *testing.T) *actors.Gate[actors.AdmissionKey] {
+var schedulerTestRuntimesInternal sync.Map
+
+func newTestAdmissionGateInternal(t *testing.T) *runs.Admission {
+ t.Helper()
+ runtime := francistest.New(t)
+ admission := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, admission.Register(runtime))
+ francistest.Start(t, runtime)
+ return admission
+}
+
+func newTestCoordinatorInternal(t testing.TB, ctx context.Context, location *time.Location, execute func(context.Context, schedulertypes.Run) (schedulertypes.Outcome, error)) (*runs.Coordinator, *francis.Runtime) {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
- gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "scheduler-test-admission", t.Name())
+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))
+ sqlDB, err := db.DB()
require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, gate.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
- return gate
+ sqlDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
+ runtime := francistest.New(t)
+ coordinator := runs.New(kv.NewKVService(&database.DB{DB: db}), runtime.Service(), location)
+ coordinator.SetExecutor(execute, nil)
+ require.NoError(t, coordinator.Register(runtime))
+ require.NoError(t, runtime.Start(t.Context(), ctx, nil))
+ t.Cleanup(func() { require.NoError(t, runtime.Stop(context.Background())) })
+ require.NoError(t, coordinator.Start(ctx))
+ coordinator.Activate()
+ t.Cleanup(func() { require.NoError(t, coordinator.Stop(context.Background())) })
+ return coordinator, runtime
}
func newJobSchedulerForTestInternal(t testing.TB, ctx context.Context, location *time.Location) *jobSchedulerInternal {
t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- created, err := NewJobScheduler(ctx, runtime, location)
- require.NoError(t, err)
- scheduler := created.(*jobSchedulerInternal)
- scheduler.SetDispatcher(&testDispatcherInternal{run: func(ctx context.Context, request schedulertypes.Request) error {
- job, ok := scheduler.GetJob(request.JobID)
+ var scheduler *jobSchedulerInternal
+ coordinator, runtime := newTestCoordinatorInternal(t, ctx, location, func(ctx context.Context, run schedulertypes.Run) (schedulertypes.Outcome, error) {
+ job, ok := scheduler.GetJob(run.JobID)
if !ok {
- return nil
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
- _, err := job.Run(ctx)
- return err
- }})
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, scheduler.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
+ return job.Run(jobcontext.WithExecution(ctx, run, nil))
})
- return scheduler
-}
-
-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
- t.Helper()
- lifecycle := fxtest.NewLifecycle(t)
- runtime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- executor, err := actors.NewExecutor(t.Context(), runtime, "settings-test", t.Name(), 3)
- require.NoError(t, err)
- effects, err := actors.NewExecutor(t.Context(), runtime, "settings-effects-test", t.Name(), 3)
+ created, err := NewJobScheduler(ctx, coordinator, location)
require.NoError(t, err)
+ scheduler = created.(*jobSchedulerInternal)
+ schedulerTestRuntimesInternal.Store(scheduler, runtime)
t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, executor.Stop(stopCtx))
- require.NoError(t, effects.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
+ require.NoError(t, scheduler.Stop(context.Background()))
+ schedulerTestRuntimesInternal.Delete(scheduler)
})
- return settings.NewSettingsService(ctx, db, executor, effects)
-}
-
-type testDispatcherInternal struct {
- run func(context.Context, schedulertypes.Request) error
- locks sync.Map
+ return scheduler
}
-func (d *testDispatcherInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
- value, _ := d.locks.LoadOrStore(request.JobID, &sync.Mutex{})
- lock := value.(*sync.Mutex)
- if !lock.TryLock() {
- return schedulertypes.Run{}, nil
+func stopJobSchedulerForTestInternal(ctx context.Context, scheduler *jobSchedulerInternal) error {
+ if err := scheduler.Stop(ctx); err != nil {
+ return err
+ }
+ if err := scheduler.coordinator.Stop(ctx); err != nil {
+ return err
}
- defer lock.Unlock()
- return schedulertypes.Run{JobID: request.JobID}, d.run(ctx, request)
+ runtime, _ := schedulerTestRuntimesInternal.Load(scheduler)
+ return runtime.(*francis.Runtime).Stop(ctx)
}
-func (*testDispatcherInternal) Checkpoint(context.Context, string, string, time.Time) error {
- return nil
+func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+ t.Helper()
+ svc, err := settings.NewSettingsService(ctx, db)
+ if err == nil {
+ t.Cleanup(func() { require.NoError(t, svc.Stop(context.Background())) })
+ }
+ return svc, err
}
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index 316d260bf..21c757345 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -3,6 +3,7 @@ package scheduler
import (
"cmp"
"context"
+ "encoding/json/v2"
"log/slog"
"slices"
"strings"
@@ -10,7 +11,6 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -19,6 +19,7 @@ import (
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -38,6 +39,17 @@ const (
)
// restartRecord tracks restart timestamps for a single container.
+type autoHealPlanInternal struct {
+ Containers []container.Summary `json:"containers"`
+ MaxRestarts int `json:"maxRestarts"`
+ WindowMinutes int `json:"windowMinutes"`
+}
+
+type autoHealBaselineInternal struct {
+ StartedAt string `json:"startedAt"`
+ RestartCount int `json:"restartCount"`
+}
+
type restartRecord struct {
timestamps []time.Time
}
@@ -47,7 +59,7 @@ type AutoHealJob struct {
settingsService *settings.SettingsService
eventService *event.EventService
notificationService *notification.NotificationService
- admissionGate *actors.Gate[actors.AdmissionKey]
+ admissionGate *runs.Admission
mu sync.Mutex
restarts map[string]*restartRecord
@@ -67,7 +79,7 @@ func NewAutoHealJob(
settingsService *settings.SettingsService,
eventService *event.EventService,
notificationService *notification.NotificationService,
- admissionGate *actors.Gate[actors.AdmissionKey],
+ admissionGate *runs.Admission,
) (*AutoHealJob, error) {
if admissionGate == nil {
return nil, errors.New("auto-heal admission gate unavailable")
@@ -109,7 +121,7 @@ func (j *AutoHealJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
- lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoHealAdmissionScopeInternal})
+ lease, admitted, err := j.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: autoHealAdmissionScopeInternal})
if err != nil {
slog.ErrorContext(ctx, "auto-heal admission failed", "error", err)
return schedulertypes.Outcome{}, err
@@ -118,7 +130,7 @@ func (j *AutoHealJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
slog.WarnContext(ctx, "auto-heal run still in progress; skipping overlapping run")
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
- defer lease.Release()
+ defer lease.Release(ctx)
dockerClient, err := j.getDockerClientInternal(ctx)
if err != nil {
@@ -140,6 +152,13 @@ func (j *AutoHealJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
selfID := j.selfContainerIDInternal(ctx)
candidates := j.filterCandidatesInternal(containers, excludedContainers, selfID)
+ frozen, err := json.Marshal(autoHealPlanInternal{Containers: candidates, MaxRestarts: maxRestarts, WindowMinutes: restartWindowMinutes})
+ if err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ResourceType: "heal_plan", ID: "auto-heal-plan", Status: schedulertypes.Succeeded, RecoveryData: frozen}); err != nil {
+ return schedulertypes.Outcome{}, err
+ }
g, groupCtx := errgroup.WithContext(ctx)
var resultMu sync.Mutex
@@ -223,7 +242,7 @@ func (j *AutoHealJob) processCandidateInternal(
restartWindowMinutes int,
) (schedulertypes.TargetOutcome, error) {
containerID := candidate.ID
- target := schedulertypes.TargetOutcome{ID: containerID, Status: schedulertypes.Skipped}
+ target := schedulertypes.TargetOutcome{ResourceType: "container", ID: containerID, Status: schedulertypes.Skipped}
if previous, ok := jobcontext.Run(ctx); ok {
for _, completed := range previous.Outcome.Targets {
if completed.ID == containerID && completed.Status == schedulertypes.Succeeded {
@@ -262,6 +281,12 @@ func (j *AutoHealJob) processCandidateInternal(
return target, nil
}
+ baseline, err := json.Marshal(autoHealBaselineInternal{StartedAt: inspect.State.StartedAt, RestartCount: inspect.RestartCount})
+ if err != nil {
+ releaseSlot()
+ return target, err
+ }
+ target.RecoveryData = baseline
target.Status = schedulertypes.Running
if err := jobcontext.Progress(ctx, target); err != nil {
releaseSlot()
@@ -514,10 +539,93 @@ func (j *AutoHealJob) RecordRestartAtExported(containerID string, t time.Time) {
}
func (j *AutoHealJob) Reconcile(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ lease, admitted, err := j.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: autoHealAdmissionScopeInternal})
+ if err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ if !admitted {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "A container healing run already owns admission"}, nil
+ }
+ defer lease.Release(ctx)
+ var plan autoHealPlanInternal
+ planned := false
for _, target := range previous.Outcome.Targets {
- if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "A container restart has an unconfirmed outcome", Targets: previous.Outcome.Targets}, nil
+ if target.ID == "auto-heal-plan" && len(target.RecoveryData) > 0 {
+ if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ planned = true
+ break
+ }
+ }
+
+ dockerClient, err := j.getDockerClientInternal(ctx)
+ if err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ message, err := j.confirmRestartTargetsInternal(ctx, dockerClient, previous.Outcome.Targets)
+ if err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ if message != "" {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: message, Targets: previous.Outcome.Targets}, nil
+ }
+ if !planned {
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: previous.Outcome.Targets}, nil
+ }
+ return j.resumeHealingPlanInternal(ctx, dockerClient, plan, previous)
+}
+
+func (j *AutoHealJob) confirmRestartTargetsInternal(ctx context.Context, dockerClient *client.Client, targets []schedulertypes.TargetOutcome) (string, error) {
+ message := ""
+ for index, target := range targets {
+ if target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped {
+ continue
+ }
+ var baseline autoHealBaselineInternal
+ if len(target.RecoveryData) == 0 || json.Unmarshal(target.RecoveryData, &baseline) != nil {
+ message = "A container restart has no persisted baseline"
+ break
+ }
+ inspect, inspectErr := j.inspectContainerInternal(ctx, dockerClient, target.ID)
+ if inspectErr != nil || inspect.State == nil || inspect.State.Restarting || (inspect.State.StartedAt == baseline.StartedAt && inspect.RestartCount == baseline.RestartCount) {
+ message = "A container restart has an unconfirmed outcome"
+ break
+ }
+ target.Status = schedulertypes.Succeeded
+ target.Message = "Container restart confirmed after recovery"
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return "", err
+ }
+ targets[index] = target
+ }
+ return message, nil
+}
+
+func (j *AutoHealJob) resumeHealingPlanInternal(ctx context.Context, dockerClient *client.Client, plan autoHealPlanInternal, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}
+ for _, candidate := range plan.Containers {
+ attempted := false
+ for _, target := range previous.Outcome.Targets {
+ if target.ID == candidate.ID {
+ attempted = true
+ break
+ }
+ }
+ if attempted {
+ continue
+ }
+ target, err := j.processCandidateInternal(ctx, dockerClient, candidate, plan.MaxRestarts, time.Duration(plan.WindowMinutes)*time.Minute, plan.WindowMinutes)
+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {
+ return schedulertypes.Outcome{}, progressErr
+ }
+ previous.Outcome.Targets = append(previous.Outcome.Targets, target)
+ if err != nil {
+ outcome.Status = schedulertypes.NeedsAttention
+ outcome.Message = err.Error()
+ break
}
}
- return j.Run(jobcontext.WithExecution(ctx, previous, nil))
+ outcome.Targets = previous.Outcome.Targets
+ return outcome, nil
}
diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.go
index 73793e0c3..8a7923895 100644
--- a/backend/pkg/scheduler/auto_update_job.go
+++ b/backend/pkg/scheduler/auto_update_job.go
@@ -7,10 +7,10 @@ import (
"strings"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/updater"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
updatertypes "github.com/getarcaneapp/arcane/types/v2/updater"
kit "go.getarcane.app/kit/pkg"
@@ -27,15 +27,15 @@ type pendingUpdateApplierInternal interface {
type AutoUpdateJob struct {
updaterService pendingUpdateApplierInternal
settingsService *settings.SettingsService
- admissionGate *actors.Gate[actors.AdmissionKey]
+ admissionGate *runs.Admission
}
-func NewAutoUpdateJob(updaterModule *updater.Module, settingsService *settings.SettingsService, admissionGate *actors.Gate[actors.AdmissionKey]) (*AutoUpdateJob, error) {
+func NewAutoUpdateJob(updaterService *updater.UpdaterService, settingsService *settings.SettingsService, admissionGate *runs.Admission) (*AutoUpdateJob, error) {
if admissionGate == nil {
return nil, errors.New("auto-update admission gate unavailable")
}
return &AutoUpdateJob{
- updaterService: updaterModule.Service(),
+ updaterService: updaterService,
settingsService: settingsService,
admissionGate: admissionGate,
}, nil
@@ -82,7 +82,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
- lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})
+ lease, admitted, err := j.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})
if err != nil {
slog.ErrorContext(ctx, "auto-update admission failed", "error", err)
return schedulertypes.Outcome{}, err
@@ -94,7 +94,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
- defer lease.Release()
+ defer lease.Release(ctx)
slog.InfoContext(ctx, "auto-update run started")
@@ -117,7 +117,10 @@ func (j *AutoUpdateJob) Reschedule(ctx context.Context) error {
return nil
}
-func (j *AutoUpdateJob) Reconcile(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+func (j *AutoUpdateJob) Reconcile(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ if service, ok := j.updaterService.(*updater.UpdaterService); ok {
+ return service.ReconcilePending(ctx, previous)
+ }
confirmed := confirmedAutoUpdateInternal(previous)
if confirmed.Status != schedulertypes.Succeeded {
confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion; review this run's results"
diff --git a/backend/pkg/scheduler/entityjobs/entityjobs.go b/backend/pkg/scheduler/entityjobs/entityjobs.go
index 9bc1038bf..0e9604f26 100644
--- a/backend/pkg/scheduler/entityjobs/entityjobs.go
+++ b/backend/pkg/scheduler/entityjobs/entityjobs.go
@@ -9,7 +9,7 @@ import (
"log/slog"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
@@ -28,7 +28,7 @@ type Registry struct {
lifecycleCtx context.Context
scheduler schedulertypes.DynamicScheduler
- admissionGate *actors.Gate[actors.AdmissionKey]
+ admissionGate *runs.Admission
}
// New creates a registry whose job names are jobPrefix+entityID and whose
@@ -40,7 +40,7 @@ func New(jobPrefix, admissionScope string) *Registry {
// SetScheduler injects the job scheduler, the admission gate and the app
// lifecycle context. The lifecycle context is what scheduled runs execute on,
// so they outlive the request or bootstrap goroutine that registered them.
-func (r *Registry) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error { //nolint:contextcheck // scheduled runs must capture the app lifecycle context, not request contexts
+func (r *Registry) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *runs.Admission) error { //nolint:contextcheck // scheduled runs must capture the app lifecycle context, not request contexts
if scheduler == nil || admissionGate == nil {
return errors.Errorf("%s scheduler dependencies unavailable", r.admissionScope)
}
@@ -104,6 +104,6 @@ func (r *Registry) Unregister(ctx context.Context, entityID string) {
// TryAcquire admits at most one in-flight run per entity ID. It refuses
// immediately when a run is already active.
-func (r *Registry) TryAcquire(ctx context.Context, entityID string) (*actors.Lease[actors.AdmissionKey], bool, error) {
- return r.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: r.admissionScope, ID: entityID})
+func (r *Registry) TryAcquire(ctx context.Context, entityID string) (*runs.Lease, bool, error) {
+ return r.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: r.admissionScope, ID: entityID})
}
diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.go
index 849524737..b684ee99b 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job.go
@@ -4,10 +4,10 @@ import (
"context"
"log/slog"
"runtime"
+ "sync"
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/template"
@@ -21,49 +21,66 @@ type FilesystemWatcherJob struct {
settingsService *settings.SettingsService
projectScanDepth int
lifecycleCtx context.Context
- projectsWatcher *actors.Resource[*fswatch.Watcher]
- templatesWatcher *actors.Resource[*fswatch.Watcher]
+ projectsWatcher *fswatch.Watcher
+ templatesWatcher *fswatch.Watcher
+ mu sync.Mutex
+ stopped bool
+ retired []*fswatch.Watcher
}
func NewFilesystemWatcherJob(
ctx context.Context,
- actorRuntime *actors.Runtime,
projectService *project.ProjectService,
templateService *template.TemplateService,
settingsService *settings.SettingsService,
projectScanDepth int,
) (*FilesystemWatcherJob, error) {
- projectsWatcher, err := actors.NewResource(ctx, actorRuntime, "filesystem-watcher", "projects", 3, (*fswatch.Watcher).Stop)
- if err != nil {
- return nil, err
- }
- templatesWatcher, err := actors.NewResource(ctx, actorRuntime, "filesystem-watcher", "templates", 3, (*fswatch.Watcher).Stop)
- if err != nil {
- return nil, errors.Combine(err, projectsWatcher.Stop(ctx))
- }
return &FilesystemWatcherJob{
projectService: projectService,
templateService: templateService,
settingsService: settingsService,
projectScanDepth: projectScanDepth,
lifecycleCtx: ctx,
- projectsWatcher: projectsWatcher,
- templatesWatcher: templatesWatcher,
}, nil
}
func (j *FilesystemWatcherJob) Start(ctx context.Context) error {
- if err := j.projectsWatcher.Restart(ctx, "start projects filesystem watcher", j.startProjectsWatcherInternal); err != nil {
+ if err := j.RestartProjectsWatcher(ctx); err != nil {
return err
}
- if err := j.templatesWatcher.Restart(ctx, "start templates filesystem watcher", j.startTemplatesWatcherInternal); err != nil {
- return errors.Combine(err, j.projectsWatcher.Clear(ctx, "clear projects watcher after template startup failure"))
+ if err := j.RestartTemplatesWatcher(ctx); err != nil {
+ j.mu.Lock()
+ watcher := j.projectsWatcher
+ j.projectsWatcher = nil
+ j.mu.Unlock()
+ if watcher != nil {
+ return errors.Combine(err, watcher.Stop())
+ }
+ return err
}
return nil
}
func (j *FilesystemWatcherJob) Stop(ctx context.Context) error {
- return errors.Combine(j.projectsWatcher.Stop(ctx), j.templatesWatcher.Stop(ctx))
+ j.mu.Lock()
+ j.stopped = true
+ projectsWatcher, templatesWatcher := j.projectsWatcher, j.templatesWatcher
+ j.projectsWatcher = nil
+ j.templatesWatcher = nil
+ retired := j.retired
+ j.retired = nil
+ j.mu.Unlock()
+ var err error
+ if projectsWatcher != nil {
+ err = projectsWatcher.Stop()
+ }
+ if templatesWatcher != nil {
+ err = errors.Combine(err, templatesWatcher.Stop())
+ }
+ for _, watcher := range retired {
+ err = errors.Combine(err, watcher.Stop())
+ }
+ return err
}
func (j *FilesystemWatcherJob) handleFilesystemChangeInternal(ctx context.Context) {
@@ -81,37 +98,54 @@ func (j *FilesystemWatcherJob) handleProjectFilePathsChangedInternal(ctx context
if len(paths) == 0 || j.projectService == nil {
return
}
- err := j.projectsWatcher.Do(ctx, "sync changed project files", func(workCtx context.Context, _ *fswatch.Watcher) error {
- j.handleFilesystemChangeInternal(workCtx)
- j.projectService.HandleProjectFilesChanged(workCtx, paths)
- return nil
- })
- if err != nil && ctx.Err() == nil && !errors.Is(err, actors.ErrResourceStopped) {
- slog.ErrorContext(ctx, "Failed to dispatch changed project files", "error", err)
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ if j.stopped || ctx.Err() != nil {
+ return
}
+ j.handleFilesystemChangeInternal(ctx)
+ j.projectService.HandleProjectFilesChanged(ctx, paths)
}
func (j *FilesystemWatcherJob) handleTemplatesChangeInternal(ctx context.Context) {
- err := j.templatesWatcher.Do(ctx, "sync changed templates", func(workCtx context.Context, _ *fswatch.Watcher) error {
- slog.InfoContext(workCtx, "Template directory change detected, syncing templates")
- if j.templateService == nil {
- return nil
- }
- if syncErr := j.templateService.SyncLocalTemplatesFromFilesystem(workCtx); syncErr != nil {
- slog.ErrorContext(workCtx, "Failed to sync templates after filesystem change", "error", syncErr)
- } else {
- slog.InfoContext(workCtx, "Template sync completed after filesystem change")
- }
- return nil
- })
- if err != nil && ctx.Err() == nil && !errors.Is(err, actors.ErrResourceStopped) {
- slog.ErrorContext(ctx, "Failed to dispatch template filesystem change", "error", err)
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ if j.stopped || ctx.Err() != nil || j.templateService == nil {
+ return
+ }
+ if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
+ slog.ErrorContext(ctx, "Failed to sync templates after filesystem change", "error", err)
}
}
func (j *FilesystemWatcherJob) RestartProjectsWatcher(ctx context.Context) error {
slog.InfoContext(ctx, "Restarting projects filesystem watcher")
- return j.projectsWatcher.Restart(ctx, "restart projects filesystem watcher", j.startProjectsWatcherInternal)
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ if j.stopped {
+ return errors.New("filesystem watcher stopped")
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if j.projectsWatcher != nil {
+ previous := j.projectsWatcher
+ j.projectsWatcher = nil
+ j.retired = append(j.retired, previous)
+ // Callbacks need the domain mutex, so stop the watch loop separately from joining them.
+ if err := previous.StopWatching(); err != nil {
+ return err
+ }
+ }
+ watcher, err := j.startProjectsWatcherInternal(ctx)
+ if err != nil {
+ if watcher != nil {
+ _ = watcher.StopWatching()
+ }
+ return err
+ }
+ j.projectsWatcher = watcher
+ return nil
}
func (j *FilesystemWatcherJob) startProjectsWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {
@@ -164,7 +198,32 @@ func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlin
func (j *FilesystemWatcherJob) RestartTemplatesWatcher(ctx context.Context) error {
slog.InfoContext(ctx, "Restarting templates filesystem watcher")
- return j.templatesWatcher.Restart(ctx, "restart templates filesystem watcher", j.startTemplatesWatcherInternal)
+ j.mu.Lock()
+ defer j.mu.Unlock()
+ if j.stopped {
+ return errors.New("filesystem watcher stopped")
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if j.templatesWatcher != nil {
+ previous := j.templatesWatcher
+ j.templatesWatcher = nil
+ j.retired = append(j.retired, previous)
+ // Callbacks need the domain mutex, so stop the watch loop separately from joining them.
+ if err := previous.StopWatching(); err != nil {
+ return err
+ }
+ }
+ watcher, err := j.startTemplatesWatcherInternal(ctx)
+ if err != nil {
+ if watcher != nil {
+ _ = watcher.StopWatching()
+ }
+ return err
+ }
+ j.templatesWatcher = watcher
+ return nil
}
func (j *FilesystemWatcherJob) startTemplatesWatcherInternal(ctx context.Context) (*fswatch.Watcher, error) {
diff --git a/backend/pkg/scheduler/filesystem_watcher_job_test.go b/backend/pkg/scheduler/filesystem_watcher_job_test.go
index 408c63d89..79d431032 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job_test.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job_test.go
@@ -6,11 +6,8 @@ import (
"testing"
"time"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/fswatch"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "go.uber.org/fx/fxtest"
)
func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *testing.T) {
@@ -29,10 +26,7 @@ func TestFilesystemWatcherJob_ConcurrentProjectRestartsAreSerializedInternal(t *
require.NoError(t, settingsService.SetStringSetting(t.Context(), "projectsDirectory", t.TempDir()))
require.NoError(t, settingsService.SetStringSetting(t.Context(), "templatesDirectory", t.TempDir()))
- lifecycle := fxtest.NewLifecycle(t)
- actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- job, err := NewFilesystemWatcherJob(t.Context(), actorRuntime, nil, nil, settingsService, 2)
+ job, err := NewFilesystemWatcherJob(t.Context(), nil, nil, settingsService, 2)
require.NoError(t, err)
restartErrors := make(chan error, 2)
@@ -49,22 +43,17 @@ func TestFilesystemWatcherJob_ConcurrentProjectRestartsAreSerializedInternal(t *
for restartErr := range restartErrors {
require.NoError(t, restartErr)
}
- var earlyWatcher *fswatch.Watcher
- require.NoError(t, job.projectsWatcher.Do(t.Context(), "capture early watcher", func(_ context.Context, watcher *fswatch.Watcher) error {
- earlyWatcher = watcher
- return nil
- }))
+ job.mu.Lock()
+ earlyWatcher := job.projectsWatcher
+ job.mu.Unlock()
require.NotNil(t, earlyWatcher)
require.NoError(t, job.Start(t.Context()))
- var startedWatcher *fswatch.Watcher
- require.NoError(t, job.projectsWatcher.Do(t.Context(), "capture started watcher", func(_ context.Context, watcher *fswatch.Watcher) error {
- startedWatcher = watcher
- return nil
- }))
+ job.mu.Lock()
+ startedWatcher := job.projectsWatcher
+ job.mu.Unlock()
require.NotSame(t, earlyWatcher, startedWatcher)
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
require.NoError(t, job.Stop(stopCtx))
- require.NoError(t, lifecycle.Stop(stopCtx))
}
diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.go
index 65a857932..8edfd3b0a 100644
--- a/backend/pkg/scheduler/image_update_watcher.go
+++ b/backend/pkg/scheduler/image_update_watcher.go
@@ -8,7 +8,6 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -16,6 +15,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -24,20 +26,9 @@ import (
)
const (
- imageUpdateTriggerMessageInternal imageUpdateMessageKindInternal = 0
- imageUpdateScheduleRefreshMessageInternal imageUpdateMessageKindInternal = 1
- imageUpdateBackfillCompletedMessageInternal imageUpdateMessageKindInternal = 2
- imageUpdateDebounceElapsedMessageInternal imageUpdateMessageKindInternal = 3
- imageUpdateScheduledPollMessageInternal imageUpdateMessageKindInternal = 4
- imageUpdateScanCompletedMessageInternal imageUpdateMessageKindInternal = 5
- imageUpdateManualScanRequestMessageInternal imageUpdateMessageKindInternal = 6
- imageUpdateManualScanAdmissionMessageInternal imageUpdateMessageKindInternal = 7
-
imageUpdateWatcherDebounce = 2 * time.Second
imageUpdateWatcherBackfillRetry = 5 * time.Second
imageUpdateWatcherDefaultSchedule = "0 0 * * * *"
- imageUpdateWatcherStopTimeout = 30 * time.Second
- imageUpdateWatcherActorID = "image-update-watcher"
)
type imageUpdateScannerInternal interface {
@@ -61,87 +52,77 @@ type projectImageRefsBackfillerInternal interface {
BackfillProjectImageRefs(ctx context.Context) (int, error)
}
-type imageUpdateMessageKindInternal uint8
-
-type imageUpdateScanCompletionInternal struct {
- automatic bool
- triggerGeneration uint64
- done *actors.Promise[error]
-}
-
-// ImageUpdateWatcher continuously reconciles image update state after Docker image changes.
+// ImageUpdateWatcher debounces Docker events and admits durable image scans.
type ImageUpdateWatcher struct {
- imageUpdateService imageUpdateScannerInternal
- settingsService pollingSettingReaderInternal
- environmentService registryCredentialLoaderInternal
- dockerService dockerEventBusProviderInternal
- projectService projectImageRefsBackfillerInternal
- dispatchMu sync.RWMutex
- dispatcher schedulertypes.Dispatcher
- eventDegraded atomic.Bool
- actorRuntime *actors.Runtime
- actorProcess atomic.Pointer[actors.Actor]
- stopping atomic.Bool
- triggerIngress *actors.Ingress[imageUpdateMessageKindInternal, time.Time]
- scheduleIngress *actors.Ingress[imageUpdateMessageKindInternal, actors.NoPayload]
- location *time.Location
- debounce time.Duration
- backfillRetry time.Duration
- metadataReady chan struct{}
- metadataReadyOnce sync.Once
- started chan struct{}
- startedOnce sync.Once
- stopped chan struct{}
- stoppedOnce sync.Once
+ imageUpdateService imageUpdateScannerInternal
+ settingsService pollingSettingReaderInternal
+ environmentService registryCredentialLoaderInternal
+ dockerService dockerEventBusProviderInternal
+ projectService projectImageRefsBackfillerInternal
+ dispatchMu sync.RWMutex
+ coordinator *runs.Coordinator
+ eventDegraded atomic.Bool
+ running atomic.Bool
+ scanRunning atomic.Bool
+ stopping atomic.Bool
+ triggerGeneration atomic.Uint64
+ acknowledgedGeneration atomic.Uint64
+ triggeredAt atomic.Int64
+ trigger chan struct{}
+ scheduleRefresh chan struct{}
+ runMu sync.Mutex
+ cancel context.CancelFunc
+ done chan struct{}
+ location *time.Location
+ debounce time.Duration
+ backfillRetry time.Duration
+ metadataReady chan struct{}
+ metadataReadyOnce sync.Once
+ started chan struct{}
+ startedOnce sync.Once
+ stopped chan struct{}
+ stoppedOnce sync.Once
}
-// NewImageUpdateWatcher constructs the image update watcher from the existing services.
-func NewImageUpdateWatcher(runtime *actors.Runtime, cfg *config.Config, imageUpdateService *imageupdate.ImageUpdateService, settingsService *settings.SettingsService, environmentService *environment.EnvironmentService, dockerService *docker.DockerClientService, projectService *project.ProjectService) (*ImageUpdateWatcher, error) {
- if runtime == nil || imageUpdateService == nil || settingsService == nil || environmentService == nil || dockerService == nil || projectService == nil {
+func NewImageUpdateWatcher(cfg *config.Config, imageUpdateService *imageupdate.ImageUpdateService, settingsService *settings.SettingsService, environmentService *environment.EnvironmentService, dockerService *docker.DockerClientService, projectService *project.ProjectService) (*ImageUpdateWatcher, error) {
+ if imageUpdateService == nil || settingsService == nil || environmentService == nil || dockerService == nil || projectService == nil {
return nil, errors.New("image update watcher dependencies unavailable")
}
location := time.UTC
if cfg != nil {
location = cfg.GetLocation()
}
- return &ImageUpdateWatcher{
- imageUpdateService: imageUpdateService,
- settingsService: settingsService,
- environmentService: environmentService,
- dockerService: dockerService,
- projectService: projectService,
- actorRuntime: runtime,
- triggerIngress: actors.NewIngress[imageUpdateMessageKindInternal, time.Time](imageUpdateTriggerMessageInternal),
- scheduleIngress: actors.NewIngress[imageUpdateMessageKindInternal, actors.NoPayload](imageUpdateScheduleRefreshMessageInternal),
- location: location,
- debounce: imageUpdateWatcherDebounce,
- backfillRetry: imageUpdateWatcherBackfillRetry,
- metadataReady: make(chan struct{}),
- started: make(chan struct{}),
- stopped: make(chan struct{}),
- }, nil
+ return &ImageUpdateWatcher{imageUpdateService: imageUpdateService, settingsService: settingsService, environmentService: environmentService, dockerService: dockerService, projectService: projectService, location: location, debounce: imageUpdateWatcherDebounce, backfillRetry: imageUpdateWatcherBackfillRetry, metadataReady: make(chan struct{}), started: make(chan struct{}), stopped: make(chan struct{}), trigger: make(chan struct{}, 1), scheduleRefresh: make(chan struct{}, 1)}, nil
}
-// Name identifies the watcher in scheduler lifecycle logs.
-func (w *ImageUpdateWatcher) Name() string {
- return "image-polling"
-}
+func (w *ImageUpdateWatcher) Name() string { return "image-polling" }
-// Start subscribes to Docker image events and owns the watcher actor until ctx is canceled.
+// Start owns the debounce loop and joins its workers before returning.
func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
if w.stopping.Load() {
return errors.New("image update watcher cannot be restarted after stop")
}
- if w.actorProcess.Load() != nil {
+ if !w.running.CompareAndSwap(false, true) {
return errors.New("image update watcher already started")
}
- if w.debounce <= 0 {
- w.debounce = imageUpdateWatcherDebounce
- }
- if w.backfillRetry <= 0 {
- w.backfillRetry = imageUpdateWatcherBackfillRetry
- }
+ defer w.running.Store(false)
+ runCtx, cancel := context.WithCancel(ctx)
+ done := make(chan struct{})
+ w.runMu.Lock()
+ if w.stopping.Load() {
+ w.runMu.Unlock()
+ cancel()
+ return errors.New("image update watcher stopped")
+ }
+ w.cancel = cancel
+ w.done = done
+ w.runMu.Unlock()
+ defer cancel()
+ defer close(done)
+ return w.runEventLoopInternal(runCtx)
+}
+func (w *ImageUpdateWatcher) runEventLoopInternal(runCtx context.Context) error {
eventBus := w.dockerService.EventBus()
if eventBus == nil {
return errors.New("docker event bus unavailable")
@@ -150,101 +131,174 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
defer func() { unsubscribe() }()
reconnect := time.NewTicker(5 * time.Second)
defer reconnect.Stop()
-
- process, err := actors.NewActor(
- ctx,
- w.actorRuntime,
- "scheduler",
- imageUpdateWatcherActorID,
- 3,
- func() actors.Behavior {
- state := &imageUpdateWatcherActorInternal{watcher: w}
- return actors.Behavior{
- Initialize: state.initializeInternal,
- Handle: state.receiveInternal,
- Cleanup: state.cleanupInternal,
- }
- },
- w.triggerIngress,
- w.scheduleIngress,
- )
- if err != nil {
- return errors.WrapIf(err, "start image update watcher actor")
- }
- w.actorProcess.Store(process)
+ timer := time.NewTimer(time.Hour)
+ timer.Stop()
+ defer timer.Stop()
+ var workers sync.WaitGroup
+ defer workers.Wait()
+ metadata := make(chan error, 1)
+ submitted := make(chan error, 1)
+ submitting := false
+ workers.Go(func() { w.runMetadataBackfillInternal(runCtx, metadata) })
w.startedOnce.Do(func() { close(w.started) })
-
- slog.InfoContext(ctx, "image update watcher started")
- defer func() {
- stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), imageUpdateWatcherStopTimeout)
- defer cancel()
- if err := process.Stop(stopCtx); err != nil {
- slog.ErrorContext(context.WithoutCancel(ctx), "failed to stop image update watcher actor", "error", err)
- } else {
- w.actorProcess.CompareAndSwap(process, nil)
- }
- slog.InfoContext(context.WithoutCancel(ctx), "image update watcher stopped")
- }()
-
+ w.RefreshSchedule()
+ ready := false
for {
select {
- case <-ctx.Done():
+ case <-runCtx.Done():
return nil
- case <-process.Done():
- if ctx.Err() != nil || w.stopping.Load() {
- return nil
+ case <-metadata:
+ ready = true
+ w.metadataReadyOnce.Do(func() { close(w.metadataReady) })
+ w.Trigger()
+ case <-w.trigger:
+ if ready && !submitting {
+ timer.Reset(w.debounce)
}
- return errors.New("image update watcher actor stopped unexpectedly")
- case <-reconnect.C:
- if eventCh == nil {
- if currentBus := w.dockerService.EventBus(); currentBus != nil {
- unsubscribe()
- eventCh, unsubscribe = currentBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
- w.eventDegraded.Store(false)
- }
+ case <-timer.C:
+ if !w.scanAdmissionReadyInternal(timer, ready, submitting) {
+ continue
}
+ submitting = true
+ generation := w.triggerGeneration.Load()
+ workers.Go(func() { w.submitTriggeredScanInternal(runCtx, generation, submitted) })
+ case err := <-submitted:
+ submitting = false
+ if err != nil && runCtx.Err() == nil {
+ slog.ErrorContext(runCtx, "image scan admission failed", "error", err)
+ }
+ if w.triggerGeneration.Load() != w.acknowledgedGeneration.Load() {
+ timer.Reset(w.debounce)
+ }
+ case <-w.scheduleRefresh:
+ if err := w.refreshScheduleInternal(runCtx); err != nil {
+ slog.ErrorContext(runCtx, "Failed to checkpoint image polling schedule", "error", err)
+ }
+ case <-reconnect.C:
+ eventCh, unsubscribe = w.reconnectEventsInternal(eventCh, unsubscribe)
case _, ok := <-eventCh:
if !ok {
- slog.WarnContext(ctx, "docker image event subscription closed; scheduled image polling remains active")
eventCh = nil
w.eventDegraded.Store(true)
continue
}
- if !w.settingsService.GetBoolSetting(ctx, "imageEventWatcherEnabled", false) {
- continue
+ if w.settingsService.GetBoolSetting(runCtx, "imageEventWatcherEnabled", false) {
+ w.Trigger()
}
- w.Trigger()
}
}
}
-// Stop terminates the watcher actor within the caller's lifecycle deadline.
-func (w *ImageUpdateWatcher) Stop(ctx context.Context) error {
- if ctx == nil {
- return errors.New("image update watcher stop context unavailable")
+func (w *ImageUpdateWatcher) reconnectEventsInternal(eventCh <-chan events.Message, unsubscribe func()) (<-chan events.Message, func()) {
+ if eventCh != nil {
+ return eventCh, unsubscribe
+ }
+ currentBus := w.dockerService.EventBus()
+ if currentBus == nil {
+ return eventCh, unsubscribe
+ }
+ unsubscribe()
+ eventCh, unsubscribe = currentBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
+ w.eventDegraded.Store(false)
+ return eventCh, unsubscribe
+}
+
+func (w *ImageUpdateWatcher) runMetadataBackfillInternal(ctx context.Context, metadata chan<- error) {
+ for attempt := 1; ctx.Err() == nil; attempt++ {
+ var err error
+ func() {
+ defer utils.RecoverToError(&err, "image metadata backfill")
+ err = w.backfillProjectImageRefsInternal(ctx, attempt)
+ }()
+ if err == nil {
+ select {
+ case metadata <- nil:
+ case <-ctx.Done():
+ }
+ return
+ }
+ delay := w.backfillRetry
+ if delay <= 0 {
+ delay = imageUpdateWatcherBackfillRetry
+ }
+ retry := time.NewTimer(delay)
+ select {
+ case <-retry.C:
+ case <-ctx.Done():
+ retry.Stop()
+ return
+ }
+ }
+}
+
+func (w *ImageUpdateWatcher) scanAdmissionReadyInternal(timer *time.Timer, ready, submitting bool) bool {
+ if !ready || submitting || w.triggerGeneration.Load() == w.acknowledgedGeneration.Load() {
+ return false
+ }
+ if remaining := time.Until(time.Unix(0, w.triggeredAt.Load()).Add(w.debounce)); remaining > 0 {
+ timer.Reset(remaining)
+ return false
+ }
+ return true
+}
+
+func (w *ImageUpdateWatcher) submitTriggeredScanInternal(ctx context.Context, generation uint64, submitted chan<- error) {
+ var err error
+ func() {
+ defer utils.RecoverToError(&err, "image scan admission")
+ w.dispatchMu.RLock()
+ coordinator := w.coordinator
+ w.dispatchMu.RUnlock()
+ if coordinator == nil {
+ err = errors.New("durable job coordinator unavailable")
+ return
+ }
+ _, err = coordinator.Submit(ctx, schedulertypes.Request{JobID: w.Name(), EnvironmentID: "0", Trigger: "watcher"})
+ }()
+ if err == nil {
+ w.acknowledgedGeneration.Store(generation)
}
+ select {
+ case submitted <- err:
+ case <-ctx.Done():
+ }
+}
+
+func (w *ImageUpdateWatcher) Stop(ctx context.Context) error {
w.stopping.Store(true)
w.stoppedOnce.Do(func() { close(w.stopped) })
- process := w.actorProcess.Load()
- if process == nil {
+ w.runMu.Lock()
+ cancel, done := w.cancel, w.done
+ w.runMu.Unlock()
+ if cancel == nil {
return nil
}
- return process.Stop(ctx)
+ cancel()
+ select {
+ case <-done:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
}
-// Trigger records a trailing-edge image scan without blocking the event publisher.
func (w *ImageUpdateWatcher) Trigger() {
- w.triggerIngress.Send(time.Now())
+ w.triggeredAt.Store(time.Now().UnixNano())
+ w.triggerGeneration.Add(1)
+ select {
+ case w.trigger <- struct{}{}:
+ default:
+ }
}
-// RefreshSchedule wakes the actor so it re-reads pollingInterval.
func (w *ImageUpdateWatcher) RefreshSchedule() {
- w.scheduleIngress.Send(actors.NoPayload{})
+ select {
+ case w.scheduleRefresh <- struct{}{}:
+ default:
+ }
}
-// RunNow requests immediate admission from the watcher actor and waits for that scan.
-// An active scan is refused immediately instead of being queued.
-func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
+func (w *ImageUpdateWatcher) RunNow(ctx context.Context) (err error) {
select {
case <-ctx.Done():
return ctx.Err()
@@ -259,28 +313,51 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
return errors.New("image update watcher is not running")
case <-w.metadataReady:
}
-
- process := w.actorProcess.Load()
- if process == nil {
+ if !w.running.Load() {
return errors.New("image update watcher is not running")
}
-
- admission, err := process.Request[imageUpdateMessageKindInternal, context.Context, *actors.Promise[error]](
- ctx,
- actors.Message[imageUpdateMessageKindInternal, context.Context]{Kind: imageUpdateManualScanRequestMessageInternal, Value: ctx},
- )
- if err != nil {
- return errors.WrapIf(err, "request image scan admission")
+ if _, executing := jobcontext.Run(ctx); executing {
+ if !w.scanRunning.CompareAndSwap(false, true) {
+ return common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress"))
+ }
+ defer w.scanRunning.Store(false)
+ defer utils.RecoverToError(&err, "image update scan worker")
+ return w.executeScanInternal(ctx)
}
- if admission.Kind != imageUpdateManualScanAdmissionMessageInternal {
- return errors.New("invalid image scan admission response")
+ if w.scanRunning.Load() {
+ return common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress"))
}
- if admission.Err != nil {
- return admission.Err
+ w.dispatchMu.RLock()
+ coordinator := w.coordinator
+ w.dispatchMu.RUnlock()
+ if coordinator == nil {
+ return errors.New("durable job coordinator unavailable")
+ }
+ run, err := coordinator.Submit(ctx, schedulertypes.Request{JobID: w.Name(), EnvironmentID: "0", Trigger: "manual"})
+ if err != nil {
+ return err
+ }
+ ticker := time.NewTicker(50 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ current, err := coordinator.Get(ctx, run.EnvironmentID, run.JobID, run.ID)
+ if err != nil {
+ return err
+ }
+ if current.Status.Terminal() || current.Status == schedulertypes.NeedsAttention {
+ if current.Status == schedulertypes.Succeeded {
+ return nil
+ }
+ return &schedulertypes.OutcomeError{Outcome: current.Outcome}
+ }
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-ticker.C:
+ case <-w.stopped:
+ return errors.New("image update watcher stopped")
+ }
}
-
- // Cancellation reaches the scan context; retain the execution guard until its worker exits.
- return <-admission.Value.Done()
}
func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
@@ -327,218 +404,39 @@ func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
return nil
}
-func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context, attempt int) (actors.NoPayload, error) {
- startedAt := time.Now()
+func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Context, attempt int) error {
+ started := time.Now()
count, err := w.projectService.BackfillProjectImageRefs(ctx)
- duration := time.Since(startedAt)
if err == nil {
- slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", duration, "attempt", attempt)
- return actors.NoPayload{}, nil
- }
- if ctx.Err() == nil {
- slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", duration, "attempt", attempt, "retryIn", w.backfillRetry, "error", err)
- }
- return actors.NoPayload{}, err
-}
-
-type imageUpdateWatcherActorInternal struct {
- watcher *ImageUpdateWatcher
- scanRunning bool
- scanSubmitting bool
- automaticPending bool
- triggerGeneration uint64
- metadataReady bool
- debounceTimer actors.Timer[imageUpdateMessageKindInternal]
- scheduleTimer actors.Timer[imageUpdateMessageKindInternal]
-}
-
-func (a *imageUpdateWatcherActorInternal) initializeInternal(ctx *actors.Context) {
- select {
- case <-a.watcher.metadataReady:
- a.metadataReady = true
- default:
- a.startBackfillInternal(ctx)
- }
- a.resetScheduleInternal(ctx)
-}
-
-func (a *imageUpdateWatcherActorInternal) cleanupInternal(*actors.Context) {
- a.debounceTimer.Stop()
- a.scheduleTimer.Stop()
-}
-
-func (a *imageUpdateWatcherActorInternal) receiveInternal(ctx *actors.Context, rawMessage any) {
- switch message := rawMessage.(type) {
- case actors.Message[imageUpdateMessageKindInternal, actors.NoPayload]:
- a.receiveSignalInternal(ctx, message)
- case actors.Message[imageUpdateMessageKindInternal, uint64]:
- a.receiveTimerInternal(ctx, message)
- case actors.Message[imageUpdateMessageKindInternal, context.Context]:
- if message.Kind != imageUpdateManualScanRequestMessageInternal {
- return
- }
- if a.scanRunning {
- ctx.Respond(actors.Message[imageUpdateMessageKindInternal, *actors.Promise[error]]{
- Kind: imageUpdateManualScanAdmissionMessageInternal,
- Err: common.Classify(common.ErrImageScanInProgress, errors.New("an image update check is already in progress")),
- })
- return
- }
- done := actors.NewPromise[error]()
- a.startScanInternal(message.Value, ctx, false, 0, done)
- ctx.Respond(actors.Message[imageUpdateMessageKindInternal, *actors.Promise[error]]{Kind: imageUpdateManualScanAdmissionMessageInternal, Value: done})
- case actors.Message[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]:
- defer message.Acknowledge()
- if message.Kind != imageUpdateScanCompletedMessageInternal {
- return
- }
- if message.Value.automatic {
- a.scanSubmitting = false
- if message.Err == nil {
- a.watcher.triggerIngress.Acknowledge(message.Value.triggerGeneration)
- }
- a.automaticPending = a.watcher.triggerIngress.Pending()
- } else {
- a.scanRunning = false
- }
- if message.Value.automatic && message.Err != nil && !errors.Is(message.Err, context.Canceled) {
- slog.ErrorContext(ctx.Context(), "image update watcher scan failed", "error", message.Err)
- }
- if a.automaticPending {
- a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, a.watcher.debounce)
- }
- if message.Value.done != nil {
- message.Value.done.Resolve(message.Err)
- }
- }
-}
-
-func (a *imageUpdateWatcherActorInternal) receiveSignalInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, actors.NoPayload]) {
- if message.Kind == imageUpdateTriggerMessageInternal {
- triggeredAt, generation := a.watcher.triggerIngress.Begin()
- a.automaticPending = true
- a.triggerGeneration = generation
- if a.metadataReady && !a.scanRunning && !a.scanSubmitting {
- a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, max(time.Until(triggeredAt.Add(a.watcher.debounce)), 0))
- }
- return
- }
- if message.Kind == imageUpdateScheduleRefreshMessageInternal {
- a.watcher.scheduleIngress.Take()
- a.resetScheduleInternal(ctx)
- return
- }
- if message.Kind != imageUpdateBackfillCompletedMessageInternal {
- return
- }
- defer message.Acknowledge()
- if message.Err != nil {
- slog.ErrorContext(ctx.Context(), "image update metadata backfill worker failed", "error", message.Err)
- return
- }
- a.metadataReady = true
- a.watcher.metadataReadyOnce.Do(func() { close(a.watcher.metadataReady) })
- a.watcher.Trigger()
-}
-
-func (a *imageUpdateWatcherActorInternal) receiveTimerInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, uint64]) {
- if message.Kind == imageUpdateDebounceElapsedMessageInternal {
- if !a.debounceTimer.Current(message.Value) || !a.automaticPending || a.scanRunning || a.scanSubmitting {
- return
- }
- if remaining := time.Until(a.watcher.triggerIngress.Latest().Add(a.watcher.debounce)); remaining > 0 {
- a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, remaining)
- return
- }
- a.startScanInternal(ctx.Context(), ctx, true, a.triggerGeneration, nil)
- return
- }
- if message.Kind != imageUpdateScheduledPollMessageInternal || !a.scheduleTimer.Current(message.Value) {
- return
+ slog.InfoContext(ctx, "project image metadata backfill completed", "projects", count, "duration", time.Since(started), "attempt", attempt)
+ } else if ctx.Err() == nil {
+ slog.WarnContext(ctx, "project image metadata backfill failed; retrying", "projects", count, "duration", time.Since(started), "attempt", attempt, "retryIn", w.backfillRetry, "error", err)
}
- a.watcher.Trigger()
- a.resetScheduleInternal(ctx)
-}
-
-func (a *imageUpdateWatcherActorInternal) startBackfillInternal(ctx *actors.Context) {
- if a.watcher.backfillRetry <= 0 {
- a.watcher.backfillRetry = imageUpdateWatcherBackfillRetry
- }
- attempt := 0
- worker := actors.Worker[imageUpdateMessageKindInternal, actors.NoPayload]{
- Actor: ctx,
- WorkContext: ctx.Context(),
- Label: "image update metadata backfill worker",
- CompletionKind: imageUpdateBackfillCompletedMessageInternal,
- RetryDelay: a.watcher.backfillRetry,
- }
- worker.Start(func(workerCtx context.Context) (actors.NoPayload, error) {
- attempt++
- return a.watcher.backfillProjectImageRefsInternal(workerCtx, attempt)
- })
+ return err
}
-func (a *imageUpdateWatcherActorInternal) startScanInternal(scanCtx context.Context, ctx *actors.Context, automatic bool, triggerGeneration uint64, done *actors.Promise[error]) {
- if automatic {
- a.scanSubmitting = true
- } else {
- a.scanRunning = true
- }
- worker := actors.Worker[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]{
- Actor: ctx,
- WorkContext: scanCtx,
- Label: "image update scan worker",
- CompletionKind: imageUpdateScanCompletedMessageInternal,
- PanicValue: imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done},
- ActorStopped: func(_ imageUpdateScanCompletionInternal, err error) {
- if done != nil {
- done.Resolve(err)
- }
- },
- }
- worker.Start(func(workerCtx context.Context) (imageUpdateScanCompletionInternal, error) {
- completion := imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done}
- if automatic {
- a.watcher.dispatchMu.RLock()
- dispatcher := a.watcher.dispatcher
- a.watcher.dispatchMu.RUnlock()
- if dispatcher == nil {
- return completion, errors.New("durable job dispatcher unavailable")
- }
- _, err := dispatcher.Submit(workerCtx, schedulertypes.Request{JobID: a.watcher.Name(), EnvironmentID: "0", Trigger: "watcher"})
- return completion, err
- }
- return completion, a.watcher.executeScanInternal(workerCtx)
- })
-}
-
-func (a *imageUpdateWatcherActorInternal) resetScheduleInternal(ctx *actors.Context) {
- spec := a.watcher.settingsService.GetStringSetting(ctx.Context(), "pollingInterval", imageUpdateWatcherDefaultSchedule)
+func (w *ImageUpdateWatcher) refreshScheduleInternal(ctx context.Context) error {
+ spec := w.settingsService.GetStringSetting(ctx, "pollingInterval", imageUpdateWatcherDefaultSchedule)
schedule, err := cronScheduleParser.Parse(spec)
if err != nil {
- slog.WarnContext(ctx.Context(), "invalid pollingInterval cron expression; using default schedule", "pollingInterval", spec, "default", imageUpdateWatcherDefaultSchedule, "error", err)
- schedule, err = cronScheduleParser.Parse(imageUpdateWatcherDefaultSchedule)
+ slog.WarnContext(ctx, "invalid pollingInterval cron expression; using default schedule", "pollingInterval", spec, "error", err)
+ spec = imageUpdateWatcherDefaultSchedule
+ schedule, err = cronScheduleParser.Parse(spec)
if err != nil {
- slog.ErrorContext(ctx.Context(), "failed to parse default image polling schedule", "error", err)
- return
+ return err
}
}
-
- next := schedule.Next(time.Now().In(a.watcher.location))
- a.watcher.dispatchMu.RLock()
- dispatcher := a.watcher.dispatcher
- a.watcher.dispatchMu.RUnlock()
- if dispatcher != nil {
- if err := dispatcher.Checkpoint(ctx.Context(), a.watcher.Name(), spec, next); err != nil {
- slog.ErrorContext(ctx.Context(), "Failed to checkpoint image polling schedule", "error", err)
- }
+ w.dispatchMu.RLock()
+ coordinator := w.coordinator
+ w.dispatchMu.RUnlock()
+ if coordinator == nil {
+ return errors.New("durable job coordinator unavailable")
}
- a.scheduleTimer.Reset(ctx, imageUpdateScheduledPollMessageInternal, time.Until(next))
+ return coordinator.Checkpoint(ctx, w.Name(), spec, schedule.Next(time.Now().In(w.location)))
}
-// SetDispatcher connects automatic image scans to durable admission.
-func (w *ImageUpdateWatcher) SetDispatcher(dispatcher schedulertypes.Dispatcher) {
+func (w *ImageUpdateWatcher) SetCoordinator(coordinator *runs.Coordinator) {
w.dispatchMu.Lock()
- defer w.dispatchMu.Unlock()
- w.dispatcher = dispatcher
+ w.coordinator = coordinator
+ w.dispatchMu.Unlock()
}
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index d682fce30..c714a630f 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -11,15 +11,14 @@ import (
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/imageupdate"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/moby/moby/api/types/events"
"github.com/stretchr/testify/require"
"go.getarcane.app/streams/bus"
- "go.uber.org/fx/fxtest"
)
type imageUpdateScannerFakeInternal struct {
@@ -189,23 +188,14 @@ func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScann
if backfiller == nil {
backfiller = &projectImageRefsBackfillerFakeInternal{}
}
- lifecycle := fxtest.NewLifecycle(t)
- actorRuntime, err := actors.NewRuntime(t.Context(), lifecycle)
- require.NoError(t, err)
- t.Cleanup(func() {
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- require.NoError(t, lifecycle.Stop(stopCtx))
- })
watcher := &ImageUpdateWatcher{
imageUpdateService: scanner,
settingsService: settings,
environmentService: registryCredentialLoaderFakeInternal{},
dockerService: dockerEventBusProviderFakeInternal{eventBus: eventBus},
projectService: backfiller,
- actorRuntime: actorRuntime,
- triggerIngress: actors.NewIngress[imageUpdateMessageKindInternal, time.Time](imageUpdateTriggerMessageInternal),
- scheduleIngress: actors.NewIngress[imageUpdateMessageKindInternal, actors.NoPayload](imageUpdateScheduleRefreshMessageInternal),
+ trigger: make(chan struct{}, 1),
+ scheduleRefresh: make(chan struct{}, 1),
location: time.UTC,
debounce: 10 * time.Millisecond,
backfillRetry: 10 * time.Millisecond,
@@ -213,7 +203,14 @@ func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScann
started: make(chan struct{}),
stopped: make(chan struct{}),
}
- watcher.SetDispatcher(&testDispatcherInternal{run: func(ctx context.Context, _ schedulertypes.Request) error { return watcher.RunNow(ctx) }})
+ coordinator, _ := newTestCoordinatorInternal(t, t.Context(), time.UTC, func(ctx context.Context, run schedulertypes.Run) (schedulertypes.Outcome, error) {
+ err := watcher.RunNow(jobcontext.WithExecution(ctx, run, nil))
+ if err != nil {
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: err.Error()}, nil
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
+ })
+ watcher.SetCoordinator(coordinator)
return watcher
}
@@ -277,23 +274,18 @@ func TestImageUpdateWatcher_EventTriggersAreOptIn(t *testing.T) {
func TestImageUpdateWatcher_TrailingEdgeDebounceExtendsWithNewTriggers(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
- scanner := &imageUpdateScannerFakeInternal{}
- settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
- watcher.debounce = 50 * time.Millisecond
- startImageUpdateWatcherForTestInternal(t, watcher)
-
- require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, time.Millisecond)
+ watcher := &ImageUpdateWatcher{debounce: 50 * time.Millisecond, trigger: make(chan struct{}, 1)}
+ timer := time.NewTimer(watcher.debounce)
+ defer timer.Stop()
watcher.Trigger()
- synctest.Wait()
time.Sleep(30 * time.Millisecond)
- synctest.Wait()
watcher.Trigger()
- synctest.Wait()
- time.Sleep(30 * time.Millisecond)
- synctest.Wait()
- require.Equal(t, 1, scanner.countInternal())
- require.Eventually(t, func() bool { return scanner.countInternal() == 2 }, time.Second, time.Millisecond)
+ <-timer.C
+ require.False(t, watcher.scanAdmissionReadyInternal(timer, true, false))
+ time.Sleep(10 * time.Millisecond)
+ require.False(t, watcher.scanAdmissionReadyInternal(timer, true, false))
+ <-timer.C
+ require.True(t, watcher.scanAdmissionReadyInternal(timer, true, false))
})
}
@@ -373,7 +365,7 @@ func TestImageUpdateWatcher_RunNowReturnsInProgressErrorDuringActiveScan(t *test
require.Equal(t, 1, scanner.maxActiveInternal())
}
-func TestImageUpdateWatcher_RunNowConsumesHollywoodResponseInternal(t *testing.T) {
+func TestImageUpdateWatcher_RunNowReturnsCompletedResultInternal(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
@@ -393,9 +385,9 @@ func TestImageUpdateWatcher_RunNowReturnsContainedScanPanicInternal(t *testing.T
settings := &pollingSettingReaderFakeInternal{enabled: true}
watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
startImageUpdateWatcherForTestInternal(t, watcher)
- // Wait for the actor to release startup admission, not just for the scanner to start.
+ // Wait for durable startup admission to finish, not just for the scanner to start.
require.Eventually(t, func() bool {
- return scanner.countInternal() == 1 && !watcher.triggerIngress.Pending()
+ return scanner.countInternal() == 1 && watcher.triggerGeneration.Load() == watcher.acknowledgedGeneration.Load()
}, time.Second, time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
@@ -513,39 +505,37 @@ func TestImageUpdateWatcher_BackfillRetriesContainedPanicInternal(t *testing.T)
}
func TestImageUpdateWatcher_BackfillFailureRetriesBeforeScanning(t *testing.T) {
- synctest.Test(t, func(t *testing.T) {
- secondAttemptStarted := make(chan struct{})
- releaseSecondAttempt := make(chan struct{})
- backfiller := &projectImageRefsBackfillerFakeInternal{
- run: func(ctx context.Context, call int) (int, error) {
- if call == 1 {
- return 0, fmt.Errorf("database statement timeout: %w", context.DeadlineExceeded)
- }
- close(secondAttemptStarted)
- select {
- case <-ctx.Done():
- return 0, ctx.Err()
- case <-releaseSecondAttempt:
- return 42, nil
- }
- },
- }
- scanner := &imageUpdateScannerFakeInternal{}
- settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
- startImageUpdateWatcherForTestInternal(t, watcher)
+ secondAttemptStarted := make(chan struct{})
+ releaseSecondAttempt := make(chan struct{})
+ backfiller := &projectImageRefsBackfillerFakeInternal{
+ run: func(ctx context.Context, call int) (int, error) {
+ if call == 1 {
+ return 0, fmt.Errorf("database statement timeout: %w", context.DeadlineExceeded)
+ }
+ close(secondAttemptStarted)
+ select {
+ case <-ctx.Done():
+ return 0, ctx.Err()
+ case <-releaseSecondAttempt:
+ return 42, nil
+ }
+ },
+ }
+ scanner := &imageUpdateScannerFakeInternal{}
+ settings := &pollingSettingReaderFakeInternal{enabled: true}
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), backfiller)
+ startImageUpdateWatcherForTestInternal(t, watcher)
- select {
- case <-secondAttemptStarted:
- case <-time.After(time.Second):
- require.FailNow(t, "backfill was not retried")
- }
- require.Zero(t, scanner.countInternal())
- close(releaseSecondAttempt)
+ select {
+ case <-secondAttemptStarted:
+ case <-time.After(time.Second):
+ require.FailNow(t, "backfill was not retried")
+ }
+ require.Zero(t, scanner.countInternal())
+ close(releaseSecondAttempt)
- require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
- require.Equal(t, 2, backfiller.countInternal())
- })
+ require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
+ require.Equal(t, 2, backfiller.countInternal())
}
func TestImageUpdateWatcher_CancellationStopsBackfillWithoutScanning(t *testing.T) {
@@ -585,7 +575,7 @@ func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T)
require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, 3*time.Second, 10*time.Millisecond)
}
-func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsActorRunning(t *testing.T) {
+func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsWatcherRunning(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true}
eventBus := bus.NewDockerEventBus()
@@ -597,36 +587,25 @@ func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsActorRunning(t *testing.
watcher.Trigger()
require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, time.Second, 5*time.Millisecond)
- require.NotNil(t, watcher.actorProcess.Load())
+ require.True(t, watcher.running.Load())
}
-func TestImageUpdateWatcher_RunnerRestartsAfterUnexpectedActorExit(t *testing.T) {
+func TestImageUpdateWatcher_SupervisorRestartsAfterCancellationInternal(t *testing.T) {
scanner := &imageUpdateScannerFakeInternal{}
- settings := &pollingSettingReaderFakeInternal{enabled: true}
- watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
- runner, err := actors.NewRunner(
- t.Context(),
- watcher.actorRuntime,
- "image-update-watcher-test",
- "supervision",
- "supervised image update watcher",
- 3,
- watcher.Start,
- )
- require.NoError(t, err)
-
- require.Eventually(t, func() bool { return watcher.actorProcess.Load() != nil }, time.Second, time.Millisecond)
- firstProcess := watcher.actorProcess.Load()
- require.NoError(t, firstProcess.Stop(t.Context()))
- require.Eventually(t, func() bool {
- current := watcher.actorProcess.Load()
- return current != nil && current != firstProcess
- }, time.Second, time.Millisecond)
-
- stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ watcher := newImageUpdateWatcherForTestInternal(t, scanner, &pollingSettingReaderFakeInternal{enabled: true}, bus.NewDockerEventBus(), nil)
+ supervisor := newWatcherSupervisorInternal(watcher)
+ ctx, cancel := context.WithCancel(t.Context())
defer cancel()
- require.NoError(t, runner.Stop(stopCtx))
- require.NoError(t, watcher.Stop(stopCtx))
+ done := make(chan error, 1)
+ go func() { done <- supervisor.runInternal(ctx) }()
+ require.Eventually(t, func() bool { return watcher.running.Load() }, time.Second, time.Millisecond)
+ supervisor.mu.Lock()
+ supervisor.restart <- struct{}{}
+ supervisor.cancel()
+ supervisor.mu.Unlock()
+ require.Eventually(t, func() bool { return scanner.countInternal() > 0 }, time.Second, time.Millisecond)
+ cancel()
+ require.ErrorIs(t, <-done, context.Canceled)
}
func TestImageUpdateWatcher_RunNowWaitsForMetadataReadiness(t *testing.T) {
diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.go
index 3534c5032..adc4a5695 100644
--- a/backend/pkg/scheduler/job_schedule_integration_test.go
+++ b/backend/pkg/scheduler/job_schedule_integration_test.go
@@ -26,13 +26,13 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background())
jobScheduler := newJobSchedulerForTestInternal(t, lifecycleCtx, appConfig.GetLocation())
- jobService := job.NewJobService(db, settingsService, appConfig)
+ jobService := job.NewJobService(db, settingsService, appConfig, jobScheduler.coordinator, nil, nil, nil)
jobService.SetScheduler(lifecycleCtx, jobScheduler)
require.NoError(t, jobScheduler.StartScheduler())
_, ok := jobScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
- require.Empty(t, jobScheduler.cron.Entries())
+ require.Empty(t, jobScheduler.ListRegisteredJobs())
requestedSchedule := "0 0 8 * * *"
updatedSchedules, err := jobService.UpdateJobSchedules(ctx, jobschedule.Update{
@@ -48,7 +48,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
_, ok = jobScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
- require.Empty(t, jobScheduler.cron.Entries())
+ require.Empty(t, jobScheduler.ListRegisteredJobs())
jobs, err := jobService.ListJobs(ctx)
require.NoError(t, err)
@@ -65,7 +65,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
restartDB, restartSettingsService := openJobScheduleTestDatabaseInternal(t, ctx, databasePath)
restartLifecycleCtx, cancelRestartLifecycle := context.WithCancel(context.Background())
restartScheduler := newJobSchedulerForTestInternal(t, restartLifecycleCtx, appConfig.GetLocation())
- restartJobService := job.NewJobService(restartDB, restartSettingsService, appConfig)
+ restartJobService := job.NewJobService(restartDB, restartSettingsService, appConfig, restartScheduler.coordinator, nil, nil, nil)
restartJobService.SetScheduler(restartLifecycleCtx, restartScheduler)
require.NoError(t, restartScheduler.StartScheduler())
t.Cleanup(func() {
@@ -76,7 +76,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
_, ok = restartScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
- require.Empty(t, restartScheduler.cron.Entries())
+ require.Empty(t, restartScheduler.ListRegisteredJobs())
require.Equal(t, requestedSchedule, restartSettingsService.GetStringSetting(ctx, "pollingInterval", ""))
restartedJobs, err := restartJobService.ListJobs(ctx)
@@ -90,7 +90,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
_, ok = restartScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
- require.Empty(t, restartScheduler.cron.Entries())
+ require.Empty(t, restartScheduler.ListRegisteredJobs())
disabledJobs, err := restartJobService.ListJobs(ctx)
require.NoError(t, err)
@@ -124,7 +124,7 @@ func closeJobScheduleTestDatabaseInternal(t *testing.T, db *database.DB) {
}
func waitForSchedulerStopInternal(jobScheduler *jobSchedulerInternal) {
- <-jobScheduler.cron.Stop().Done()
+ _ = stopJobSchedulerForTestInternal(context.Background(), jobScheduler)
}
func findJobStatusInternal(t *testing.T, jobs *jobschedule.JobListResponse, jobID string) jobschedule.JobStatus {
diff --git a/backend/pkg/scheduler/queue/activity.go b/backend/pkg/scheduler/queue/activity.go
deleted file mode 100644
index 25a25e523..000000000
--- a/backend/pkg/scheduler/queue/activity.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package queue
-
-import (
- "context"
- "log/slog"
- "time"
-
- "github.com/getarcaneapp/arcane/types/v2/scheduler"
- kit "go.getarcane.app/kit/pkg"
-)
-
-// SetObserver connects activity projection before the queue starts accepting work.
-func (q *Queue) SetObserver(observer scheduler.RunObserver) {
- q.observer = observer
-}
-
-func (q *Queue) associateActivityInternal(run *scheduler.Run) {
- if q.observer != nil && run.ActivityID == "" {
- run.ActivityID = q.observer.ActivityID(*run)
- }
- if run.ActivityID != "" {
- run.ActivityEnvironmentID = run.EnvironmentID
- }
-}
-
-func (q *Queue) observeRunInternal(ctx context.Context, run scheduler.Run) {
- if q.observer == nil {
- return
- }
- // Serialize repair bookkeeping with notifications. The observer rereads the
- // durable run, so delayed notifications cannot restore an older activity state.
- q.observerMu.Lock()
- defer q.observerMu.Unlock()
- q.syncActivityInternal(ctx, run)
- if len(q.pendingSync) > 0 {
- q.signalInternal()
- }
-}
-
-func (q *Queue) syncActivityInternal(ctx context.Context, run scheduler.Run) {
- ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
- defer cancel()
- key := queuePrefixInternal + kit.SHA256Hex(run.EnvironmentID+"\x00"+run.JobID) + "/" + run.ID
- if err := q.observer.SyncRunActivity(ctx, run); err != nil {
- if q.pendingSync == nil {
- q.pendingSync = make(map[string]scheduler.Run)
- }
- q.pendingSync[key] = run
- slog.ErrorContext(ctx, "Job activity synchronization failed; will retry", "runId", run.ID, "error", err)
- return
- }
- delete(q.pendingSync, key)
-}
-
-func (q *Queue) retryActivitySyncInternal(ctx context.Context) {
- q.observerMu.Lock()
- defer q.observerMu.Unlock()
- for _, run := range q.pendingSync {
- if ctx.Err() != nil {
- return
- }
- q.syncActivityInternal(ctx, run)
- }
-}
-
-func (q *Queue) hasPendingActivitySyncInternal() bool {
- q.observerMu.Lock()
- defer q.observerMu.Unlock()
- return len(q.pendingSync) > 0
-}
diff --git a/backend/pkg/scheduler/queue/queue_store.go b/backend/pkg/scheduler/queue/queue_store.go
deleted file mode 100644
index 71e648fa7..000000000
--- a/backend/pkg/scheduler/queue/queue_store.go
+++ /dev/null
@@ -1,217 +0,0 @@
-package queue
-
-import (
- "context"
- "encoding/json/v2"
- "slices"
- "sort"
- "time"
-
- "emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
- st "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/google/uuid"
- kit "go.getarcane.app/kit/pkg"
-)
-
-const queuePrefixInternal = "jobs."
-
-func (q *Queue) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {
- key := queuePrefixInternal + kit.SHA256Hex(environmentID+"\x00"+jobID)
- for range 20 {
- previous, found, err := q.store.Get(ctx, key)
- if err != nil {
- return err
- }
- record := st.QueueRecord{JobID: jobID, EnvironmentID: environmentID}
- if found {
- if err := json.Unmarshal([]byte(previous), &record); err != nil {
- return errors.WrapIf(err, "decode job queue")
- }
- }
- if err := change(&record); err != nil {
- return err
- }
- for index := range record.Runs {
- q.associateActivityInternal(&record.Runs[index])
- }
- next, err := json.Marshal(record)
- if err != nil {
- return err
- }
- if found && string(next) == previous {
- return nil
- }
- var swapped bool
- if found {
- swapped, err = q.store.CompareAndSwap(ctx, key, previous, string(next))
- } else {
- swapped, err = q.store.CreateIfAbsent(ctx, key, string(next))
- }
- if err != nil {
- return err
- }
- if swapped {
- return nil
- }
- }
- return errors.New("job queue contention; retry request with the same run ID")
-}
-
-// Records reads all persisted queue records, including run history and receipts.
-// Storage or decoding errors fail the read instead of returning an incomplete snapshot.
-func (q *Queue) Records(ctx context.Context) ([]st.QueueRecord, error) {
- entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)
- if err != nil {
- return nil, err
- }
- records := make([]st.QueueRecord, 0, len(entries))
- for _, entry := range entries {
- var record st.QueueRecord
- if err := json.Unmarshal([]byte(entry.Value), &record); err != nil {
- return nil, errors.WrapIff(err, "decode job queue %s", entry.Key)
- }
- for index := range record.Runs {
- q.associateActivityInternal(&record.Runs[index])
- }
- records = append(records, record)
- }
- return records, nil
-}
-
-// Get retrieves a run by environment, job, and run ID, including compacted
-// idempotency receipts. It returns ErrRunNotFound when no matching record exists.
-func (q *Queue) Get(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
- raw, found, err := q.store.Get(ctx, queuePrefixInternal+kit.SHA256Hex(environmentID+"\x00"+jobID))
- if err != nil {
- return st.Run{}, err
- }
- if !found {
- return st.Run{}, ErrRunNotFound
- }
- var record st.QueueRecord
- if err := json.Unmarshal([]byte(raw), &record); err != nil {
- return st.Run{}, err
- }
- for _, run := range record.Runs {
- if run.ID == runID {
- q.associateActivityInternal(&run)
- return run, nil
- }
- }
- if run, ok := record.Receipts[runID]; ok {
- q.associateActivityInternal(&run)
- return run, nil
- }
- return st.Run{}, ErrRunNotFound
-}
-
-// List returns retained run history newest first, excluding compacted receipts.
-// An empty job ID includes every job in the environment; pages are one-based.
-func (q *Queue) List(ctx context.Context, environmentID, jobID string, page, limit int) (st.RunList, error) {
- records, err := q.Records(ctx)
- if err != nil {
- return st.RunList{}, err
- }
- runs := []st.Run{}
- for _, record := range records {
- if record.EnvironmentID == environmentID && (jobID == "" || record.JobID == jobID) {
- runs = append(runs, record.Runs...)
- }
- }
- sort.Slice(runs, func(i, j int) bool { return runs[i].CreatedAt.After(runs[j].CreatedAt) })
- page = max(1, page)
- limit = min(100, max(1, limit))
- start := len(runs)
- if page-1 <= len(runs)/limit {
- start = min(len(runs), (page-1)*limit)
- }
- end := min(len(runs), start+limit)
- return st.RunList{Runs: runs[start:end], Total: len(runs), Page: page, Limit: limit}, nil
-}
-
-// UpdateRun atomically changes a persisted run or terminal receipt using compare
-// and swap. The callback may run more than once and must not perform external side effects.
-func (q *Queue) UpdateRun(ctx context.Context, run st.Run, change func(*st.Run) error) error {
- wake := false
- err := q.mutateInternal(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {
- for index := range record.Runs {
- if record.Runs[index].ID == run.ID {
- previous := record.Runs[index].Status
- if err := change(&record.Runs[index]); err != nil {
- return err
- }
- wake = previous != st.Queued && record.Runs[index].Status == st.Queued
- return nil
- }
- }
- if receipt, found := record.Receipts[run.ID]; found {
- if err := change(&receipt); err != nil {
- return err
- }
- if !receipt.Status.Terminal() {
- return errors.New("archived runs cannot be restarted")
- }
- record.Receipts[run.ID] = receipt
- return nil
- }
- return ErrRunNotFound
- })
- if err == nil && wake {
- q.signalInternal()
- }
- if err == nil {
- q.observeRunInternal(ctx, run)
- }
- return err
-}
-
-func (q *Queue) pruneInternal(ctx context.Context) error {
- records, err := q.Records(ctx)
- if err != nil {
- return err
- }
- now := time.Now().UTC()
- for _, record := range records {
- if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
- pruneRunsInternal(current, now)
- return nil
- }); err != nil {
- return err
- }
- }
- return nil
-}
-
-func pruneRunsInternal(record *st.QueueRecord, now time.Time) {
- terminal := 0
- for index, run := range slices.Backward(record.Runs) {
- if !run.Status.Terminal() || (run.RemoteAccepted && !run.RemoteSettled) {
- continue
- }
- terminal++
- if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {
- continue
- }
- if run.Trigger == "manual" || run.Trigger == "remote" {
- if record.Receipts == nil {
- record.Receipts = make(map[string]st.Run)
- }
- run.Attempts = nil
- run.Outcome.Targets = nil
- record.Receipts[run.ID] = run
- }
- record.Runs = append(record.Runs[:index], record.Runs[index+1:]...)
- }
-}
-
-// ErrRunNotFound indicates that neither a run nor its receipt exists.
-var ErrRunNotFound = errors.Sentinel("job run not found")
-
-// ErrRunConflict indicates that execution ownership or state changed.
-var ErrRunConflict = errors.Sentinel("job run state changed")
-
-// New uses the application's existing KV store; no in-memory admission exists.
-func New(store *kv.KVService, execute, reconcile func(context.Context, st.Run) (st.Outcome, error)) *Queue {
- return &Queue{execute: execute, reconcile: reconcile, store: store, owner: uuid.NewString(), active: make(map[string]bool), checkpointed: make(map[string]bool), wake: make(chan struct{}, 1), done: make(chan struct{})}
-}
diff --git a/backend/pkg/scheduler/runs/activity.go b/backend/pkg/scheduler/runs/activity.go
new file mode 100644
index 000000000..94a99c615
--- /dev/null
+++ b/backend/pkg/scheduler/runs/activity.go
@@ -0,0 +1,112 @@
+package runs
+
+import (
+ "context"
+ "log/slog"
+ "time"
+
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ kit "go.getarcane.app/kit/pkg"
+)
+
+// SetObserver connects activity projection before accepting work.
+func (q *Coordinator) SetObserver(observer scheduler.RunObserver) { q.observer = observer }
+
+func (q *Coordinator) associateActivityInternal(run *scheduler.Run) {
+ if q.observer != nil && run.ActivityID == "" {
+ run.ActivityID = q.observer.ActivityID(*run)
+ }
+ if run.ActivityID != "" {
+ run.ActivityEnvironmentID = run.EnvironmentID
+ }
+}
+
+func (q *Coordinator) observeRunInternal(ctx context.Context, run scheduler.Run) {
+ if q.observer == nil {
+ return
+ }
+ q.observerMu.Lock()
+ defer q.observerMu.Unlock()
+ q.syncActivityInternal(ctx, run)
+ q.signalInternal()
+}
+
+func (q *Coordinator) syncActivityInternal(ctx context.Context, run scheduler.Run) {
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ var state scheduler.CoordinatorState
+ id := kit.SHA256Hex(run.EnvironmentID + "\x00" + run.JobID)
+ if err := q.service.GetState(ctx, coordinatorTypeInternal, id, &state); err != nil {
+ return
+ }
+ version, pending := state.PendingActivitySync[run.ID]
+ if !pending {
+ return
+ }
+ current, err := q.Get(ctx, run.EnvironmentID, run.JobID, run.ID)
+ if err == nil {
+ err = q.observer.SyncRunActivity(ctx, current)
+ }
+ if err == nil {
+ _, err = q.service.Invoke(ctx, coordinatorTypeInternal, id, "activity-synced", scheduler.ActivitySyncCommand{RunID: run.ID, UpdatedAt: version})
+ }
+ if err != nil {
+ slog.ErrorContext(ctx, "Job activity synchronization failed; will retry", "runId", run.ID, "error", err)
+ }
+}
+
+func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {
+ if q.observer == nil {
+ return
+ }
+ q.observerMu.Lock()
+ defer q.observerMu.Unlock()
+ cursor := ""
+ for {
+ page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ if err != nil {
+ slog.ErrorContext(ctx, "List pending job activities", "error", err)
+ return
+ }
+ for _, entry := range page.States {
+ var state scheduler.CoordinatorState
+ if err := entry.Data.Decode(&state); err != nil {
+ slog.ErrorContext(ctx, "Decode pending job activities", "error", err)
+ continue
+ }
+ for runID := range state.PendingActivitySync {
+ if ctx.Err() != nil {
+ return
+ }
+ q.syncActivityInternal(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})
+ }
+ }
+ cursor = page.AfterID()
+ if cursor == "" {
+ return
+ }
+ }
+}
+
+func (q *Coordinator) hasPendingActivitySyncInternal(ctx context.Context) bool {
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ cursor := ""
+ for {
+ page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ if err != nil {
+ return true
+ }
+ for _, entry := range page.States {
+ var state scheduler.CoordinatorState
+ if entry.Data.Decode(&state) != nil || len(state.PendingActivitySync) > 0 {
+ return true
+ }
+ }
+ cursor = page.AfterID()
+ if cursor == "" {
+ return false
+ }
+ }
+}
diff --git a/backend/pkg/scheduler/queue/activity_test.go b/backend/pkg/scheduler/runs/activity_test.go
similarity index 90%
rename from backend/pkg/scheduler/queue/activity_test.go
rename to backend/pkg/scheduler/runs/activity_test.go
index e9e94b641..6f80597b3 100644
--- a/backend/pkg/scheduler/queue/activity_test.go
+++ b/backend/pkg/scheduler/runs/activity_test.go
@@ -1,4 +1,4 @@
-package queue
+package runs
import (
"context"
@@ -40,6 +40,7 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {
require.NoError(t, err)
require.Equal(t, run.ID, run.ActivityID)
require.NoError(t, q.Start(t.Context()))
+ q.Activate()
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
@@ -48,11 +49,11 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {
require.Eventually(t, func() bool {
persisted, getErr := q.Get(t.Context(), "0", run.JobID, run.ID)
return getErr == nil && persisted.Status == st.Succeeded
- }, time.Second, time.Millisecond)
- require.True(t, q.hasPendingActivitySyncInternal())
+ }, 5*time.Second, 10*time.Millisecond)
+ require.True(t, q.hasPendingActivitySyncInternal(t.Context()))
observer.unavailable.Store(false)
q.retryActivitySyncInternal(t.Context())
- require.False(t, q.hasPendingActivitySyncInternal())
+ require.False(t, q.hasPendingActivitySyncInternal(t.Context()))
require.Positive(t, observer.observed.Load())
duplicate, err := q.Submit(t.Context(), st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "manual"})
require.NoError(t, err)
diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.go
new file mode 100644
index 000000000..1f4f95ef2
--- /dev/null
+++ b/backend/pkg/scheduler/runs/actors.go
@@ -0,0 +1,498 @@
+package runs
+
+import (
+ "context"
+ "encoding/json/v2"
+ "fmt"
+ "log/slog"
+ "strings"
+ "time"
+ "uuid"
+
+ "emperror.dev/errors"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/host/local"
+ "github.com/robfig/cron/v3"
+ kit "go.getarcane.app/kit/pkg"
+)
+
+const (
+ coordinatorTypeInternal = "job-coordinator"
+ executorTypeInternal = "job-executor"
+ healthExecutorTypeInternal = "job-health-executor"
+)
+
+type coordinatorActorInternal struct {
+ id string
+ q *Coordinator
+}
+type executorActorInternal struct {
+ q *Coordinator
+}
+
+func (q *Coordinator) Register(runtime *francis.Runtime) error {
+ if err := runtime.RegisterActor(coordinatorTypeInternal, func(id string, _ *actor.Service) actor.Actor {
+ return &coordinatorActorInternal{id: id, q: q}
+ }); err != nil {
+ return err
+ }
+ executorFactory := func(_ string, _ *actor.Service) actor.Actor {
+ return &executorActorInternal{q: q}
+ }
+ if err := runtime.RegisterActor(
+ executorTypeInternal, executorFactory,
+ local.WithCapacityGroup("jobs", 4),
+ local.WithCompletedJobRetention(7*24*time.Hour),
+ ); err != nil {
+ return err
+ }
+ return runtime.RegisterActor(
+ healthExecutorTypeInternal, executorFactory,
+ local.WithCapacityGroup("jobs-health", 2),
+ local.WithCompletedJobRetention(7*24*time.Hour),
+ )
+}
+
+func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {
+ var state st.CoordinatorState
+ if err := a.q.service.GetState(ctx, coordinatorTypeInternal, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return nil, err
+ }
+ normalizeRecordTimesInternal(&state.Record)
+ switch method {
+ case "replace":
+ var next st.CoordinatorState
+ if err := data.Decode(&next); err != nil {
+ return nil, err
+ }
+ if next.Revision != state.Revision {
+ return false, nil
+ }
+ next.Revision++
+ if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, next, nil); err != nil {
+ return nil, err
+ }
+ // Acceptance is persisted; dispatch failures are retried.
+ if err := a.flushInternal(ctx, &next); err != nil {
+ slog.WarnContext(ctx, "Job dispatch requires reconciliation", "actorId", a.id, "error", err)
+ a.q.signalInternal()
+ }
+ return true, nil
+ case "activity-synced":
+ var command st.ActivitySyncCommand
+ if err := data.Decode(&command); err != nil {
+ return nil, err
+ }
+ version, pending := state.PendingActivitySync[command.RunID]
+ if !pending || !version.Equal(command.UpdatedAt) {
+ return nil, nil
+ }
+ delete(state.PendingActivitySync, command.RunID)
+ state.Revision++
+ return nil, a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil)
+ case "repair":
+ return nil, a.flushInternal(ctx, &state)
+ case "import":
+ var command st.CoordinatorImport
+ if err := data.Decode(&command); err != nil {
+ return nil, err
+ }
+ if state.Imported {
+ if state.ImportHash != command.SourceHash {
+ return nil, errors.New("legacy job record changed during import")
+ }
+ return true, nil
+ }
+ if state.Revision != 0 {
+ return nil, errors.New("native job state exists before legacy import")
+ }
+ state.ImportHash = command.SourceHash
+ state.Record = command.Record
+ for _, run := range state.Record.Runs {
+ markActivitySyncInternal(&state, run)
+ }
+ state.Imported = true
+ state.Revision = 1
+ return true, a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil)
+ default:
+ return nil, errors.Errorf("unknown coordinator command: %s", method)
+ }
+}
+
+func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.CoordinatorState) error {
+ if !a.q.enabled.Load() {
+ return nil
+ }
+ record := &state.Record
+ if record.Schedule != "" && !record.NextRun.IsZero() {
+ name := fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)
+ properties := actor.AlarmProperties{
+ DueTime: record.NextRun,
+ Data: st.ScheduleOccurrence{Generation: record.Generation, Sequence: record.Sequence},
+ }
+ if err := a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, properties); err != nil {
+ return err
+ }
+ }
+ var selected *st.Run
+ for i := range record.Runs {
+ if record.Runs[i].Status == st.Running {
+ selected = &record.Runs[i]
+ break
+ }
+ }
+ var retryAt time.Time
+ if selected == nil {
+ selected, retryAt = a.q.nextRunInternal(record.Runs)
+ }
+ if selected == nil {
+ if retryAt.IsZero() {
+ return nil
+ }
+ name := fmt.Sprintf("retry-%d", retryAt.UnixNano())
+ return a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, actor.AlarmProperties{DueTime: retryAt})
+ }
+ if selected.Status == st.Running {
+ active, err := a.reconcileDispatchInternal(ctx, state, *selected)
+ if err != nil {
+ return err
+ }
+ if active {
+ return nil
+ }
+ }
+ return a.dispatchRunInternal(ctx, state, *selected, selected.AttemptCount+1)
+}
+
+// reconcileDispatchInternal recovers a lost job ID before considering redelivery.
+func (a *coordinatorActorInternal) reconcileDispatchInternal(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {
+ key := fmt.Sprintf("%s-%d", run.ID, run.AttemptCount)
+ previous, exists := state.Dispatches[key]
+ if !exists {
+ return false, nil
+ }
+ if previous.JobID == "" {
+ command := st.ExecutionCommand{
+ EnvironmentID: run.EnvironmentID,
+ JobID: run.JobID,
+ RunID: run.ID,
+ Attempt: run.AttemptCount,
+ }
+ actorType := executorTypeInternal
+ if strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep" {
+ actorType = healthExecutorTypeInternal
+ }
+ id, _, err := a.q.service.Dispatch(ctx, actorType, a.id, "execute", command, actor.WithIdempotencyKey(key))
+ if err != nil {
+ return false, err
+ }
+ previous.JobID = id
+ state.Dispatches[key] = previous
+ state.Revision++
+ if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil); err != nil {
+ return false, err
+ }
+ }
+ job, err := a.q.service.GetJob(ctx, previous.JobID)
+ if errors.Is(err, actor.ErrJobNotFound) {
+ return false, nil
+ }
+ if err != nil {
+ return false, err
+ }
+ return !job.Status.IsTerminal(), nil
+}
+
+func (a *coordinatorActorInternal) dispatchRunInternal(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {
+ key := fmt.Sprintf("%s-%d", run.ID, attempt)
+ if state.Dispatches == nil {
+ state.Dispatches = make(map[string]st.DispatchIntent)
+ }
+ actorType := executorTypeInternal
+ if strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep" {
+ actorType = healthExecutorTypeInternal
+ }
+ intent := state.Dispatches[key]
+ if intent.JobID != "" {
+ job, err := a.q.service.GetJob(ctx, intent.JobID)
+ if err == nil && !job.Status.IsTerminal() {
+ return nil
+ }
+ if err != nil && !errors.Is(err, actor.ErrJobNotFound) {
+ return err
+ }
+ if err == nil {
+ if err := a.q.service.DeleteJob(ctx, actorType, a.id, intent.JobID); err != nil && !errors.Is(err, actor.ErrJobNotFound) {
+ return err
+ }
+ }
+ }
+ intent = st.DispatchIntent{RunID: run.ID, Attempt: attempt}
+ state.Dispatches[key] = intent
+ state.Revision++
+ if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil); err != nil {
+ return err
+ }
+ options := []actor.JobOption{actor.WithIdempotencyKey(key)}
+ if run.NextAttempt != nil && run.NextAttempt.After(time.Now()) {
+ options = append(options, actor.WithJobDueTime(*run.NextAttempt))
+ }
+ command := st.ExecutionCommand{
+ EnvironmentID: run.EnvironmentID,
+ JobID: run.JobID,
+ RunID: run.ID,
+ Attempt: attempt,
+ }
+ id, _, err := a.q.service.Dispatch(ctx, actorType, a.id, "execute", command, options...)
+ if err != nil {
+ return err
+ }
+ job, err := a.q.service.GetJob(ctx, id)
+ if err != nil {
+ return err
+ }
+ if job.Status.IsTerminal() {
+ if err := a.q.service.DeleteJob(ctx, actorType, a.id, id); err != nil && !errors.Is(err, actor.ErrJobNotFound) {
+ return err
+ }
+ id, _, err = a.q.service.Dispatch(ctx, actorType, a.id, "execute", command, options...)
+ if err != nil {
+ return err
+ }
+ }
+ intent.JobID = id
+ state.Dispatches[key] = intent
+ state.Revision++
+ return a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil)
+}
+
+func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data actor.Envelope) error {
+ if !a.q.enabled.Load() {
+ return nil
+ }
+ var state st.CoordinatorState
+ if err := a.q.service.GetState(ctx, coordinatorTypeInternal, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return err
+ }
+ normalizeRecordTimesInternal(&state.Record)
+ if strings.HasPrefix(name, "retry-") {
+ return a.flushInternal(ctx, &state)
+ }
+ var occurrence st.ScheduleOccurrence
+ if err := data.Decode(&occurrence); err != nil {
+ return err
+ }
+ record := &state.Record
+ if record.Schedule == "" {
+ return nil
+ }
+ if occurrence.Generation != record.Generation || occurrence.Sequence != record.Sequence {
+ return nil
+ }
+ now := time.Now().UTC()
+ schedule, err := scheduleutil.Parser().Parse(record.Schedule)
+ if err != nil {
+ return err
+ }
+ if spec, ok := schedule.(*cron.SpecSchedule); ok {
+ spec.Location = a.q.location
+ }
+ trigger := "scheduled"
+ nextOccurrence := schedule.Next(record.NextRun.In(a.q.location))
+ if !nextOccurrence.After(now) {
+ trigger = "recovery"
+ }
+ pending := false
+ for _, run := range record.Runs {
+ if run.Trigger == "manual" || run.Trigger == "remote" {
+ continue
+ }
+ if run.Status.Terminal() || run.Status == st.NeedsAttention {
+ continue
+ }
+ pending = true
+ break
+ }
+ if !pending {
+ run := st.Run{
+ ID: uuid.New().String(),
+ EnvironmentID: record.EnvironmentID,
+ JobID: record.JobID,
+ Trigger: trigger,
+ Status: st.Queued,
+ CreatedAt: now,
+ UpdatedAt: now,
+ }
+ a.q.associateActivityInternal(&run)
+ record.Runs = append(record.Runs, run)
+ markActivitySyncInternal(&state, run)
+ }
+ record.LastEnqueuedAt = now
+ record.NextRun = schedule.Next(now.In(a.q.location))
+ record.Sequence++
+ state.Revision++
+ if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil); err != nil {
+ return err
+ }
+ return a.flushInternal(ctx, &state)
+}
+
+func (q *Coordinator) repairInternal(ctx context.Context) error {
+ records, err := q.Records(ctx)
+ if err != nil {
+ return err
+ }
+ for _, record := range records {
+ if _, err := q.service.Invoke(ctx, coordinatorTypeInternal, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (q *Coordinator) importLegacyInternal(ctx context.Context) error {
+ entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)
+ if err != nil {
+ return err
+ }
+ for _, entry := range entries {
+ marker := "francis-import/" + entry.Key
+ _, done, err := q.store.Get(ctx, marker)
+ if err != nil {
+ return err
+ }
+ if done {
+ continue
+ }
+ var record st.QueueRecord
+ if err := json.Unmarshal([]byte(entry.Value), &record); err != nil {
+ return errors.WrapIf(err, "decode legacy job record")
+ }
+ id := kit.SHA256Hex(record.EnvironmentID + "\x00" + record.JobID)
+ if entry.Key != queuePrefixInternal+id {
+ return errors.New("invalid legacy job key")
+ }
+ sourceHash := kit.SHA256Hex(entry.Value)
+ command := st.CoordinatorImport{Record: record, SourceHash: sourceHash}
+ if _, err := q.service.Invoke(ctx, coordinatorTypeInternal, id, "import", command); err != nil {
+ return err
+ }
+ var imported st.CoordinatorState
+ if err := q.service.GetState(ctx, coordinatorTypeInternal, id, &imported); err != nil {
+ return err
+ }
+ if !imported.Imported || imported.ImportHash != sourceHash {
+ return errors.New("legacy import verification failed")
+ }
+ if imported.Record.EnvironmentID != record.EnvironmentID || imported.Record.JobID != record.JobID {
+ return errors.New("legacy import identity mismatch")
+ }
+ if err := q.store.Set(ctx, marker, "complete"); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Envelope) error {
+ q := a.q
+ q.mu.Lock()
+ lifetime := q.lifecycleCtx
+ q.mu.Unlock()
+ if lifetime != nil {
+ workCtx, cancel := context.WithCancel(ctx)
+ stop := context.AfterFunc(lifetime, cancel) //nolint:contextcheck // Application shutdown cancels work before the host drains its jobs.
+ defer stop()
+ defer cancel()
+ ctx = workCtx
+ if lifetime.Err() != nil {
+ cancel()
+ }
+ }
+ if !q.enabled.Load() {
+ return actor.ErrJobRejected
+ }
+ var command st.ExecutionCommand
+ if err := data.Decode(&command); err != nil {
+ return errors.Combine(err, actor.ErrJobPermanentFailure)
+ }
+ claim, err := q.claimExecutionInternal(ctx, command)
+ if err != nil {
+ return err
+ }
+ if claim.Deferred {
+ return actor.ErrJobRejected
+ }
+ if !claim.Claimed {
+ return nil
+ }
+ outcome, runErr := q.invokeInternal(ctx, claim.Run, claim.Reconcile)
+ if outcome.Status == "" {
+ if runErr != nil {
+ outcome.Status = st.Failed
+ } else {
+ outcome.Status = st.Succeeded
+ }
+ }
+ if runErr != nil && outcome.Message == "" {
+ outcome.Message = runErr.Error()
+ }
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ q.persistOutcomeInternal(ctx, claim.Run, outcome)
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ if outcome.Status == st.NeedsAttention {
+ return actor.ErrJobPermanentFailure
+ }
+ return nil
+}
+
+func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.ExecutionCommand) (st.ExecutionClaim, error) {
+ var claim st.ExecutionClaim
+ err := q.mutateInternal(ctx, command.EnvironmentID, command.JobID, func(record *st.QueueRecord) error {
+ claim = st.ExecutionClaim{}
+ for i := range record.Runs {
+ current := &record.Runs[i]
+ if current.ID != command.RunID {
+ continue
+ }
+ if current.Status.Terminal() || current.Status == st.NeedsAttention {
+ return nil
+ }
+ if current.AttemptCount > command.Attempt {
+ return nil
+ }
+ if current.AttemptCount == command.Attempt && current.Status != st.Running {
+ return nil
+ }
+ // Executor turns serialize attempts for this coordinator.
+ now := time.Now().UTC()
+ if current.NextAttempt != nil && current.NextAttempt.After(now) {
+ claim.Deferred = true
+ return nil
+ }
+ claim.Reconcile = current.Status == st.Running
+ current.Status = st.Running
+ current.Owner = q.owner + "/" + uuid.New().String()
+ current.UpdatedAt = now
+ current.StartedAt = &now
+ current.NextAttempt = nil
+ current.AttemptCount = command.Attempt
+ current.Attempts = append(current.Attempts, st.Attempt{Number: current.AttemptCount, StartedAt: now})
+ if len(current.Attempts) > 100 {
+ current.Attempts = current.Attempts[len(current.Attempts)-100:]
+ }
+ claim.Run = *current
+ claim.Claimed = true
+ return nil
+ }
+ return nil
+ })
+ return claim, err
+}
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
new file mode 100644
index 000000000..f3aea251b
--- /dev/null
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -0,0 +1,315 @@
+package runs
+
+import (
+ "context"
+ "encoding/json/jsontext"
+ "encoding/json/v2"
+ "errors"
+ "fmt"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+ "uuid"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ "github.com/libtnb/sqlite"
+ "github.com/stretchr/testify/require"
+ kit "go.getarcane.app/kit/pkg"
+ "gorm.io/gorm"
+)
+
+func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ sqlDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
+ store := kv.NewKVService(&database.DB{DB: db})
+ legacy := st.QueueRecord{JobID: "legacy", EnvironmentID: "0", Runs: []st.Run{{ID: uuid.New().String(), JobID: "legacy", EnvironmentID: "0", Trigger: "manual", Status: st.Succeeded, CreatedAt: time.Now().UTC()}}}
+ encoded, err := json.Marshal(legacy)
+ require.NoError(t, err)
+ key := queuePrefixInternal + kit.SHA256Hex("0"+"\x00"+legacy.JobID)
+ require.NoError(t, store.Set(t.Context(), key, string(encoded)))
+ databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))
+ first := francistest.New(t, databaseURL)
+ q := New(store, first.Service(), time.UTC)
+ require.NoError(t, q.Register(first))
+ francistest.Start(t, first)
+ require.NoError(t, q.importLegacyInternal(t.Context()))
+ require.NoError(t, q.importLegacyInternal(t.Context()))
+ observer := &activityObserverFakeInternal{}
+ observer.unavailable.Store(true)
+ q.SetObserver(observer)
+ _, err = q.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})
+ require.NoError(t, err)
+ require.NoError(t, q.UpdateRun(t.Context(), legacy.Runs[0], func(run *st.Run) error {
+ run.Outcome.Targets = []st.TargetOutcome{{ID: "artifact", Status: st.Succeeded, RecoveryData: jsontext.Value(`{"backupId":"artifact"}`)}}
+ return nil
+ }))
+ require.True(t, q.hasPendingActivitySyncInternal(t.Context()))
+ require.NoError(t, first.Stop(t.Context()))
+
+ second := francistest.New(t, databaseURL)
+ restored := New(store, second.Service(), time.UTC)
+ require.NoError(t, restored.Register(second))
+ francistest.Start(t, second)
+ require.NoError(t, restored.importLegacyInternal(t.Context()))
+ duplicate, err := restored.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})
+ require.NoError(t, err)
+ require.Equal(t, st.Succeeded, duplicate.Status)
+ require.Equal(t, jsontext.Value(`{"backupId":"artifact"}`), duplicate.Outcome.Targets[0].RecoveryData)
+ history, err := restored.List(t.Context(), "0", legacy.JobID, 1, 20)
+ require.NoError(t, err)
+ require.Equal(t, 1, history.Total)
+ require.Equal(t, time.UTC, history.Runs[0].CreatedAt.Location())
+ observer.unavailable.Store(false)
+ restored.SetObserver(observer)
+ restored.retryActivitySyncInternal(t.Context())
+ require.False(t, restored.hasPendingActivitySyncInternal(t.Context()))
+ require.Positive(t, observer.observed.Load())
+ _, exists, err := store.Get(t.Context(), key)
+ require.NoError(t, err)
+ require.True(t, exists)
+}
+
+type occurrenceEnvelopeInternal struct{ occurrence st.ScheduleOccurrence }
+
+func (e occurrenceEnvelopeInternal) Decode(value any) error {
+ *value.(*st.ScheduleOccurrence) = e.occurrence
+ return nil
+}
+
+func TestAlarmIgnoresStaleGeneration(t *testing.T) {
+ q := newResolutionQueueInternal(t)
+ next := time.Now().UTC().Add(time.Hour)
+ require.NoError(t, q.Checkpoint(t.Context(), "schedule", "0 * * * * *", next))
+ before, err := q.ScheduleState(t.Context(), "schedule")
+ require.NoError(t, err)
+ q.enabled.Store(true)
+ defer q.enabled.Store(false)
+ coordinator := &coordinatorActorInternal{id: kit.SHA256Hex("0" + "\x00" + "schedule"), q: q}
+ require.NoError(t, coordinator.Alarm(t.Context(), "old", occurrenceEnvelopeInternal{st.ScheduleOccurrence{Generation: before.Generation - 1, Sequence: before.Sequence}}))
+ after, err := q.ScheduleState(t.Context(), "schedule")
+ require.NoError(t, err)
+ require.Equal(t, before.Generation, after.Generation)
+ require.Equal(t, before.Sequence, after.Sequence)
+ require.True(t, before.NextRun.Equal(after.NextRun))
+ require.Empty(t, after.Runs)
+}
+
+func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {
+ for _, cancel := range []bool{false, true} {
+ name := "duplicate"
+ if cancel {
+ name = "canceled"
+ }
+ t.Run(name, func(t *testing.T) {
+ q := newResolutionQueueInternal(t)
+ var executed atomic.Int32
+ q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {
+ executed.Add(1)
+ return st.Outcome{Status: st.Succeeded}, nil
+ }, nil)
+ run, err := q.Submit(t.Context(), st.Request{JobID: "dispatch", Trigger: "manual"})
+ require.NoError(t, err)
+ id := kit.SHA256Hex("0" + "\x00" + run.JobID)
+ command := st.ExecutionCommand{EnvironmentID: "0", JobID: run.JobID, RunID: run.ID, Attempt: 1}
+ jobID, created, err := q.service.Dispatch(t.Context(), executorTypeInternal, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))
+ require.NoError(t, err)
+ require.True(t, created)
+ duplicateID, created, err := q.service.Dispatch(t.Context(), executorTypeInternal, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))
+ require.NoError(t, err)
+ require.False(t, created)
+ require.Equal(t, jobID, duplicateID)
+ if cancel {
+ _, err = q.Cancel(t.Context(), "0", run.JobID, run.ID)
+ require.NoError(t, err)
+ }
+ require.NoError(t, q.Start(t.Context()))
+ q.Activate()
+ t.Cleanup(func() { require.NoError(t, q.Stop(context.WithoutCancel(t.Context()))) })
+ require.Eventually(t, func() bool {
+ info, err := q.service.GetJob(t.Context(), jobID)
+ return err == nil && info.Status.IsTerminal() || cancel && errors.Is(err, actor.ErrJobNotFound)
+ }, 5*time.Second, 10*time.Millisecond)
+ persisted, err := q.Get(t.Context(), "0", run.JobID, run.ID)
+ require.NoError(t, err)
+ if cancel {
+ require.Equal(t, st.Canceled, persisted.Status)
+ require.Zero(t, executed.Load())
+ require.Zero(t, persisted.AttemptCount)
+ } else {
+ require.Equal(t, st.Succeeded, persisted.Status)
+ require.Equal(t, int32(1), executed.Load())
+ require.Equal(t, 1, persisted.AttemptCount)
+ }
+ })
+ }
+}
+
+func TestCapacityGroupsReserveHealthSlots(t *testing.T) {
+ q := newResolutionQueueInternal(t)
+ var ordinary, health atomic.Int32
+ release := make(chan struct{})
+ var once sync.Once
+ t.Cleanup(func() { once.Do(func() { close(release) }) })
+ q.SetExecutor(func(ctx context.Context, run st.Run) (st.Outcome, error) {
+ counter := &ordinary
+ if strings.HasPrefix(run.JobID, "environment-health") {
+ counter = &health
+ }
+ counter.Add(1)
+ defer counter.Add(-1)
+ select {
+ case <-ctx.Done():
+ return st.Outcome{}, ctx.Err()
+ case <-release:
+ return st.Outcome{Status: st.Succeeded}, nil
+ }
+ }, nil)
+ for _, job := range []string{"one", "two", "three", "four", "five", "six", "environment-health-one", "environment-health-two"} {
+ _, err := q.Submit(t.Context(), st.Request{JobID: job, Trigger: "manual"})
+ require.NoError(t, err)
+ }
+ require.NoError(t, q.Start(t.Context()))
+ q.Activate()
+ t.Cleanup(func() {
+ once.Do(func() { close(release) })
+ require.NoError(t, q.Stop(context.WithoutCancel(t.Context())))
+ })
+ require.Eventually(t, func() bool { return ordinary.Load() == 4 && health.Load() == 2 }, 5*time.Second, 10*time.Millisecond)
+ once.Do(func() { close(release) })
+ require.Eventually(t, func() bool {
+ history, err := q.List(t.Context(), "0", "", 1, 20)
+ if err != nil {
+ return false
+ }
+ for _, run := range history.Runs {
+ if run.Status != st.Succeeded {
+ return false
+ }
+ }
+ return history.Total == 8
+ }, 5*time.Second, 10*time.Millisecond)
+}
+
+func TestReconciliationRepairsMissingAlarm(t *testing.T) {
+ q := newResolutionQueueInternal(t)
+ require.NoError(t, q.Checkpoint(t.Context(), "repair", "0 * * * * *", time.Now().UTC().Add(time.Hour)))
+ q.enabled.Store(true)
+ defer q.enabled.Store(false)
+ record, err := q.ScheduleState(t.Context(), "repair")
+ require.NoError(t, err)
+ id := kit.SHA256Hex("0" + "\x00" + "repair")
+ err = q.repairInternal(t.Context())
+ require.NoError(t, err)
+ require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
+ err = q.repairInternal(t.Context())
+ require.NoError(t, err)
+ require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
+ after, err := q.ScheduleState(t.Context(), "repair")
+ require.NoError(t, err)
+ require.Equal(t, record.Generation, after.Generation)
+ require.Equal(t, record.Sequence, after.Sequence)
+ require.Empty(t, after.Runs)
+}
+
+func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {
+ host := francistest.New(t)
+ first := NewAdmission(host.Service(), "first")
+ require.NoError(t, first.Register(host))
+ francistest.Start(t, host)
+ key := st.AdmissionKey{Scope: "volume-backup", ID: "shared"}
+ old, admitted, err := first.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.True(t, admitted)
+ _, admitted, err = first.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.False(t, admitted)
+ restarted := NewAdmission(host.Service(), "next")
+ current, admitted, err := restarted.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.True(t, admitted)
+ old.Release(t.Context())
+ _, admitted, err = restarted.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.False(t, admitted)
+ current.Release(t.Context())
+ replacement, admitted, err := restarted.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.True(t, admitted)
+ replacement.Release(t.Context())
+
+ q := newResolutionQueueInternal(t)
+ run, err := q.Submit(t.Context(), st.Request{JobID: "tokens"})
+ require.NoError(t, err)
+ require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error { current.Status = st.Running; current.Owner = "new-token"; return nil }))
+ run.Owner = "stale-token"
+ q.persistOutcomeInternal(t.Context(), run, st.Outcome{Status: st.Succeeded})
+ persisted, err := q.Get(t.Context(), "0", run.JobID, run.ID)
+ require.NoError(t, err)
+ require.Equal(t, st.Running, persisted.Status)
+ require.Equal(t, "new-token", persisted.Owner)
+}
+
+func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
+ q := newResolutionQueueInternal(t)
+ entered := make(chan struct{})
+ finish := make(chan struct{})
+ var once sync.Once
+ t.Cleanup(func() { once.Do(func() { close(finish) }) })
+ q.SetExecutor(func(ctx context.Context, _ st.Run) (st.Outcome, error) {
+ close(entered)
+ select {
+ case <-ctx.Done():
+ return st.Outcome{}, ctx.Err()
+ case <-finish:
+ return st.Outcome{Status: st.Succeeded}, nil
+ }
+ }, nil)
+ run, err := q.Submit(t.Context(), st.Request{JobID: "lost-ack"})
+ require.NoError(t, err)
+ require.NoError(t, q.Start(t.Context()))
+ q.Activate()
+ t.Cleanup(func() {
+ once.Do(func() { close(finish) })
+ require.NoError(t, q.Stop(context.WithoutCancel(t.Context())))
+ })
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("execution did not start")
+ }
+ id := kit.SHA256Hex("0" + "\x00" + run.JobID)
+ var state st.CoordinatorState
+ require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))
+ key := run.ID + "-1"
+ original := state.Dispatches[key].JobID
+ require.NotEmpty(t, original)
+ intent := state.Dispatches[key]
+ intent.JobID = ""
+ state.Dispatches[key] = intent
+ state.Revision++
+ require.NoError(t, q.service.SetState(t.Context(), coordinatorTypeInternal, id, state, nil))
+ err = q.repairInternal(t.Context())
+ require.NoError(t, err)
+ require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))
+ require.Equal(t, original, state.Dispatches[key].JobID)
+ deliveries, err := q.service.ListJobs(t.Context(), executorTypeInternal, id)
+ require.NoError(t, err)
+ require.Len(t, deliveries, 1)
+ once.Do(func() { close(finish) })
+ require.Eventually(t, func() bool {
+ stored, err := q.Get(t.Context(), "0", run.JobID, run.ID)
+ return err == nil && stored.Status == st.Succeeded
+ }, 5*time.Second, 10*time.Millisecond)
+}
diff --git a/backend/pkg/scheduler/runs/admission.go b/backend/pkg/scheduler/runs/admission.go
new file mode 100644
index 000000000..b176066c5
--- /dev/null
+++ b/backend/pkg/scheduler/runs/admission.go
@@ -0,0 +1,146 @@
+package runs
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+ "time"
+ "uuid"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ kit "go.getarcane.app/kit/pkg"
+)
+
+const admissionTypeInternal = "resource-admission"
+
+type Admission struct {
+ service *actor.Service
+ epoch string
+ mu sync.Mutex
+ pending map[string]map[string]st.AdmissionCommand
+}
+type Lease struct {
+ admission *Admission
+ id string
+ token string
+ once sync.Once
+}
+type admissionActorInternal struct {
+ id string
+ service *actor.Service
+}
+
+func NewAdmission(service *actor.Service, epoch string) *Admission {
+ return &Admission{service: service, epoch: epoch, pending: make(map[string]map[string]st.AdmissionCommand)}
+}
+
+func (a *Admission) Register(runtime *francis.Runtime) error {
+ return runtime.RegisterActor(admissionTypeInternal, func(id string, service *actor.Service) actor.Actor {
+ return &admissionActorInternal{id: id, service: service}
+ })
+}
+
+func (a *Admission) TryAcquire(ctx context.Context, key st.AdmissionKey) (*Lease, bool, error) {
+ id := kit.SHA256Hex(key.Scope + "\x00" + key.ID)
+ token := uuid.New().String()
+ if err := ctx.Err(); err != nil {
+ return nil, false, err
+ }
+ if err := a.retryReleasesInternal(ctx, id); err != nil {
+ return nil, false, err
+ }
+ // Finish admission after submission so cancellation cannot orphan a lease.
+ invokeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ envelope, err := a.service.Invoke(invokeCtx, admissionTypeInternal, id, "acquire", st.AdmissionCommand{Epoch: a.epoch, Token: token})
+ if err != nil {
+ a.releaseInternal(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})
+ return nil, false, err
+ }
+ var acquired bool
+ if err := envelope.Decode(&acquired); err != nil {
+ a.releaseInternal(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})
+ return nil, false, err
+ }
+ if !acquired {
+ return nil, false, nil
+ }
+ return &Lease{admission: a, id: id, token: token}, true, nil
+}
+
+func (l *Lease) Release(ctx context.Context) {
+ if l == nil {
+ return
+ }
+ l.once.Do(func() {
+ l.admission.releaseInternal(ctx, l.id, st.AdmissionCommand{Epoch: l.admission.epoch, Token: l.token})
+ })
+}
+
+func (a *Admission) releaseInternal(ctx context.Context, id string, command st.AdmissionCommand) {
+ a.mu.Lock()
+ if a.pending[id] == nil {
+ a.pending[id] = make(map[string]st.AdmissionCommand)
+ }
+ a.pending[id][command.Token] = command
+ a.mu.Unlock()
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+ defer cancel()
+ if err := a.retryReleasesInternal(ctx, id); err != nil {
+ slog.ErrorContext(ctx, "Resource admission release failed; will retry before further admission", "resource", id, "error", err)
+ }
+}
+
+func (a *Admission) retryReleasesInternal(ctx context.Context, id string) error {
+ a.mu.Lock()
+ pending := make([]st.AdmissionCommand, 0, len(a.pending[id]))
+ for _, command := range a.pending[id] {
+ pending = append(pending, command)
+ }
+ a.mu.Unlock()
+ for _, command := range pending {
+ if _, err := a.service.Invoke(ctx, admissionTypeInternal, id, "release", command); err != nil {
+ return err
+ }
+ a.mu.Lock()
+ delete(a.pending[id], command.Token)
+ if len(a.pending[id]) == 0 {
+ delete(a.pending, id)
+ }
+ a.mu.Unlock()
+ }
+ return nil
+}
+
+func (a *admissionActorInternal) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {
+ var state st.AdmissionState
+ err := a.service.GetState(ctx, admissionTypeInternal, a.id, &state)
+ if err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return nil, err
+ }
+ var command st.AdmissionCommand
+ if err := data.Decode(&command); err != nil {
+ return nil, err
+ }
+ switch method {
+ case "acquire":
+ if state.Epoch == command.Epoch && state.Token != "" {
+ return false, nil
+ }
+ state = st.AdmissionState{Epoch: command.Epoch, Token: command.Token, AcquiredAt: time.Now().UTC()}
+ case "release":
+ if state.Epoch != command.Epoch || state.Token != command.Token {
+ return false, nil
+ }
+ state.Token = ""
+ default:
+ return nil, errors.New("unknown resource admission command")
+ }
+ if err := a.service.SetState(ctx, admissionTypeInternal, a.id, state, nil); err != nil {
+ return nil, err
+ }
+ return true, nil
+}
diff --git a/backend/pkg/scheduler/queue/queue.go b/backend/pkg/scheduler/runs/coordinator.go
similarity index 69%
rename from backend/pkg/scheduler/queue/queue.go
rename to backend/pkg/scheduler/runs/coordinator.go
index 3897e416f..9d49fa483 100644
--- a/backend/pkg/scheduler/queue/queue.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -1,5 +1,5 @@
-// Package queue persists scheduler runs and coordinates their execution and recovery.
-package queue
+// Package runs coordinates durable scheduler execution and recovery.
+package runs
import (
"cmp"
@@ -9,42 +9,43 @@ import (
"log/slog"
"strings"
"sync"
+ "sync/atomic"
"time"
+ "uuid"
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/google/uuid"
+ "github.com/italypaleale/francis/actor"
kit "go.getarcane.app/kit/pkg"
)
-// Queue persists work before dispatching it to the owning domain.
-type Queue struct {
+// Coordinator persists work before dispatching it to the owning domain.
+type Coordinator struct {
store *kv.KVService
+ service *actor.Service
owner string
mu sync.Mutex
- active map[string]bool
checkpointed map[string]bool
- ordinary int
- health int
stopping bool
started bool
cancel context.CancelFunc
+ lifecycleCtx context.Context
done chan struct{}
wake chan struct{}
- workers sync.WaitGroup
+ enabled atomic.Bool
execute func(context.Context, st.Run) (st.Outcome, error)
reconcile func(context.Context, st.Run) (st.Outcome, error)
observer st.RunObserver
observerMu sync.Mutex
- pendingSync map[string]st.Run
+ location *time.Location
}
// Submit persists acceptance before waking the dispatcher. Run IDs deduplicate
// requests within a job and environment. Manual and remote requests stay separate;
// other triggers coalesce pending work.
// Cancellation of ctx affects admission only, not an accepted run's lifetime.
-func (q *Queue) Submit(ctx context.Context, request st.Request) (st.Run, error) {
+func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, error) {
q.mu.Lock()
defer q.mu.Unlock()
if q.stopping {
@@ -56,7 +57,7 @@ func (q *Queue) Submit(ctx context.Context, request st.Request) (st.Run, error)
}
request.Trigger = cmp.Or(request.Trigger, "manual")
if request.RunID == "" {
- request.RunID = uuid.NewString()
+ request.RunID = uuid.New().String()
}
if _, err := uuid.Parse(request.RunID); err != nil {
return st.Run{}, errors.New("invalid run ID")
@@ -93,7 +94,7 @@ func (q *Queue) Submit(ctx context.Context, request st.Request) (st.Run, error)
return result, err
}
-func (q *Queue) signalInternal() {
+func (q *Coordinator) signalInternal() {
select {
case q.wake <- struct{}{}:
default:
@@ -101,13 +102,15 @@ func (q *Queue) signalInternal() {
}
// Checkpoint records the next due occurrence atomically with overdue recovery.
-func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {
+func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {
q.mu.Lock()
defer q.mu.Unlock()
first := !q.checkpointed[jobID]
recovered := false
var recoveredRun st.Run
err := q.mutateInternal(ctx, "0", jobID, func(record *st.QueueRecord) error {
+ recovered = false
+ recoveredRun = st.Run{}
now := time.Now().UTC()
if first && record.Schedule == schedule && !record.NextRun.IsZero() && !record.NextRun.After(now) && record.LastEnqueuedAt.Before(record.NextRun) {
pending := false
@@ -120,12 +123,16 @@ func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next tim
if !pending {
recovered = true
record.LastEnqueuedAt = now
- recoveredRun = st.Run{ID: uuid.NewString(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now}
+ recoveredRun = st.Run{ID: uuid.New().String(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now}
record.Runs = append(record.Runs, recoveredRun)
}
}
- record.Schedule = schedule
- record.NextRun = next
+ if record.Schedule != schedule || record.NextRun.IsZero() || (first && !record.NextRun.After(now)) {
+ record.Generation++
+ record.Sequence++
+ record.Schedule = schedule
+ record.NextRun = next
+ }
return nil
})
if err == nil {
@@ -140,7 +147,7 @@ func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next tim
// Start validates persisted state and starts dispatching on ctx. Repeated calls
// are harmless, but a stopped queue cannot be restarted; create a new instance.
-func (q *Queue) Start(ctx context.Context) error {
+func (q *Coordinator) Start(ctx context.Context) error {
q.mu.Lock()
defer q.mu.Unlock()
if q.started {
@@ -149,6 +156,9 @@ func (q *Queue) Start(ctx context.Context) error {
if q.execute == nil {
return errors.New("job executor unavailable")
}
+ if err := q.importLegacyInternal(ctx); err != nil {
+ return err
+ }
// Fail startup if persisted state cannot be read. Never replace it with empty state.
records, err := q.Records(ctx)
if err != nil {
@@ -163,8 +173,10 @@ func (q *Queue) Start(ctx context.Context) error {
}
}
}
+
runCtx, cancel := context.WithCancel(ctx)
q.cancel = cancel
+ q.lifecycleCtx = runCtx
q.started = true
go q.dispatchInternal(runCtx)
return nil
@@ -172,9 +184,10 @@ func (q *Queue) Start(ctx context.Context) error {
// Stop rejects new submissions, cancels execution, and waits for workers until
// ctx expires. Unfinished claims remain persisted for startup reconciliation.
-func (q *Queue) Stop(ctx context.Context) error {
+func (q *Coordinator) Stop(ctx context.Context) error {
q.mu.Lock()
q.stopping = true
+ q.enabled.Store(false)
started := q.started
if q.cancel != nil {
q.cancel()
@@ -191,9 +204,9 @@ func (q *Queue) Stop(ctx context.Context) error {
}
}
-func (q *Queue) dispatchInternal(ctx context.Context) {
+func (q *Coordinator) dispatchInternal(ctx context.Context) {
defer close(q.done)
- defer q.workers.Wait()
+
nextRetention := time.Now().Add(time.Hour)
for ctx.Err() == nil {
q.retryActivitySyncInternal(ctx)
@@ -203,17 +216,19 @@ func (q *Queue) dispatchInternal(ctx context.Context) {
}
nextRetention = time.Now().Add(time.Hour)
}
- nextRetry, err := q.dispatchReadyInternal(ctx)
- if err != nil && ctx.Err() == nil {
+ nextWake := time.Now().Add(15 * time.Second)
+ if err := q.repairInternal(ctx); err != nil && ctx.Err() == nil {
slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)
- nextRetry = time.Now().Add(5 * time.Second)
- }
- nextWake := nextRetention
- if q.hasPendingActivitySyncInternal() {
nextWake = time.Now().Add(5 * time.Second)
}
- if !nextRetry.IsZero() && nextRetry.Before(nextWake) {
- nextWake = nextRetry
+ if nextRetention.Before(nextWake) {
+ nextWake = nextRetention
+ }
+ if q.hasPendingActivitySyncInternal(ctx) {
+ activityRetry := time.Now().Add(5 * time.Second)
+ if activityRetry.Before(nextWake) {
+ nextWake = activityRetry
+ }
}
timer := time.NewTimer(max(time.Until(nextWake), 0))
select {
@@ -227,98 +242,11 @@ func (q *Queue) dispatchInternal(ctx context.Context) {
}
}
-func (q *Queue) dispatchReadyInternal(ctx context.Context) (time.Time, error) {
- records, err := q.Records(ctx)
- if err != nil {
- return time.Time{}, err
- }
- var nextRetry time.Time
- for _, record := range records {
- key := queuePrefixInternal + kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID)
- reserved := strings.HasPrefix(record.JobID, "environment-health") || record.JobID == "docker-client-refresh" || record.JobID == "activity-sweep"
- q.mu.Lock()
- if q.active[key] || q.stopping || (!reserved && q.ordinary >= 4) || (reserved && q.health >= 2) {
- q.mu.Unlock()
- continue
- }
- selected, retryAt := q.nextRunInternal(record.Runs)
- if !retryAt.IsZero() && (nextRetry.IsZero() || retryAt.Before(nextRetry)) {
- nextRetry = retryAt
- }
- if selected == nil {
- q.mu.Unlock()
- continue
- }
- run := *selected
- reconcile := run.Status == st.Running
- err = q.UpdateRun(ctx, run, func(current *st.Run) error {
- if current.Status != run.Status || current.Owner != run.Owner {
- return ErrRunConflict
- }
- now := time.Now().UTC()
- current.Status = st.Running
- current.Owner = q.owner
- current.UpdatedAt = now
- current.StartedAt = &now
- current.NextAttempt = nil
- current.AttemptCount++
- current.Attempts = append(current.Attempts, st.Attempt{Number: current.AttemptCount, StartedAt: now})
- if len(current.Attempts) > 100 {
- current.Attempts = current.Attempts[len(current.Attempts)-100:]
- }
- // Preserve the previous outcome for reconciliation and completed target filtering.
- run = *current
- return nil
- })
- if err != nil {
- q.mu.Unlock()
- return time.Time{}, err
- }
- q.active[key] = true
- if reserved {
- q.health++
- } else {
- q.ordinary++
- }
- q.workers.Add(1)
- q.mu.Unlock()
- go q.executeInternal(ctx, key, run, reserved, reconcile)
- }
- return nextRetry, nil
-}
-
-func (q *Queue) executeInternal(ctx context.Context, key string, run st.Run, reserved, reconcile bool) {
- defer q.workers.Done()
- defer func() {
- q.mu.Lock()
- delete(q.active, key)
- if reserved {
- q.health--
- } else {
- q.ordinary--
- }
- q.mu.Unlock()
- q.signalInternal()
- }()
- outcome, err := q.invokeInternal(ctx, run, reconcile)
- if outcome.Status == "" {
- outcome.Status = kit.Ternary(err != nil, st.Failed, st.Succeeded)
- }
- if err != nil && outcome.Message == "" {
- outcome.Message = err.Error()
- }
- // Shutdown leaves a running claim for startup reconciliation, never a false success.
- if ctx.Err() != nil {
- return
- }
- q.persistOutcomeInternal(ctx, run, outcome)
-}
-
-func (q *Queue) persistOutcomeInternal(ctx context.Context, run st.Run, outcome st.Outcome) {
+func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, outcome st.Outcome) {
// Keep the execution slot until the terminal result has been durably recorded.
for {
err := q.UpdateRun(ctx, run, func(current *st.Run) error {
- if current.Owner != q.owner || current.Status != st.Running {
+ if current.Owner != run.Owner || current.Status != st.Running {
return ErrRunConflict
}
now := time.Now().UTC()
@@ -365,7 +293,7 @@ func (q *Queue) persistOutcomeInternal(ctx context.Context, run st.Run, outcome
}
}
-func (q *Queue) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {
+func (q *Coordinator) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {
defer func() {
if recovered := recover(); recovered != nil {
outcome = st.Outcome{Status: st.NeedsAttention, Message: "Job panicked; inspect logs before retrying"}
@@ -381,7 +309,7 @@ func (q *Queue) invokeInternal(ctx context.Context, run st.Run, reconcile bool)
// Cancel marks an unstarted queued run canceled. Runs already attempted locally
// or dispatched remotely must be reconciled rather than canceled as pending work.
-func (q *Queue) Cancel(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+func (q *Coordinator) Cancel(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
run, err := q.Get(ctx, environmentID, jobID, runID)
if err != nil {
return run, err
@@ -397,12 +325,27 @@ func (q *Queue) Cancel(ctx context.Context, environmentID, jobID, runID string)
run = *current
return nil
})
+ if err == nil {
+ var state st.CoordinatorState
+ id := kit.SHA256Hex(environmentID + "\x00" + jobID)
+ actorType := executorTypeInternal
+ if strings.HasPrefix(jobID, "environment-health") || jobID == "docker-client-refresh" || jobID == "activity-sweep" {
+ actorType = healthExecutorTypeInternal
+ }
+ if readErr := q.service.GetState(ctx, coordinatorTypeInternal, id, &state); readErr == nil {
+ for _, intent := range state.Dispatches {
+ if intent.RunID == runID && intent.JobID != "" {
+ _ = q.service.DeleteJob(ctx, actorType, id, intent.JobID, actor.WithLiveJobsOnly())
+ }
+ }
+ }
+ }
return run, err
}
// Retry requeues a failed, partial, or attention-required run without changing its
// ID or dropping target progress. Unconfirmed remote work cannot be retried here.
-func (q *Queue) Retry(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+func (q *Coordinator) Retry(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
run, err := q.Get(ctx, environmentID, jobID, runID)
if err != nil {
return run, err
@@ -428,15 +371,15 @@ func (q *Queue) Retry(ctx context.Context, environmentID, jobID, runID string) (
return run, err
}
-func (q *Queue) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {
+func (q *Coordinator) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {
var interrupted *st.Run
blocked := false
for index := range runs {
run := &runs[index]
- if run.Status == st.Running && run.Owner == q.owner {
+ if run.Status == st.Running && strings.HasPrefix(run.Owner, q.owner+"/") {
blocked = true
}
- if run.Status == st.Running && run.Owner != q.owner && interrupted == nil {
+ if run.Status == st.Running && !strings.HasPrefix(run.Owner, q.owner+"/") && interrupted == nil {
interrupted = run
}
}
@@ -476,3 +419,5 @@ func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {
outcome.ActivityID = cmp.Or(outcome.ActivityID, previous.ActivityID)
return outcome
}
+
+func (q *Coordinator) Active() bool { return q.enabled.Load() }
diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.go
new file mode 100644
index 000000000..5a598671f
--- /dev/null
+++ b/backend/pkg/scheduler/runs/coordinator_store.go
@@ -0,0 +1,365 @@
+package runs
+
+import (
+ "bytes"
+ "context"
+ "encoding/json/v2"
+ "slices"
+ "sort"
+ "time"
+ "uuid"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/italypaleale/francis/actor"
+ kit "go.getarcane.app/kit/pkg"
+)
+
+const queuePrefixInternal = "jobs."
+
+func (q *Coordinator) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {
+ id := kit.SHA256Hex(environmentID + "\x00" + jobID)
+ for range 20 {
+ var state st.CoordinatorState
+ err := q.service.GetState(ctx, coordinatorTypeInternal, id, &state)
+ if err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ return err
+ }
+ if err := q.prepareMutationInternal(&state, environmentID, jobID, change); err != nil {
+ return err
+ }
+ result, err := q.service.Invoke(ctx, coordinatorTypeInternal, id, "replace", state)
+ if err != nil {
+ return err
+ }
+ var swapped bool
+ if err = result.Decode(&swapped); err != nil {
+ return err
+ }
+ if swapped {
+ return nil
+ }
+ }
+ return errors.New("job coordinator state is busy; retry mutation")
+}
+
+func (q *Coordinator) prepareMutationInternal(state *st.CoordinatorState, environmentID, jobID string, change func(*st.QueueRecord) error) error {
+ normalizeRecordTimesInternal(&state.Record)
+ previousRuns := slices.Clone(state.Record.Runs)
+ previous, versions, err := snapshotRunsInternal(state.Record)
+ if err != nil {
+ return err
+ }
+ if state.Record.JobID == "" {
+ state.Record = st.QueueRecord{JobID: jobID, EnvironmentID: environmentID}
+ }
+ if err := change(&state.Record); err != nil {
+ return err
+ }
+ preserveUnsyncedRunsInternal(state, previousRuns)
+ for i := range state.Record.Runs {
+ q.associateActivityInternal(&state.Record.Runs[i])
+ if err := markChangedActivityInternal(state, &state.Record.Runs[i], previous, versions); err != nil {
+ return err
+ }
+ }
+ for id, run := range state.Record.Receipts {
+ q.associateActivityInternal(&run)
+ if err := markChangedActivityInternal(state, &run, previous, versions); err != nil {
+ return err
+ }
+ state.Record.Receipts[id] = run
+ }
+ pruneDispatchesInternal(state)
+ return nil
+}
+
+func snapshotRunsInternal(record st.QueueRecord) (map[string][]byte, map[string]time.Time, error) {
+ encoded := make(map[string][]byte, len(record.Runs)+len(record.Receipts))
+ versions := make(map[string]time.Time, len(record.Runs)+len(record.Receipts))
+ for _, run := range append(slices.Clone(record.Runs), receiptRunsInternal(record.Receipts)...) {
+ data, err := json.Marshal(run)
+ if err != nil {
+ return nil, nil, err
+ }
+ encoded[run.ID] = data
+ versions[run.ID] = run.UpdatedAt
+ }
+ return encoded, versions, nil
+}
+
+func preserveUnsyncedRunsInternal(state *st.CoordinatorState, previous []st.Run) {
+ retained := make(map[string]bool, len(state.Record.Runs))
+ for _, run := range state.Record.Runs {
+ retained[run.ID] = true
+ }
+ for _, run := range previous {
+ if _, pending := state.PendingActivitySync[run.ID]; pending && !retained[run.ID] {
+ state.Record.Runs = append(state.Record.Runs, run)
+ delete(state.Record.Receipts, run.ID)
+ }
+ }
+}
+
+func pruneDispatchesInternal(state *st.CoordinatorState) {
+ attempts := make(map[string]int, len(state.Record.Runs))
+ for _, run := range state.Record.Runs {
+ attempts[run.ID] = run.AttemptCount
+ }
+ for key, intent := range state.Dispatches {
+ attempt, retained := attempts[intent.RunID]
+ if !retained || intent.Attempt < attempt-99 {
+ delete(state.Dispatches, key)
+ }
+ }
+}
+
+func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
+ records := []st.QueueRecord{}
+ cursor := ""
+ for {
+ page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ if err != nil {
+ return nil, err
+ }
+ for _, entry := range page.States {
+ var state st.CoordinatorState
+ if err := entry.Data.Decode(&state); err != nil {
+ return nil, err
+ }
+ normalizeRecordTimesInternal(&state.Record)
+ records = append(records, state.Record)
+ }
+ cursor = page.AfterID()
+ if cursor == "" {
+ return records, nil
+ }
+ }
+}
+
+// Get retrieves a run by environment, job, and run ID, including compacted
+// idempotency receipts. It returns ErrRunNotFound when no matching record exists.
+func (q *Coordinator) Get(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ var state st.CoordinatorState
+ err := q.service.GetState(ctx, coordinatorTypeInternal, kit.SHA256Hex(environmentID+"\x00"+jobID), &state)
+ if errors.Is(err, actor.ErrStateNotFound) {
+ return st.Run{}, ErrRunNotFound
+ }
+ if err != nil {
+ return st.Run{}, err
+ }
+ normalizeRecordTimesInternal(&state.Record)
+ record := state.Record
+ for _, run := range record.Runs {
+ if run.ID == runID {
+ q.associateActivityInternal(&run)
+ return run, nil
+ }
+ }
+ if run, ok := record.Receipts[runID]; ok {
+ q.associateActivityInternal(&run)
+ return run, nil
+ }
+ return st.Run{}, ErrRunNotFound
+}
+
+// List returns retained run history newest first, excluding compacted receipts.
+// An empty job ID includes every job in the environment; pages are one-based.
+func (q *Coordinator) List(ctx context.Context, environmentID, jobID string, page, limit int) (st.RunList, error) {
+ records, err := q.Records(ctx)
+ if err != nil {
+ return st.RunList{}, err
+ }
+ runs := []st.Run{}
+ for _, record := range records {
+ if record.EnvironmentID == environmentID && (jobID == "" || record.JobID == jobID) {
+ runs = append(runs, record.Runs...)
+ }
+ }
+ sort.Slice(runs, func(i, j int) bool { return runs[i].CreatedAt.After(runs[j].CreatedAt) })
+ page = max(1, page)
+ limit = min(100, max(1, limit))
+ start := len(runs)
+ if page-1 <= len(runs)/limit {
+ start = min(len(runs), (page-1)*limit)
+ }
+ end := min(len(runs), start+limit)
+ return st.RunList{Runs: runs[start:end], Total: len(runs), Page: page, Limit: limit}, nil
+}
+
+// UpdateRun atomically changes a persisted run or terminal receipt using compare
+// and swap. The callback may run more than once and must not perform external side effects.
+func (q *Coordinator) UpdateRun(ctx context.Context, run st.Run, change func(*st.Run) error) error {
+ wake := false
+ err := q.mutateInternal(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {
+ for index := range record.Runs {
+ if record.Runs[index].ID == run.ID {
+ previous := record.Runs[index].Status
+ if err := change(&record.Runs[index]); err != nil {
+ return err
+ }
+ wake = previous != st.Queued && record.Runs[index].Status == st.Queued
+ return nil
+ }
+ }
+ if receipt, found := record.Receipts[run.ID]; found {
+ if err := change(&receipt); err != nil {
+ return err
+ }
+ if !receipt.Status.Terminal() {
+ return errors.New("archived runs cannot be restarted")
+ }
+ record.Receipts[run.ID] = receipt
+ return nil
+ }
+ return ErrRunNotFound
+ })
+ if err == nil && wake {
+ q.signalInternal()
+ }
+ if err == nil {
+ q.observeRunInternal(ctx, run)
+ }
+ return err
+}
+
+func (q *Coordinator) pruneInternal(ctx context.Context) error {
+ records, err := q.Records(ctx)
+ if err != nil {
+ return err
+ }
+ now := time.Now().UTC()
+ for _, record := range records {
+ if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
+ pruneRunsInternal(current, now)
+ return nil
+ }); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func pruneRunsInternal(record *st.QueueRecord, now time.Time) {
+ terminal := 0
+ for index, run := range slices.Backward(record.Runs) {
+ if !run.Status.Terminal() || ((run.RemoteDeliveryAttempted || run.RemoteAccepted) && !run.RemoteSettled) {
+ continue
+ }
+ terminal++
+ if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {
+ continue
+ }
+ if run.Trigger == "manual" || run.Trigger == "remote" {
+ if record.Receipts == nil {
+ record.Receipts = make(map[string]st.Run)
+ }
+ run.Attempts = nil
+ run.Outcome.Targets = nil
+ record.Receipts[run.ID] = run
+ }
+ record.Runs = append(record.Runs[:index], record.Runs[index+1:]...)
+ }
+}
+
+// ErrRunNotFound indicates that neither a run nor its receipt exists.
+var ErrRunNotFound = errors.Sentinel("job run not found")
+
+// ErrRunConflict indicates that execution ownership or state changed.
+var ErrRunConflict = errors.Sentinel("job run state changed")
+
+func New(store *kv.KVService, service *actor.Service, location *time.Location) *Coordinator {
+ if location == nil {
+ location = time.UTC
+ }
+ return &Coordinator{store: store, service: service, location: location, owner: uuid.New().String(), checkpointed: make(map[string]bool), wake: make(chan struct{}, 1), done: make(chan struct{})}
+}
+
+func (q *Coordinator) SetExecutor(execute, reconcile func(context.Context, st.Run) (st.Outcome, error)) {
+ q.execute = execute
+ q.reconcile = reconcile
+}
+
+func (q *Coordinator) ScheduleState(ctx context.Context, jobID string) (st.QueueRecord, error) {
+ var state st.CoordinatorState
+ err := q.service.GetState(ctx, coordinatorTypeInternal, kit.SHA256Hex("0"+"\x00"+jobID), &state)
+ normalizeRecordTimesInternal(&state.Record)
+ return state.Record, err
+}
+
+func (q *Coordinator) Activate() { q.enabled.Store(true); q.signalInternal() }
+
+func receiptRunsInternal(receipts map[string]st.Run) []st.Run {
+ runs := make([]st.Run, 0, len(receipts))
+ for _, run := range receipts {
+ runs = append(runs, run)
+ }
+ return runs
+}
+
+func markChangedActivityInternal(state *st.CoordinatorState, run *st.Run, previous map[string][]byte, previousUpdated map[string]time.Time) error {
+ normalizeRunTimesInternal(run)
+ if run.ActivityID == "" {
+ return nil
+ }
+ encoded, err := json.Marshal(run)
+ if err != nil {
+ return err
+ }
+ if bytes.Equal(previous[run.ID], encoded) {
+ return nil
+ }
+ if run.UpdatedAt.IsZero() || run.UpdatedAt.Equal(previousUpdated[run.ID]) {
+ run.UpdatedAt = time.Now().UTC()
+ }
+ if state.PendingActivitySync == nil {
+ state.PendingActivitySync = make(map[string]time.Time)
+ }
+ state.PendingActivitySync[run.ID] = run.UpdatedAt
+ return nil
+}
+
+func markActivitySyncInternal(state *st.CoordinatorState, run st.Run) {
+ if run.ActivityID == "" {
+ return
+ }
+ if state.PendingActivitySync == nil {
+ state.PendingActivitySync = make(map[string]time.Time)
+ }
+ state.PendingActivitySync[run.ID] = run.UpdatedAt
+}
+
+func normalizeRecordTimesInternal(record *st.QueueRecord) {
+ record.LastEnqueuedAt = record.LastEnqueuedAt.UTC()
+ record.NextRun = record.NextRun.UTC()
+ for i := range record.Runs {
+ normalizeRunTimesInternal(&record.Runs[i])
+ }
+ for id, run := range record.Receipts {
+ normalizeRunTimesInternal(&run)
+ record.Receipts[id] = run
+ }
+}
+
+func normalizeRunTimesInternal(run *st.Run) {
+ run.CreatedAt = run.CreatedAt.UTC()
+ run.UpdatedAt = run.UpdatedAt.UTC()
+ normalizeTimePointerInternal(run.StartedAt)
+ normalizeTimePointerInternal(run.FinishedAt)
+ normalizeTimePointerInternal(run.NextAttempt)
+ normalizeTimePointerInternal(run.LastConfirmedAt)
+ if run.Resolution != nil {
+ run.Resolution.ResolvedAt = run.Resolution.ResolvedAt.UTC()
+ }
+ for i := range run.Attempts {
+ run.Attempts[i].StartedAt = run.Attempts[i].StartedAt.UTC()
+ normalizeTimePointerInternal(run.Attempts[i].FinishedAt)
+ }
+}
+
+func normalizeTimePointerInternal(value *time.Time) {
+ if value != nil {
+ *value = value.UTC()
+ }
+}
diff --git a/backend/pkg/scheduler/queue/resolution.go b/backend/pkg/scheduler/runs/resolution.go
similarity index 78%
rename from backend/pkg/scheduler/queue/resolution.go
rename to backend/pkg/scheduler/runs/resolution.go
index 3ccde9f39..fea90a1e4 100644
--- a/backend/pkg/scheduler/queue/resolution.go
+++ b/backend/pkg/scheduler/runs/resolution.go
@@ -1,4 +1,4 @@
-package queue
+package runs
import (
"context"
@@ -6,16 +6,10 @@ import (
"emperror.dev/errors"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
- kit "go.getarcane.app/kit/pkg"
)
// Resolve releases reviewed, inactive work without discarding its execution evidence.
-func (q *Queue) Resolve(ctx context.Context, environmentID, jobID, runID, resolvedBy string) (st.Run, error) {
- q.mu.Lock()
- defer q.mu.Unlock()
- if q.active[queuePrefixInternal+kit.SHA256Hex(environmentID+"\x00"+jobID)] {
- return st.Run{}, errors.New("job is still active")
- }
+func (q *Coordinator) Resolve(ctx context.Context, environmentID, jobID, runID, resolvedBy string) (st.Run, error) {
run, err := q.Get(ctx, environmentID, jobID, runID)
if err != nil {
return run, err
diff --git a/backend/pkg/scheduler/queue/resolution_test.go b/backend/pkg/scheduler/runs/resolution_test.go
similarity index 81%
rename from backend/pkg/scheduler/queue/resolution_test.go
rename to backend/pkg/scheduler/runs/resolution_test.go
index 5b854ba87..27d6fd87c 100644
--- a/backend/pkg/scheduler/queue/resolution_test.go
+++ b/backend/pkg/scheduler/runs/resolution_test.go
@@ -1,19 +1,20 @@
-package queue
+package runs
import (
"context"
"testing"
+ "time"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
- kit "go.getarcane.app/kit/pkg"
"gorm.io/gorm"
)
-func newResolutionQueueInternal(t *testing.T) *Queue {
+func newResolutionQueueInternal(t *testing.T) *Coordinator {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -22,7 +23,11 @@ func newResolutionQueueInternal(t *testing.T) *Queue {
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
- return New(kv.NewKVService(&database.DB{DB: db}), nil, nil)
+ runtime := francistest.New(t)
+ q := New(kv.NewKVService(&database.DB{DB: db}), runtime.Service(), time.UTC)
+ require.NoError(t, q.Register(runtime))
+ francistest.Start(t, runtime)
+ return q
}
func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {
@@ -68,7 +73,11 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {
require.Equal(t, pending.ID, selected.ID)
duplicate, err := q.Resolve(ctx, "0", "auto-update", run.ID, "different operator")
require.NoError(t, err)
- require.Equal(t, resolved, duplicate)
+ require.Equal(t, resolved.ID, duplicate.ID)
+ require.Equal(t, resolved.Status, duplicate.Status)
+ require.Equal(t, resolved.Outcome, duplicate.Outcome)
+ require.Equal(t, resolved.Resolution.ResolvedBy, duplicate.Resolution.ResolvedBy)
+ require.True(t, resolved.Resolution.ResolvedAt.Equal(duplicate.Resolution.ResolvedAt))
}
func TestResolveRejectsUnsafeStates(t *testing.T) {
@@ -81,7 +90,7 @@ func TestResolveRejectsUnsafeStates(t *testing.T) {
}{
{name: "queued", status: st.Queued, environment: "0"},
{name: "running", status: st.Running, environment: "0"},
- {name: "active worker", status: st.NeedsAttention, active: true, environment: "0"},
+ {name: "active worker", status: st.Running, active: true, environment: "0"},
{name: "uncertain remote", status: st.NeedsAttention, environment: "remote", delivered: true},
} {
t.Run(test.name, func(t *testing.T) {
@@ -91,9 +100,11 @@ func TestResolveRejectsUnsafeStates(t *testing.T) {
require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error {
current.Status = test.status
current.RemoteDeliveryAttempted = test.delivered
+ if test.active {
+ current.Owner = "active-worker"
+ }
return nil
}))
- q.active[queuePrefixInternal+kit.SHA256Hex(test.environment+"\x00"+"auto-update")] = test.active
_, err = q.Resolve(context.Background(), test.environment, "auto-update", run.ID, "operator")
require.Error(t, err)
stored, err := q.Get(t.Context(), test.environment, "auto-update", run.ID)
diff --git a/backend/pkg/scheduler/scheduled_prune_job.go b/backend/pkg/scheduler/scheduled_prune_job.go
index 0d2d787a5..54ab99c98 100644
--- a/backend/pkg/scheduler/scheduled_prune_job.go
+++ b/backend/pkg/scheduler/scheduled_prune_job.go
@@ -8,7 +8,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
@@ -22,9 +21,9 @@ type ScheduledPruneJob struct {
notificationService *notification.NotificationService
}
-func NewScheduledPruneJob(systemModule *system.Module, settingsService *settings.SettingsService, notificationService *notification.NotificationService) *ScheduledPruneJob {
+func NewScheduledPruneJob(systemService *system.SystemService, settingsService *settings.SettingsService, notificationService *notification.NotificationService) *ScheduledPruneJob {
return &ScheduledPruneJob{
- systemService: systemModule.Service,
+ systemService: systemService,
settingsService: settingsService,
notificationService: notificationService,
}
@@ -72,7 +71,7 @@ func (j *ScheduledPruneJob) Run(ctx context.Context) (schedulertypes.Outcome, er
"build_cache", req.BuildCache,
)
- result, started, err := j.systemService.PruneAll(ctx, "0", req)
+ result, started, err := j.systemService.PruneScheduled(ctx, "0", req)
if err != nil {
slog.ErrorContext(ctx, "scheduled prune run failed", "error", err)
return schedulertypes.Outcome{}, err
@@ -185,6 +184,6 @@ func buildScheduledBuildCachePruneOptionsInternal(ctx context.Context, settingsS
}
}
-func (j *ScheduledPruneJob) Reconcile(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
- return jobcontext.ConfirmedTarget(previous, "scheduled-prune"), nil
+func (j *ScheduledPruneJob) Reconcile(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ return j.systemService.ReconcileScheduledPrune(ctx, previous)
}
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index 57436a632..93105596c 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -3,407 +3,215 @@ package scheduler
import (
"context"
"log/slog"
- "maps"
- "slices"
"sync"
"time"
"emperror.dev/errors"
- "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
- "github.com/robfig/cron/v3"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
-// cronScheduleParser is the shared parser for all cron settings: six fields
-// with seconds, plus @-descriptors. The image update watcher parses its poll
-// schedule with the same spec so Jobs-UI cron values behave identically.
-var cronScheduleParser = scheduleutil.Parser()
-
-var errJobSchedulerStoppedInternal = errors.Sentinel("job scheduler stopped")
-
-type schedulerStateInternal struct {
- jobs []schedulertypes.Job
- jobsByID map[string]schedulertypes.Job
- watchers map[string]schedulertypes.BusWatcher
- allWatchers map[string]schedulertypes.BusWatcher
- runners map[string]*actors.Runner
- entryIDs map[string]cron.EntryID
- schedules map[string]string
- supervisors map[string]*watcherSupervisorInternal
- stopping bool
-}
+var (
+ cronScheduleParser = scheduleutil.Parser()
+ errJobSchedulerStoppedInternal = errors.Sentinel("job scheduler stopped")
+)
type watcherRegistrationInternal struct {
- watcher schedulertypes.BusWatcher
- runner *actors.Runner
+ watcher st.BusWatcher
+ done chan struct{}
}
-
-type schedulerShutdownInternal struct {
- cronDone <-chan struct{}
- watchers []watcherRegistrationInternal
-}
-
-// jobSchedulerInternal serializes its control plane through one shared actor executor.
-// robfig/cron remains the timing engine and invokes job bodies outside the actor.
type jobSchedulerInternal struct {
- cron *cron.Cron
- parser cron.Parser
- context context.Context
- location *time.Location
- runtime *actors.Runtime
- state *actors.State[schedulerStateInternal]
- cancel context.CancelFunc
- dispatchMu sync.RWMutex
- dispatcher schedulertypes.Dispatcher
+ mu sync.RWMutex
+ jobsByID map[string]st.Job
+ watchers map[string]st.BusWatcher
+ allWatchers map[string]watcherRegistrationInternal
+ supervisors map[string]*watcherSupervisorInternal
+ context context.Context
+ cancel context.CancelFunc
+ location *time.Location
+ coordinator *runs.Coordinator
+ stopping bool
+ started bool
}
-// NewJobScheduler creates an actor-owned scheduler control plane.
-func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *time.Location) (schedulertypes.JobScheduler, error) {
- if ctx == nil {
- return nil, errors.New("job scheduler context unavailable")
- }
- if runtime == nil {
- return nil, errors.New("actor runtime unavailable")
+func NewJobScheduler(ctx context.Context, coordinator *runs.Coordinator, location *time.Location) (st.JobScheduler, error) {
+ if ctx == nil || coordinator == nil {
+ return nil, errors.New("scheduler dependencies unavailable")
}
if location == nil {
location = time.UTC
}
- initial := schedulerStateInternal{
- jobs: []schedulertypes.Job{},
- jobsByID: make(map[string]schedulertypes.Job),
- watchers: make(map[string]schedulertypes.BusWatcher),
- allWatchers: make(map[string]schedulertypes.BusWatcher),
- runners: make(map[string]*actors.Runner),
- entryIDs: make(map[string]cron.EntryID),
- schedules: make(map[string]string),
- supervisors: make(map[string]*watcherSupervisorInternal),
- }
- state, err := actors.NewState(ctx, runtime, "scheduler", "control-plane", 3, initial, func(value schedulerStateInternal) schedulerStateInternal {
- value.jobs = slices.Clone(value.jobs)
- value.jobsByID = maps.Clone(value.jobsByID)
- value.watchers = maps.Clone(value.watchers)
- value.allWatchers = maps.Clone(value.allWatchers)
- value.runners = maps.Clone(value.runners)
- value.entryIDs = maps.Clone(value.entryIDs)
- value.schedules = maps.Clone(value.schedules)
- value.supervisors = maps.Clone(value.supervisors)
- return value
- })
- if err != nil {
- return nil, err
- }
lifetime, cancel := context.WithCancel(ctx)
- parser := cronScheduleParser
- scheduler := &jobSchedulerInternal{
- cron: cron.New(cron.WithParser(parser), cron.WithLocation(location)),
- parser: parser,
- context: lifetime,
- cancel: cancel,
- location: location,
- runtime: runtime,
- state: state,
- }
- slog.InfoContext(ctx, "Initializing job scheduler", "timezone", location.String())
- return scheduler, nil
+ return &jobSchedulerInternal{context: lifetime, cancel: cancel, location: location, coordinator: coordinator, jobsByID: map[string]st.Job{}, watchers: map[string]st.BusWatcher{}, allWatchers: map[string]watcherRegistrationInternal{}, supervisors: map[string]*watcherSupervisorInternal{}}, nil
}
-// RegisterJob records a static job to be scheduled when StartScheduler runs.
-func (js *jobSchedulerInternal) RegisterJob(job schedulertypes.Job) error {
+func (js *jobSchedulerInternal) RegisterJob(job st.Job) error {
if job == nil {
return errors.New("scheduler job unavailable")
}
- return js.state.Apply(js.context, "register scheduler job", func(_ context.Context, state *schedulerStateInternal) error {
- if state.stopping {
- return errJobSchedulerStoppedInternal
- }
- state.jobs = append(state.jobs, job)
- state.jobsByID[job.Name()] = job
- return nil
- })
+ js.mu.Lock()
+ defer js.mu.Unlock()
+ if js.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ js.jobsByID[job.Name()] = job
+ return nil
}
-// RegisterBusWatcher starts a continuous watcher through a shared actor runner.
-func (js *jobSchedulerInternal) RegisterBusWatcher(watcher schedulertypes.BusWatcher, canRunManually bool) error {
- if watcher == nil {
- return errors.New("scheduler bus watcher unavailable")
+func (js *jobSchedulerInternal) RegisterBusWatcher(watcher st.BusWatcher, manual bool) error {
+ js.mu.Lock()
+ defer js.mu.Unlock()
+ if js.stopping {
+ return errJobSchedulerStoppedInternal
}
- return js.state.Apply(js.context, "register scheduler bus watcher", func(_ context.Context, state *schedulerStateInternal) error {
- if state.stopping {
- return errJobSchedulerStoppedInternal
- }
- if _, exists := state.allWatchers[watcher.Name()]; exists {
- return errors.New("watcher already registered")
- }
- supervisor := newWatcherSupervisorInternal(watcher)
- runner, err := actors.NewRunner(js.context, js.runtime, "scheduler-watchers", watcher.Name(), "bus watcher "+watcher.Name(), 3, supervisor.runInternal) //nolint:contextcheck // watcher lifetime belongs to the scheduler, not one state-actor receiver incarnation.
- if err != nil {
- return err
- }
- state.runners[watcher.Name()] = runner
- state.supervisors[watcher.Name()] = supervisor
- state.allWatchers[watcher.Name()] = watcher
- if canRunManually {
- state.watchers[watcher.Name()] = watcher
+ if _, found := js.allWatchers[watcher.Name()]; found {
+ return errors.New("watcher already registered")
+ }
+ supervisor := newWatcherSupervisorInternal(watcher)
+ done := make(chan struct{})
+ js.supervisors[watcher.Name()] = supervisor
+ js.allWatchers[watcher.Name()] = watcherRegistrationInternal{watcher: watcher, done: done}
+ if manual {
+ js.watchers[watcher.Name()] = watcher
+ }
+ go func() {
+ defer close(done)
+ if err := supervisor.runInternal(js.context); err != nil && js.context.Err() == nil {
+ slog.ErrorContext(js.context, "Watcher stopped", "name", watcher.Name(), "error", err)
}
- return nil
- })
+ }()
+ return nil
}
-// RunBusWatcherNow runs a watcher through its serialized manual path.
-func (js *jobSchedulerInternal) RunBusWatcherNow(ctx context.Context, watcherID string) error {
- snapshot, ok := js.state.Load()
- if !ok {
- return errors.New("job scheduler state unavailable")
- }
- watcher, ok := snapshot.watchers[watcherID]
- if !ok {
- return errors.Errorf("bus watcher %s is not manually runnable", watcherID)
+func (js *jobSchedulerInternal) RunBusWatcherNow(ctx context.Context, id string) error {
+ js.mu.RLock()
+ watcher, found := js.watchers[id]
+ js.mu.RUnlock()
+ if !found {
+ return errors.New("watcher is not manually runnable")
}
return watcher.RunNow(ctx)
}
-func (js *jobSchedulerInternal) GetJob(jobID string) (schedulertypes.Job, bool) {
- snapshot, ok := js.state.Load()
- if !ok {
- return nil, false
- }
- job, ok := snapshot.jobsByID[jobID]
+func (js *jobSchedulerInternal) GetJob(id string) (st.Job, bool) {
+ js.mu.RLock()
+ defer js.mu.RUnlock()
+ job, ok := js.jobsByID[id]
return job, ok
}
-
-// GetJobRuntimeState returns the schedule currently installed for a registered job.
-func (js *jobSchedulerInternal) GetJobRuntimeState(jobID string) (schedulertypes.JobRuntimeState, bool) {
- snapshot, ok := js.state.Load()
- if !ok {
- return schedulertypes.JobRuntimeState{}, false
- }
- if _, ok := snapshot.jobsByID[jobID]; !ok {
- return schedulertypes.JobRuntimeState{}, false
- }
-
- state := schedulertypes.JobRuntimeState{Schedule: snapshot.schedules[jobID]}
- entryID, ok := snapshot.entryIDs[jobID]
- if !ok {
- return state, true
- }
- entry := js.cron.Entry(entryID)
- if entry.ID == 0 {
- return state, true
+func (js *jobSchedulerInternal) HasJob(id string) bool { _, ok := js.GetJob(id); return ok }
+func (js *jobSchedulerInternal) GetJobRuntimeState(id string) (st.JobRuntimeState, bool) {
+ if !js.HasJob(id) {
+ return st.JobRuntimeState{}, false
}
- state.Scheduled = true
- nextRun := entry.Next
- if nextRun.IsZero() && entry.Schedule != nil {
- nextRun = entry.Schedule.Next(time.Now().In(js.location))
+ record, err := js.coordinator.ScheduleState(js.context, id)
+ if err != nil {
+ return st.JobRuntimeState{}, true
}
- if !nextRun.IsZero() {
- state.NextRun = new(nextRun)
+ state := st.JobRuntimeState{Schedule: record.Schedule, Scheduled: !record.NextRun.IsZero()}
+ if state.Scheduled {
+ state.NextRun = new(record.NextRun)
}
return state, true
}
-// HasJob reports whether a job with the given name is registered.
-func (js *jobSchedulerInternal) HasJob(jobID string) bool {
- snapshot, ok := js.state.Load()
- if !ok {
- return false
- }
- _, ok = snapshot.jobsByID[jobID]
- return ok
-}
-
-// StartScheduler installs static schedules and starts cron.
func (js *jobSchedulerInternal) StartScheduler() error {
- return js.state.Apply(js.context, "start job scheduler", func(actorCtx context.Context, state *schedulerStateInternal) error {
- if state.stopping {
- return errJobSchedulerStoppedInternal
- }
- for _, job := range state.jobs {
- if err := js.upsertJobInternal(actorCtx, state, job); err != nil {
- slog.ErrorContext(actorCtx, "Failed to schedule job; continuing scheduler startup", "name", job.Name(), "error", err)
- }
- }
- js.cron.Start()
- return nil
- })
-}
-
-// AddJob registers and schedules a job at runtime.
-func (js *jobSchedulerInternal) AddJob(ctx context.Context, job schedulertypes.Job) error {
- return js.state.Apply(ctx, "add scheduler job", func(_ context.Context, state *schedulerStateInternal) error {
- if state.stopping {
- return errJobSchedulerStoppedInternal
- }
- return js.upsertJobInternal(ctx, state, job)
- })
-}
-
-// RemoveJob unschedules and forgets a job. Actor failures are logged because
-// DynamicScheduler preserves the historical no-result removal contract.
-func (js *jobSchedulerInternal) RemoveJob(ctx context.Context, jobName string) {
- err := js.state.Apply(ctx, "remove scheduler job", func(_ context.Context, state *schedulerStateInternal) error {
- if state.stopping {
- return errJobSchedulerStoppedInternal
- }
- if entryID, ok := state.entryIDs[jobName]; ok {
- js.cron.Remove(entryID)
- delete(state.entryIDs, jobName)
- }
- delete(state.jobsByID, jobName)
- delete(state.schedules, jobName)
- for index, job := range state.jobs {
- if job.Name() == jobName {
- state.jobs = append(state.jobs[:index], state.jobs[index+1:]...)
- break
- }
- }
- slog.DebugContext(ctx, "Job removed", "name", jobName)
- return nil
- })
- if err != nil && !errors.Is(err, errJobSchedulerStoppedInternal) {
- slog.ErrorContext(ctx, "Failed to remove scheduler job", "name", jobName, "error", err)
- }
-}
-
-func (js *jobSchedulerInternal) RescheduleJob(ctx context.Context, job schedulertypes.Job) error {
- return js.state.Apply(ctx, "reschedule job", func(_ context.Context, state *schedulerStateInternal) error {
- if state.stopping {
- return errJobSchedulerStoppedInternal
- }
- return js.upsertJobInternal(ctx, state, job)
- })
-}
-
-func (js *jobSchedulerInternal) GetLocation() *time.Location {
- return js.location
-}
-
-// Stop fences the control plane, stops cron, and joins every actor-owned watcher.
-func (js *jobSchedulerInternal) Stop(ctx context.Context) error {
- var shutdown schedulerShutdownInternal
- err := js.state.Apply(ctx, "stop job scheduler", func(_ context.Context, state *schedulerStateInternal) error {
- if state.stopping {
- return errJobSchedulerStoppedInternal
- }
- state.stopping = true
- registrations := make([]watcherRegistrationInternal, 0, len(state.runners))
- for name, runner := range state.runners {
- registrations = append(registrations, watcherRegistrationInternal{watcher: state.allWatchers[name], runner: runner})
- }
- shutdown = schedulerShutdownInternal{cronDone: js.cron.Stop().Done(), watchers: registrations}
- return nil
- })
- if errors.Is(err, errJobSchedulerStoppedInternal) {
- return js.state.Stop(ctx)
- }
- var stopErr error
+ js.mu.Lock()
+ if js.stopping {
+ js.mu.Unlock()
+ return errJobSchedulerStoppedInternal
+ }
+ js.started = true
+ js.mu.Unlock()
+ records, err := js.coordinator.Records(js.context)
if err != nil {
- if snapshot, ok := js.state.Load(); ok && snapshot.stopping {
- return nil
- }
- stopErr = err
- shutdown.cronDone = js.cron.Stop().Done()
- if snapshot, ok := js.state.Load(); ok {
- shutdown.watchers = make([]watcherRegistrationInternal, 0, len(snapshot.runners))
- for name, runner := range snapshot.runners {
- shutdown.watchers = append(shutdown.watchers, watcherRegistrationInternal{watcher: snapshot.allWatchers[name], runner: runner})
+ return err
+ }
+ for _, record := range records {
+ if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) {
+ if err := js.coordinator.Checkpoint(js.context, record.JobID, "", time.Time{}); err != nil {
+ return err
}
}
}
-
- js.cancel()
- for _, registration := range shutdown.watchers {
- stopErr = errors.Combine(stopErr, registration.runner.Stop(ctx))
- if stopper, ok := registration.watcher.(schedulertypes.StoppableBusWatcher); ok {
- stopErr = errors.Combine(stopErr, stopper.Stop(ctx))
+ var schedulingErr error
+ for _, job := range js.ListRegisteredJobs() {
+ if err := js.installInternal(js.context, job); err != nil {
+ schedulingErr = errors.Combine(schedulingErr, errors.WrapIf(err, "schedule "+job.Name()))
}
}
- select {
- case <-shutdown.cronDone:
- case <-ctx.Done():
- stopErr = errors.Combine(stopErr, ctx.Err())
- }
- return errors.Combine(stopErr, js.state.Stop(ctx))
+ return schedulingErr
}
-func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *schedulerStateInternal, job schedulertypes.Job) error {
- if job == nil {
- return errors.New("scheduler job unavailable")
- }
- jobName := job.Name()
- previousSchedule := state.schedules[jobName]
- previousEntryID, hadPreviousEntry := state.entryIDs[jobName]
+func (js *jobSchedulerInternal) installInternal(ctx context.Context, job st.Job) error {
+ next := time.Time{}
schedule := job.Schedule(ctx)
-
- shouldSchedule := true
- if conditionalJob, ok := job.(schedulertypes.ConditionalJob); ok {
- shouldSchedule = conditionalJob.ShouldSchedule(ctx)
+ should := true
+ if conditional, ok := job.(st.ConditionalJob); ok {
+ should = conditional.ShouldSchedule(ctx)
}
-
- var (
- parsedSchedule cron.Schedule
- entryID cron.EntryID
- nextRun *time.Time
- )
- if shouldSchedule {
- var err error
- parsedSchedule, err = js.parser.Parse(schedule)
+ if should {
+ parsed, err := cronScheduleParser.Parse(schedule)
if err != nil {
return err
}
- } else {
- slog.DebugContext(ctx, "Job disabled; not scheduling", "name", jobName)
+ next = parsed.Next(time.Now().In(js.location))
}
-
- if shouldSchedule {
- if err := js.checkpointInternal(ctx, jobName, schedule, parsedSchedule.Next(time.Now().In(js.location))); err != nil {
- return err
- }
+ if !should {
+ schedule = ""
}
- if hadPreviousEntry {
- js.cron.Remove(previousEntryID)
- delete(state.entryIDs, jobName)
+ return js.coordinator.Checkpoint(ctx, job.Name(), schedule, next)
+}
+
+func (js *jobSchedulerInternal) AddJob(ctx context.Context, job st.Job) error {
+ if err := js.RegisterJob(job); err != nil {
+ return err
}
- if shouldSchedule {
- entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule)
- state.entryIDs[jobName] = entryID
+ js.mu.RLock()
+ started := js.started
+ js.mu.RUnlock()
+ if !started {
+ return nil
}
- state.jobsByID[jobName] = job
- state.schedules[jobName] = schedule
+ return js.installInternal(ctx, job)
+}
+
+func (js *jobSchedulerInternal) RescheduleJob(ctx context.Context, job st.Job) error {
+ return js.AddJob(ctx, job)
+}
- if previousSchedule == "" && shouldSchedule {
- slog.InfoContext(ctx, "Starting Job", "name", jobName, "schedule", schedule)
- } else if previousSchedule != schedule || hadPreviousEntry != shouldSchedule {
- var nextRunValue any
- if nextRun != nil {
- nextRunValue = *nextRun
+func (js *jobSchedulerInternal) RemoveJob(ctx context.Context, id string) {
+ js.mu.Lock()
+ delete(js.jobsByID, id)
+ started := js.started
+ js.mu.Unlock()
+ if started {
+ if err := js.coordinator.Checkpoint(ctx, id, "", time.Time{}); err != nil {
+ slog.ErrorContext(ctx, "Unschedule failed", "job", id, "error", err)
}
- slog.InfoContext(ctx, "Job rescheduled", "name", jobName, "previousSchedule", previousSchedule, "newSchedule", schedule, "nextRun", nextRunValue)
}
- slog.DebugContext(ctx, "Job scheduled", "name", jobName, "scheduled", shouldSchedule, "contextCanceled", ctx.Err() != nil)
- return nil
}
-
-func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {
- entryID := js.cron.Schedule(parsedSchedule, cron.FuncJob(func() {
- defer utils.RecoverToError(nil, "scheduled job admission", "name", job.Name())
- if _, err := js.Submit(js.context, schedulertypes.Request{JobID: job.Name(), EnvironmentID: "0", Trigger: "scheduled"}); err != nil {
- slog.ErrorContext(js.context, "Failed to queue scheduled job", "name", job.Name(), "error", err)
- return
+func (js *jobSchedulerInternal) GetLocation() *time.Location { return js.location }
+func (js *jobSchedulerInternal) Stop(ctx context.Context) error {
+ js.mu.Lock()
+ js.stopping = true
+ js.cancel()
+ watchers := make([]watcherRegistrationInternal, 0, len(js.allWatchers))
+ for _, w := range js.allWatchers {
+ watchers = append(watchers, w)
+ }
+ js.mu.Unlock()
+ var err error
+ for _, w := range watchers {
+ select {
+ case <-w.done:
+ case <-ctx.Done():
+ return errors.Combine(err, ctx.Err())
}
- if err := js.checkpointInternal(js.context, job.Name(), schedule, parsedSchedule.Next(time.Now().In(js.location))); err != nil {
- slog.ErrorContext(js.context, "Failed to checkpoint scheduled job", "name", job.Name(), "error", err)
+ if stop, ok := w.watcher.(st.StoppableBusWatcher); ok {
+ err = errors.Combine(err, stop.Stop(ctx))
}
- }))
-
- entry := js.cron.Entry(entryID)
- nextRun := entry.Next
- if nextRun.IsZero() && entry.Schedule != nil {
- nextRun = entry.Schedule.Next(time.Now().In(js.location))
- }
- if nextRun.IsZero() {
- return entryID, nil
}
- return entryID, new(nextRun)
+ return err
}
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index e73c67240..1205fac7b 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -107,22 +107,23 @@ func TestJobScheduler_StartScheduler_SkipsDisabledConditionalJobs(t *testing.T)
require.NoError(t, js.RegisterJob(job))
require.NoError(t, js.StartScheduler())
- defer js.cron.Stop()
state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
require.False(t, state.Scheduled)
- require.Empty(t, js.cron.Entries())
+ record, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.NoError(t, err)
+ require.True(t, record.NextRun.IsZero())
}
-func TestJobScheduler_StartScheduler_ContinuesAfterInvalidJobSchedule(t *testing.T) {
+func TestJobScheduler_StartScheduler_ReportsInvalidJobSchedule(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
invalid := &testSchedulerJob{name: "invalid-startup-job", schedule: "not a cron schedule"}
valid := &testSchedulerJob{name: "valid-startup-job", schedule: "*/1 * * * * *"}
require.NoError(t, js.RegisterJob(invalid))
require.NoError(t, js.RegisterJob(valid))
- require.NoError(t, js.StartScheduler())
+ require.Error(t, js.StartScheduler())
invalidState, ok := js.GetJobRuntimeState(invalid.Name())
require.True(t, ok)
@@ -130,7 +131,7 @@ func TestJobScheduler_StartScheduler_ContinuesAfterInvalidJobSchedule(t *testing
validState, ok := js.GetJobRuntimeState(valid.Name())
require.True(t, ok)
require.True(t, validState.Scheduled)
- require.Len(t, js.cron.Entries(), 1)
+ requireScheduledJobInternal(t, js, valid.Name())
}
func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {
@@ -213,6 +214,7 @@ func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {
func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ require.NoError(t, js.StartScheduler())
enabled := true
job := &conditionalTestSchedulerJob{
@@ -234,11 +236,14 @@ func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {
state, ok = js.GetJobRuntimeState(job.Name())
require.True(t, ok)
require.False(t, state.Scheduled)
- require.Empty(t, js.cron.Entries())
+ record, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.NoError(t, err)
+ require.True(t, record.NextRun.IsZero())
}
func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ require.NoError(t, js.StartScheduler())
enabled := false
job := &conditionalTestSchedulerJob{
@@ -260,7 +265,7 @@ func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {
state, ok = js.GetJobRuntimeState(job.Name())
require.True(t, ok)
require.True(t, state.Scheduled)
- require.Len(t, js.cron.Entries(), 1)
+ requireScheduledJobInternal(t, js, job.Name())
}
func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {
@@ -273,12 +278,11 @@ func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {
require.NoError(t, js.RegisterJob(job))
require.NoError(t, js.StartScheduler())
- defer js.cron.Stop()
state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
require.True(t, state.Scheduled)
- require.Len(t, js.cron.Entries(), 1)
+ requireScheduledJobInternal(t, js, job.Name())
}
func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {
@@ -297,8 +301,7 @@ func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {
}
require.NoError(t, js.RescheduleJob(runCtx, job))
- js.cron.Start()
- defer js.cron.Stop()
+ require.NoError(t, js.StartScheduler())
select {
case err := <-runErrCh:
@@ -331,8 +334,7 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T
}
require.NoError(t, js.RescheduleJob(lifecycleCtx, job))
- js.cron.Start()
- defer js.cron.Stop()
+ require.NoError(t, js.StartScheduler())
select {
case <-startedCh:
@@ -351,19 +353,23 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T
func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ require.NoError(t, js.StartScheduler())
job := &testSchedulerJob{name: "dyn-upsert", schedule: "*/5 * * * * *"}
require.NoError(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
- require.Len(t, js.cron.Entries(), 1)
- firstEntry := js.cron.Entries()[0].ID
+ requireScheduledJobInternal(t, js, job.Name())
+ firstState, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.NoError(t, err)
// Re-adding with a changed schedule (e.g. a new sync interval) must replace the
// existing cron entry, not leak a second one that keeps firing forever.
job.schedule = "*/10 * * * * *"
require.NoError(t, js.AddJob(context.Background(), job))
- require.Len(t, js.cron.Entries(), 1)
- require.NotEqual(t, firstEntry, js.cron.Entries()[0].ID)
+ requireScheduledJobInternal(t, js, job.Name())
+ nextState, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.NoError(t, err)
+ require.Greater(t, nextState.Generation, firstState.Generation)
state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
@@ -374,18 +380,22 @@ func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {
func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ require.NoError(t, js.StartScheduler())
job := &testSchedulerJob{name: "dyn-invalid-reschedule", schedule: "*/5 * * * * *"}
require.NoError(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
- require.Len(t, js.cron.Entries(), 1)
- firstEntry := js.cron.Entries()[0].ID
+ requireScheduledJobInternal(t, js, job.Name())
+ firstState, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.NoError(t, err)
job.schedule = "not a cron schedule"
require.Error(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
- require.Equal(t, firstEntry, js.cron.Entries()[0].ID)
- require.Len(t, js.cron.Entries(), 1)
+ nextState, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.NoError(t, err)
+ require.Equal(t, firstState.Generation, nextState.Generation)
+ requireScheduledJobInternal(t, js, job.Name())
state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
@@ -395,6 +405,7 @@ func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {
func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ require.NoError(t, js.StartScheduler())
// Removing an unknown job must be a safe no-op (e.g. deleting a sync that never
// had auto-sync enabled).
@@ -403,15 +414,18 @@ func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
job := &testSchedulerJob{name: "dyn-remove", schedule: "*/5 * * * * *"}
require.NoError(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
- require.Len(t, js.cron.Entries(), 1)
+ requireScheduledJobInternal(t, js, job.Name())
js.RemoveJob(context.Background(), job.Name())
require.False(t, js.HasJob(job.Name()))
- require.Empty(t, js.cron.Entries())
+ record, err := js.coordinator.ScheduleState(t.Context(), job.Name())
+ require.NoError(t, err)
+ require.True(t, record.NextRun.IsZero())
}
func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ require.NoError(t, js.StartScheduler())
job := &schedulertypes.GenericJob{
JobName: "generic-dyn",
@@ -420,7 +434,7 @@ func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T)
}
require.NoError(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
- require.Len(t, js.cron.Entries(), 1)
+ requireScheduledJobInternal(t, js, job.Name())
}
func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {
@@ -468,7 +482,7 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {
}
stopDone := make(chan error, 1)
- go func() { stopDone <- js.Stop(context.Background()) }()
+ go func() { stopDone <- stopJobSchedulerForTestInternal(context.Background(), js) }()
select {
case err := <-stopDone:
@@ -489,3 +503,11 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {
require.FailNow(t, "scheduler did not stop after the running job finished")
}
}
+
+func requireScheduledJobInternal(t *testing.T, js *jobSchedulerInternal, id string) {
+ t.Helper()
+ state, found := js.GetJobRuntimeState(id)
+ require.True(t, found)
+ require.True(t, state.Scheduled)
+ require.NotNil(t, state.NextRun)
+}
diff --git a/backend/pkg/scheduler/supervision.go b/backend/pkg/scheduler/supervision.go
index f642279b1..2af61d7a4 100644
--- a/backend/pkg/scheduler/supervision.go
+++ b/backend/pkg/scheduler/supervision.go
@@ -105,44 +105,18 @@ func (s *watcherSupervisorInternal) startInternal(ctx context.Context) (panicked
return false, s.watcher.Start(ctx)
}
-func (js *jobSchedulerInternal) SetDispatcher(dispatcher schedulertypes.Dispatcher) {
- js.dispatchMu.Lock()
- defer js.dispatchMu.Unlock()
- js.dispatcher = dispatcher
-}
-
-func (js *jobSchedulerInternal) dispatcherInternal() schedulertypes.Dispatcher {
- js.dispatchMu.RLock()
- defer js.dispatchMu.RUnlock()
- return js.dispatcher
-}
-
func (js *jobSchedulerInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
if js.context.Err() != nil {
return schedulertypes.Run{}, errJobSchedulerStoppedInternal
}
- dispatcher := js.dispatcherInternal()
- if dispatcher == nil {
- return schedulertypes.Run{}, errors.New("durable job dispatcher unavailable")
- }
- return dispatcher.Submit(ctx, request)
-}
-
-func (js *jobSchedulerInternal) checkpointInternal(ctx context.Context, jobID, schedule string, next time.Time) error {
- dispatcher := js.dispatcherInternal()
- if dispatcher == nil {
- return errors.New("durable job dispatcher unavailable")
- }
- return dispatcher.Checkpoint(ctx, jobID, schedule, next)
+ return js.coordinator.Submit(ctx, request)
}
func (js *jobSchedulerInternal) ListRegisteredJobs() []schedulertypes.Job {
- snapshot, ok := js.state.Load()
- if !ok {
- return nil
- }
- jobs := make([]schedulertypes.Job, 0, len(snapshot.jobsByID))
- for _, job := range snapshot.jobsByID {
+ js.mu.RLock()
+ defer js.mu.RUnlock()
+ jobs := make([]schedulertypes.Job, 0, len(js.jobsByID))
+ for _, job := range js.jobsByID {
jobs = append(jobs, job)
}
return jobs
@@ -152,11 +126,13 @@ func (js *jobSchedulerInternal) RestartWatcher(ctx context.Context, id string) e
if err := ctx.Err(); err != nil {
return err
}
- snapshot, ok := js.state.Load()
- if !ok || snapshot.stopping {
+ js.mu.RLock()
+ supervisor, ok := js.supervisors[id]
+ stopping := js.stopping
+ js.mu.RUnlock()
+ if stopping {
return errJobSchedulerStoppedInternal
}
- supervisor, ok := snapshot.supervisors[id]
if !ok {
return errors.New("watcher not found")
}
@@ -173,11 +149,9 @@ func (js *jobSchedulerInternal) RestartWatcher(ctx context.Context, id string) e
}
func (js *jobSchedulerInternal) WatcherHealth(id string) (schedulertypes.WorkerHealth, bool) {
- snapshot, ok := js.state.Load()
- if !ok {
- return schedulertypes.WorkerHealth{}, false
- }
- supervisor, ok := snapshot.supervisors[id]
+ js.mu.RLock()
+ supervisor, ok := js.supervisors[id]
+ js.mu.RUnlock()
if !ok {
return schedulertypes.WorkerHealth{}, false
}
diff --git a/backend/pkg/scheduler/volume_helper_reaper_job.go b/backend/pkg/scheduler/volume_helper_reaper_job.go
index 1813bf66e..8a58d32ce 100644
--- a/backend/pkg/scheduler/volume_helper_reaper_job.go
+++ b/backend/pkg/scheduler/volume_helper_reaper_job.go
@@ -28,9 +28,9 @@ type PruningVolumeHelperJob struct {
settingsService *settings.SettingsService
}
-func NewPruningVolumeHelperJob(volumeModule *volume.Module, settingsService *settings.SettingsService) *PruningVolumeHelperJob {
+func NewPruningVolumeHelperJob(volumeService *volume.VolumeService, settingsService *settings.SettingsService) *PruningVolumeHelperJob {
return &PruningVolumeHelperJob{
- volumeService: volumeModule.Service(),
+ volumeService: volumeService,
settingsService: settingsService,
}
}
diff --git a/backend/pkg/utils/concurrency/background.go b/backend/pkg/utils/concurrency/background.go
new file mode 100644
index 000000000..fce6e3807
--- /dev/null
+++ b/backend/pkg/utils/concurrency/background.go
@@ -0,0 +1,73 @@
+package concurrency
+
+import (
+ "context"
+ "crypto/rand"
+ "log/slog"
+ "math/big"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+)
+
+// StartSupervised starts a background operation and returns its cancel-and-join function.
+func StartSupervised(ctx context.Context, label string, run func(context.Context) error) (func(context.Context) error, error) {
+ if ctx == nil || run == nil {
+ return nil, errors.New("background operation dependencies unavailable")
+ }
+ runCtx, cancel := context.WithCancel(ctx)
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ delay := 5 * time.Second
+ unexpected := 0
+ for runCtx.Err() == nil {
+ started := time.Now()
+ var runErr error
+ panicked := true
+ func() { defer utils.RecoverToError(&runErr, label); runErr = run(runCtx); panicked = false }()
+ if runCtx.Err() != nil {
+ return
+ }
+ if panicked || runErr == nil {
+ unexpected++
+ }
+ if runErr == nil {
+ runErr = errors.New("background operation exited unexpectedly")
+ }
+ if time.Since(started) > 5*time.Minute {
+ delay = 5 * time.Second
+ unexpected = 0
+ }
+ if unexpected >= 3 {
+ slog.ErrorContext(runCtx, "Background operation requires attention", "label", label, "error", runErr)
+ return
+ }
+ jitter, err := rand.Int(rand.Reader, big.NewInt(int64(delay/2)))
+ if err != nil {
+ slog.ErrorContext(runCtx, "Failed to schedule background retry", "label", label, "error", err)
+ return
+ }
+ wait := delay/2 + time.Duration(jitter.Int64())
+ slog.WarnContext(runCtx, "Background operation failed; retrying", "label", label, "error", runErr, "retryIn", wait)
+ timer := time.NewTimer(wait)
+ select {
+ case <-runCtx.Done():
+ timer.Stop()
+ return
+ case <-timer.C:
+ }
+ delay = min(delay*2, 5*time.Minute)
+ }
+ }()
+ return func(stopCtx context.Context) error {
+ cancel()
+ select {
+ case <-done:
+ return nil
+ case <-stopCtx.Done():
+ return stopCtx.Err()
+ }
+ }, nil
+}
diff --git a/backend/internal/actors/signal.go b/backend/pkg/utils/concurrency/signal.go
similarity index 94%
rename from backend/internal/actors/signal.go
rename to backend/pkg/utils/concurrency/signal.go
index 2fc9c49b4..edfee0279 100644
--- a/backend/internal/actors/signal.go
+++ b/backend/pkg/utils/concurrency/signal.go
@@ -1,4 +1,4 @@
-package actors
+package concurrency
import (
"sync"
@@ -55,7 +55,7 @@ func (s *Signal[T]) Publish(value T) {
for _, callback := range subscribers {
func() {
var err error
- defer utils.RecoverToError(&err, "actor signal subscriber")
+ defer utils.RecoverToError(&err, "signal subscriber")
callback(value)
}()
}
diff --git a/backend/internal/actors/signal_test.go b/backend/pkg/utils/concurrency/signal_test.go
similarity index 97%
rename from backend/internal/actors/signal_test.go
rename to backend/pkg/utils/concurrency/signal_test.go
index ea9d72f16..c6a517ea5 100644
--- a/backend/internal/actors/signal_test.go
+++ b/backend/pkg/utils/concurrency/signal_test.go
@@ -1,4 +1,4 @@
-package actors
+package concurrency
import (
"testing"
diff --git a/backend/internal/actors/snapshot.go b/backend/pkg/utils/concurrency/snapshot.go
similarity index 96%
rename from backend/internal/actors/snapshot.go
rename to backend/pkg/utils/concurrency/snapshot.go
index 4ce5c0c4f..aa9599eb5 100644
--- a/backend/internal/actors/snapshot.go
+++ b/backend/pkg/utils/concurrency/snapshot.go
@@ -1,4 +1,4 @@
-package actors
+package concurrency
import "sync/atomic"
diff --git a/backend/internal/actors/state_map.go b/backend/pkg/utils/concurrency/state_map.go
similarity index 79%
rename from backend/internal/actors/state_map.go
rename to backend/pkg/utils/concurrency/state_map.go
index d4619c43d..3f2e3e57a 100644
--- a/backend/internal/actors/state_map.go
+++ b/backend/pkg/utils/concurrency/state_map.go
@@ -1,4 +1,4 @@
-package actors
+package concurrency
import (
"context"
@@ -11,7 +11,6 @@ import (
// StateMap owns mutable keyed state while publishing immutable snapshots for readers.
type StateMap[K comparable, V any] struct {
- executor *Executor
values map[K]V
snapshot atomic.Pointer[map[K]V]
mu sync.Mutex
@@ -22,25 +21,14 @@ type stateMapValueInternal[V any] struct {
found bool
}
-// NewStateMap creates a locally serialized state map for callers without an actor runtime.
+// NewStateMap creates a locally serialized state map for process-local state.
func NewStateMap[K comparable, V any]() *StateMap[K, V] {
state := &StateMap[K, V]{values: make(map[K]V)}
state.publishInternal()
return state
}
-// NewActorStateMap creates a state map whose mutations use the shared actor runtime.
-func NewActorStateMap[K comparable, V any](ctx context.Context, runtime *Runtime, kind, id string, maxRestarts int32) (*StateMap[K, V], error) {
- executor, err := NewExecutor(ctx, runtime, kind, id, maxRestarts)
- if err != nil {
- return nil, err
- }
- state := &StateMap[K, V]{executor: executor, values: make(map[K]V)}
- state.publishInternal()
- return state, nil
-}
-
-// Get reads one value from the latest immutable snapshot without entering the mailbox.
+// Get reads one value from the latest immutable snapshot without acquiring the mutation lock.
func (s *StateMap[K, V]) Get(key K) (V, bool) {
var zero V
if s == nil {
@@ -75,9 +63,9 @@ func (s *StateMap[K, V]) Apply(ctx context.Context, label string, mutate func(ma
if mutate == nil {
return errors.New("state map mutation unavailable")
}
- _, err := s.ApplyTyped(ctx, label, func(values map[K]V) (NoPayload, bool, error) {
+ _, err := s.ApplyTyped(ctx, label, func(values map[K]V) (struct{}, bool, error) {
changed, err := mutate(values)
- return NoPayload{}, changed, err
+ return struct{}{}, changed, err
})
return err
}
@@ -109,12 +97,12 @@ func (s *StateMap[K, V]) ApplyTyped[R any](
}
return result, err
}
- if s.executor != nil {
- return s.executor.Execute(ctx, label, work, nil)
- }
s.mu.Lock()
defer s.mu.Unlock()
+ if err := ctx.Err(); err != nil {
+ return zero, err
+ }
return work(ctx)
}
@@ -162,14 +150,6 @@ func (s *StateMap[K, V]) Drain(ctx context.Context, label string) ([]V, error) {
return s.RemoveWhere(ctx, label, func(K, V) bool { return true })
}
-// Stop drains actor-backed mutations and joins the shared executor.
-func (s *StateMap[K, V]) Stop(ctx context.Context) error {
- if s == nil || s.executor == nil {
- return nil
- }
- return s.executor.Stop(ctx)
-}
-
func (s *StateMap[K, V]) publishInternal() {
snapshot := make(map[K]V, len(s.values))
maps.Copy(snapshot, s.values)
diff --git a/backend/pkg/utils/concurrency/state_map_test.go b/backend/pkg/utils/concurrency/state_map_test.go
new file mode 100644
index 000000000..7b27c4951
--- /dev/null
+++ b/backend/pkg/utils/concurrency/state_map_test.go
@@ -0,0 +1,58 @@
+package concurrency
+
+import (
+ "testing"
+
+ "emperror.dev/errors"
+ "github.com/stretchr/testify/require"
+)
+
+func TestStateMapSerializesMutationsAndPublishesSnapshotsInternal(t *testing.T) {
+ state := NewStateMap[string, int]()
+
+ require.NoError(t, state.Apply(t.Context(), "store value", func(values map[string]int) (bool, error) {
+ values["answer"] = 42
+ return true, nil
+ }))
+ value, ok := state.Get("answer")
+ require.True(t, ok)
+ require.Equal(t, 42, value)
+}
+
+func TestStateMapPublishesPartialMutationBeforeReturningErrorInternal(t *testing.T) {
+ state := NewStateMap[string, int]()
+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
+ values["removed"] = 1
+ values["retained"] = 2
+ return true, nil
+ }))
+
+ expectedErr := errors.New("mutation failed after change")
+ err := state.Apply(t.Context(), "partial mutation", func(values map[string]int) (bool, error) {
+ delete(values, "removed")
+ return true, expectedErr
+ })
+ require.ErrorIs(t, err, expectedErr)
+ _, found := state.Get("removed")
+ require.False(t, found)
+ retained, found := state.Get("retained")
+ require.True(t, found)
+ require.Equal(t, 2, retained)
+}
+
+func TestStateMapPublishesPartialMutationBeforePropagatingPanicInternal(t *testing.T) {
+ state := NewStateMap[string, int]()
+ require.NoError(t, state.Apply(t.Context(), "seed", func(values map[string]int) (bool, error) {
+ values["removed"] = 1
+ return true, nil
+ }))
+
+ require.Panics(t, func() {
+ _ = state.Apply(t.Context(), "panic after mutation", func(values map[string]int) (bool, error) {
+ delete(values, "removed")
+ panic("mutation panic")
+ })
+ })
+ _, found := state.Get("removed")
+ require.False(t, found)
+}
diff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.go
new file mode 100644
index 000000000..d0e652204
--- /dev/null
+++ b/backend/pkg/utils/francis/provider.go
@@ -0,0 +1,84 @@
+package francis
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "strings"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ dbtypes "github.com/getarcaneapp/arcane/types/v2/database"
+ "github.com/italypaleale/francis/components"
+ "github.com/italypaleale/francis/components/postgres"
+ "github.com/italypaleale/francis/components/sqlite"
+ "github.com/italypaleale/francis/host/local"
+)
+
+func providerOptionInternal(databaseURL string) (local.HostOption, error) {
+ switch {
+ case strings.HasPrefix(databaseURL, "file:"):
+ dsn, err := database.ParseSQLiteConnectionString(databaseURL, dbtypes.SQLiteConnectionOptions{
+ IgnoreForeignKeys: true,
+ JournalMode: "WAL",
+ BusyTimeout: 2500 * time.Millisecond,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return local.WithSQLiteProvider(sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: tablePrefixInternal}), nil
+ case strings.HasPrefix(databaseURL, "postgres"):
+ return local.WithPostgresProvider(postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: tablePrefixInternal}), nil
+ default:
+ return nil, errors.New("unsupported actor database URL")
+ }
+}
+
+// ClearRestoredHosts removes copied live host ownership from an offline restored
+// database. Never call this against a running Arcane database.
+func ClearRestoredHosts(ctx context.Context, databaseURL string) (err error) {
+ cfg := components.NewProviderConfig()
+ cfg.HostHealthCheckDeadline = 90 * time.Second
+ cfg.MaxHosts = 1
+ var provider components.ActorProvider
+ switch {
+ case strings.HasPrefix(databaseURL, "file:"):
+ dsn, dsnErr := database.ParseSQLiteConnectionString(databaseURL, dbtypes.SQLiteConnectionOptions{
+ IgnoreForeignKeys: true,
+ JournalMode: "WAL",
+ BusyTimeout: 2500 * time.Millisecond,
+ })
+ if dsnErr != nil {
+ return dsnErr
+ }
+ provider, err = sqlite.NewSQLiteProvider(slog.Default(), sqlite.SQLiteProviderOptions{ConnectionString: dsn, TablePrefix: tablePrefixInternal}, cfg)
+ case strings.HasPrefix(databaseURL, "postgres"):
+ provider, err = postgres.NewPostgresProvider(slog.Default(), postgres.PostgresProviderOptions{ConnectionString: databaseURL, TablePrefix: tablePrefixInternal}, cfg)
+ default:
+ return errors.New("unsupported restored actor database URL")
+ }
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if closeErr := provider.Close(); closeErr != nil {
+ err = errors.Join(err, fmt.Errorf("close restored actor provider: %w", closeErr))
+ }
+ }()
+ if err = provider.Init(ctx); err != nil {
+ return fmt.Errorf("initialize restored actor provider: %w", err)
+ }
+ hosts, err := provider.ListHosts(ctx)
+ if err != nil {
+ return fmt.Errorf("list restored actor hosts: %w", err)
+ }
+ for _, host := range hosts {
+ err = provider.UnregisterHost(ctx, host.HostID, components.UnregisterHostOpts{})
+ if err != nil && !errors.Is(err, components.ErrHostUnregistered) {
+ return fmt.Errorf("remove restored actor host: %w", err)
+ }
+ }
+ // Expired registrations are excluded by ListHosts and pruned by RegisterHost.
+ return nil
+}
diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.go
new file mode 100644
index 000000000..f5ddbf924
--- /dev/null
+++ b/backend/pkg/utils/francis/runtime.go
@@ -0,0 +1,300 @@
+// Package francis owns the embedded Francis host lifecycle.
+package francis
+
+import (
+ "context"
+ "crypto/hkdf"
+ "crypto/sha256"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net"
+ "strconv"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/italypaleale/francis/actor"
+ "github.com/italypaleale/francis/components"
+ "github.com/italypaleale/francis/host/local"
+ "github.com/quic-go/quic-go"
+ "github.com/quic-go/quic-go/http3"
+)
+
+const tablePrefixInternal = "arcane_francis"
+
+// Runtime starts one durable actor host and publishes its service after readiness.
+// Services must wait for Ready before using Service, including during startup retries.
+type Runtime struct {
+ mu sync.Mutex
+ options []local.HostOption
+ registrations []registrationInternal
+ service *actor.Service
+ ready chan struct{}
+ done chan struct{}
+ cancel context.CancelFunc
+ started bool
+ runError error
+ address string
+}
+
+type registrationInternal struct {
+ actorType string
+ factory actor.Factory
+ options []local.RegisterActorOption
+}
+
+func New(databaseURL, encryptionKey, instanceID, port string, options ...local.HostOption) (*Runtime, error) {
+ if port == "" {
+ port = "3551"
+ }
+ portNumber, err := strconv.Atoi(port)
+ if err != nil || portNumber < 1 || portNumber > 65535 {
+ return nil, errors.New("ACTOR_PORT must be between 1 and 65535")
+ }
+ providerOption, err := providerOptionInternal(databaseURL)
+ if err != nil {
+ return nil, err
+ }
+ address := net.JoinHostPort("127.0.0.1", port)
+ runtime := &Runtime{
+ service: &actor.Service{}, ready: make(chan struct{}), done: make(chan struct{}), address: address,
+ options: []local.HostOption{
+ local.WithAddress(address), providerOption,
+ local.WithLogger(slog.Default().With("scope", "actor-host")),
+ local.WithMaxHosts(1), local.WithHostHealthCheckDeadline(90 * time.Second),
+ local.WithShutdownGracePeriod(10 * time.Second), local.WithAlarmsPollInterval(time.Second),
+ local.WithAlarmsFetchAheadInterval(30 * time.Second), local.WithAlarmsLeaseDuration(180 * time.Second),
+ },
+ }
+ if encryptionKey != "" && instanceID != "" {
+ if err := runtime.ConfigureIdentity(encryptionKey, instanceID); err != nil {
+ return nil, err
+ }
+ }
+ runtime.options = append(runtime.options, options...)
+ return runtime, nil
+}
+
+// ConfigureIdentity binds the final persisted identity before Start.
+func (r *Runtime) ConfigureIdentity(encryptionKey, instanceID string) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if r.started {
+ return errors.New("actor identity cannot change after startup")
+ }
+ if encryptionKey == "" || instanceID == "" {
+ return errors.New("actor host requires the encryption key and instance ID")
+ }
+ psk, err := hkdf.Key(sha256.New, []byte(encryptionKey), nil, "arcane/actors-psk/"+instanceID, 32)
+ if err != nil {
+ return fmt.Errorf("derive actor authentication key: %w", err)
+ }
+ r.options = append(r.options, local.WithRuntimePSKs(psk))
+ return nil
+}
+
+func (r *Runtime) Service() *actor.Service { return r.service }
+func (r *Runtime) Ready() <-chan struct{} { return r.ready }
+
+func (r *Runtime) RegisterActor(actorType string, factory actor.Factory, options ...local.RegisterActorOption) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if r.started {
+ return errors.New("actor factories must be registered before startup")
+ }
+ for _, registration := range r.registrations {
+ if registration.actorType == actorType {
+ return fmt.Errorf("actor type %q is already registered", actorType)
+ }
+ }
+ r.registrations = append(r.registrations, registrationInternal{actorType: actorType, factory: factory, options: append([]local.RegisterActorOption(nil), options...)})
+ return nil
+}
+
+// Start waits for registration and the peer listener. appCtx owns the running host;
+// ctx only limits startup. onFailure must cancel the application without blocking.
+func (r *Runtime) Start(ctx, appCtx context.Context, onFailure func(error)) error {
+ r.mu.Lock()
+ if r.started {
+ r.mu.Unlock()
+ return errors.New("actor host already started")
+ }
+ r.started = true
+ runCtx, cancel := context.WithCancel(appCtx)
+ r.cancel = cancel
+ r.mu.Unlock()
+ var listenerConfig net.ListenConfig
+ listener, err := listenerConfig.ListenPacket(ctx, "udp", r.address)
+ if err != nil {
+ cancel()
+ err = fmt.Errorf("bind actor loopback UDP listener: %w", err)
+ r.finishInternal(err)
+ return err
+ }
+ if err = listener.Close(); err != nil {
+ cancel()
+ r.finishInternal(err)
+ return err
+ }
+ startupCtx, startupCancel := context.WithTimeout(ctx, 120*time.Second)
+ defer startupCancel()
+ // An ephemeral probe socket can claim the port between the preflight and host bind.
+ bindRetries := 0
+
+ for {
+ host, errCh, err := r.runHostInternal(runCtx)
+ if err != nil {
+ cancel()
+ r.finishInternal(err)
+ return err
+ }
+ select {
+ case <-host.Ready():
+ err = waitForPeerInternal(startupCtx, r.address, errCh)
+ if err != nil {
+ if errors.Is(err, syscall.EADDRINUSE) && bindRetries < 3 {
+ <-errCh
+ bindRetries++
+ continue
+ }
+ cancel()
+ r.finishInternal(<-errCh)
+ return err
+ }
+ select {
+ case runErr := <-errCh:
+ cancel()
+ r.finishInternal(runErr)
+ return fmt.Errorf("actor host stopped during startup: %w", runErr)
+ default:
+ }
+ r.publishReadyInternal(runCtx, host, errCh, onFailure)
+ return nil
+ case err = <-errCh:
+ if errors.Is(err, syscall.EADDRINUSE) && bindRetries < 3 {
+ bindRetries++
+ continue
+ }
+ if !errors.Is(err, components.ErrClusterFull) && !errors.Is(err, components.ErrHostAlreadyRegistered) {
+ cancel()
+ r.finishInternal(err)
+ return err
+ }
+ slog.WarnContext(ctx, "Waiting for the previous actor host registration to expire", "error", err)
+ case <-startupCtx.Done():
+ cancel()
+ r.finishInternal(<-errCh)
+ return startupCtx.Err()
+ }
+ timer := time.NewTimer(time.Second)
+ select {
+ case <-timer.C:
+ case <-runCtx.Done():
+ timer.Stop()
+ r.finishInternal(runCtx.Err())
+ return runCtx.Err()
+ case <-startupCtx.Done():
+ timer.Stop()
+ cancel()
+ err = fmt.Errorf("another Arcane process owns this database or its registration has not expired: %w", startupCtx.Err())
+ r.finishInternal(err)
+ return err
+ }
+ }
+}
+
+func (r *Runtime) runHostInternal(ctx context.Context) (*local.Host, chan error, error) {
+ host, err := local.NewHost(r.options...)
+ if err != nil {
+ return nil, nil, err
+ }
+ for _, registration := range r.registrations {
+ if err := host.RegisterActor(registration.actorType, registration.factory, registration.options...); err != nil {
+ // Running a canceled host closes its provider-owned connections.
+ cleanupCtx, cancel := context.WithCancel(ctx)
+ cancel()
+ if cleanupErr := host.Run(cleanupCtx); cleanupErr != nil {
+ return nil, nil, errors.Join(err, fmt.Errorf("clean up unregistered actor host: %w", cleanupErr))
+ }
+ return nil, nil, err
+ }
+ }
+ errors := make(chan error, 1)
+ go func() { errors <- host.Run(ctx) }()
+ return host, errors, nil
+}
+
+func (r *Runtime) publishReadyInternal(ctx context.Context, host *local.Host, runErrors <-chan error, onFailure func(error)) {
+ *r.service = *host.Service()
+ close(r.ready)
+ go func() {
+ err := <-runErrors
+ r.finishInternal(err)
+ if ctx.Err() != nil || onFailure == nil {
+ return
+ }
+ if err == nil {
+ err = errors.New("actor host stopped unexpectedly")
+ }
+ onFailure(err)
+ }()
+}
+
+func (r *Runtime) Stop(ctx context.Context) error {
+ r.mu.Lock()
+ cancel, started := r.cancel, r.started
+ r.mu.Unlock()
+ if !started {
+ return nil
+ }
+ cancel()
+ select {
+ case <-r.done:
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.runError
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+func (r *Runtime) finishInternal(err error) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.runError = err
+ close(r.done)
+}
+
+func waitForPeerInternal(ctx context.Context, address string, runErrors chan error) error {
+ // A TLS rejection proves the loopback listener is serving before shutdown.
+ //nolint:gosec // This readiness probe never sends credentials or application data.
+ tlsConfig := &tls.Config{InsecureSkipVerify: true, NextProtos: []string{http3.NextProtoH3}}
+ for {
+ probeCtx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
+ connection, err := quic.DialAddr(probeCtx, address, tlsConfig, &quic.Config{})
+ cancel()
+ if connection != nil {
+ if err := connection.CloseWithError(0, "readiness probe complete"); err != nil {
+ return fmt.Errorf("close actor readiness probe: %w", err)
+ }
+ return nil
+ }
+ var transportError *quic.TransportError
+ if errors.As(err, &transportError) && transportError.Remote {
+ return nil
+ }
+ timer := time.NewTimer(10 * time.Millisecond)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return ctx.Err()
+ case runErr := <-runErrors:
+ timer.Stop()
+ runErrors <- runErr
+ return fmt.Errorf("actor peer listener stopped before readiness: %w", runErr)
+ case <-timer.C:
+ }
+ }
+}
diff --git a/backend/pkg/utils/francis/runtime_test.go b/backend/pkg/utils/francis/runtime_test.go
new file mode 100644
index 000000000..41bdea34a
--- /dev/null
+++ b/backend/pkg/utils/francis/runtime_test.go
@@ -0,0 +1,73 @@
+package francis
+
+import (
+ "context"
+ "net"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/italypaleale/francis/actor"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSQLiteHostRestartPersistence(t *testing.T) {
+ databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))
+ testRestartPersistenceInternal(t, databaseURL)
+}
+
+func TestPostgresHostRestartPersistence(t *testing.T) {
+ databaseURL := os.Getenv("ARCANE_TEST_POSTGRES_DSN")
+ if databaseURL == "" {
+ t.Skip("ARCANE_TEST_POSTGRES_DSN is not set")
+ }
+ testRestartPersistenceInternal(t, databaseURL)
+}
+
+func testRestartPersistenceInternal(t *testing.T, databaseURL string) {
+ t.Helper()
+ actorID := t.Name() + time.Now().Format("150405.000000000")
+ port := freePortInternal(t)
+ ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
+ defer cancel()
+ first, err := New(databaseURL, "test-encryption-key", "test-instance", port)
+ require.NoError(t, err)
+ service := first.Service()
+ require.NoError(t, first.RegisterActor("test", func(string, *actor.Service) actor.Actor { return struct{}{} }))
+ require.NoError(t, first.Start(ctx, ctx, func(err error) { t.Errorf("unexpected host failure: %v", err) }))
+ t.Cleanup(func() { require.NoError(t, first.Stop(context.WithoutCancel(ctx))) })
+ require.Same(t, service, first.Service())
+ state := struct {
+ Time time.Time
+ Name string
+ }{Time: time.Now().UTC().Truncate(time.Millisecond), Name: "durable"}
+ require.NoError(t, service.SetState(ctx, "test", actorID, state, nil))
+ require.NoError(t, first.Stop(ctx))
+
+ second, err := New(databaseURL, "test-encryption-key", "test-instance", port)
+ require.NoError(t, err)
+ require.NoError(t, second.RegisterActor("test", func(string, *actor.Service) actor.Actor { return struct{}{} }))
+ require.NoError(t, second.Start(ctx, ctx, nil))
+ t.Cleanup(func() { require.NoError(t, second.Stop(context.WithoutCancel(ctx))) })
+ var restored struct {
+ Time time.Time
+ Name string
+ }
+ require.NoError(t, second.Service().GetState(ctx, "test", actorID, &restored))
+ require.Equal(t, state.Name, restored.Name)
+ require.True(t, state.Time.Equal(restored.Time))
+ require.NoError(t, second.Service().DeleteState(ctx, "test", actorID))
+ require.Error(t, second.RegisterActor("late", nil))
+}
+
+func freePortInternal(t *testing.T) string {
+ t.Helper()
+ var lc net.ListenConfig
+ listener, err := lc.ListenPacket(t.Context(), "udp", "127.0.0.1:0")
+ require.NoError(t, err)
+ port := strings.TrimPrefix(listener.LocalAddr().String(), "127.0.0.1:")
+ require.NoError(t, listener.Close())
+ return port
+}
diff --git a/backend/pkg/utils/francis/testing/host.go b/backend/pkg/utils/francis/testing/host.go
new file mode 100644
index 000000000..22e3e5f55
--- /dev/null
+++ b/backend/pkg/utils/francis/testing/host.go
@@ -0,0 +1,62 @@
+// Package testing starts isolated Francis hosts for owning-logic tests.
+package testing
+
+import (
+ "context"
+ "net"
+ "strings"
+ stdtesting "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ "github.com/italypaleale/francis/components/standalone"
+ "github.com/italypaleale/francis/host/local"
+)
+
+// New constructs a host whose factories can be registered before Start.
+// With no database URL it uses an isolated in-memory provider. Pass a temporary
+// SQLite URL or the gated Postgres test URL to exercise durable persistence.
+func New(t stdtesting.TB, databaseURLs ...string) *francis.Runtime {
+ t.Helper()
+ var lc net.ListenConfig
+ listener, err := lc.ListenPacket(t.Context(), "udp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ port := strings.TrimPrefix(listener.LocalAddr().String(), "127.0.0.1:")
+ if err := listener.Close(); err != nil {
+ t.Fatal(err)
+ }
+ databaseURL := "file:actor-test-unused.db"
+ options := []local.HostOption{local.WithShutdownGracePeriod(time.Second), local.WithAlarmsPollInterval(110 * time.Millisecond), local.WithAlarmsFetchAheadInterval(time.Second), local.WithAlarmsLeaseDuration(2 * time.Second)}
+ if len(databaseURLs) > 0 {
+ databaseURL = databaseURLs[0]
+ } else {
+ options = append(options, local.WithStandaloneMemoryProvider(standalone.StandaloneMemoryOptions{}))
+ }
+ runtime, err := francis.New(databaseURL, "arcane-test-actor-key-32-characters", "test", port, options...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return runtime
+}
+
+// Start starts a previously registered host, verifies its listener, and joins
+// its shutdown during test cleanup.
+func Start(t stdtesting.TB, runtime *francis.Runtime) {
+ t.Helper()
+ appCtx, cancel := context.WithCancel(t.Context())
+ t.Cleanup(func() {
+ cancel()
+ stopCtx, stopCancel := context.WithTimeout(context.WithoutCancel(t.Context()), 15*time.Second)
+ defer stopCancel()
+ if err := runtime.Stop(stopCtx); err != nil {
+ t.Error(err)
+ }
+ })
+ startupCtx, startupCancel := context.WithTimeout(t.Context(), 15*time.Second)
+ defer startupCancel()
+ if err := runtime.Start(startupCtx, appCtx, func(err error) { t.Error(err); cancel() }); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.go
index 04f9813c1..58d98aae1 100644
--- a/backend/pkg/utils/s3/s3.go
+++ b/backend/pkg/utils/s3/s3.go
@@ -13,6 +13,7 @@ import (
"path"
"slices"
"strings"
+ "uuid"
"emperror.dev/errors"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -21,7 +22,6 @@ import (
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
- "github.com/google/uuid"
kit "go.getarcane.app/kit/pkg"
)
@@ -174,7 +174,7 @@ func TestConnection(ctx context.Context, configuration Configuration) (err error
return fmt.Errorf("failed to upload S3 connection test object: %w", err)
}
payload := []byte(connectionTestPayload)
- remoteKey := path.Join(configuration.Prefix, ".arcane-connection-test-"+uuid.NewString())
+ remoteKey := path.Join(configuration.Prefix, ".arcane-connection-test-"+uuid.New().String())
if err := putObjectInternal(ctx, client, configuration.Bucket, remoteKey, payload); err != nil {
return fmt.Errorf("failed to upload S3 connection test object: %w", err)
}
diff --git a/backend/resources/migrations/postgres/090_add_event_deduplication_key.sql b/backend/resources/migrations/postgres/090_add_event_deduplication_key.sql
new file mode 100644
index 000000000..b96aa9afe
--- /dev/null
+++ b/backend/resources/migrations/postgres/090_add_event_deduplication_key.sql
@@ -0,0 +1,7 @@
+-- +goose Up
+ALTER TABLE events ADD COLUMN deduplication_key TEXT;
+CREATE UNIQUE INDEX idx_events_deduplication_key ON events (deduplication_key);
+
+-- +goose Down
+DROP INDEX idx_events_deduplication_key;
+ALTER TABLE events DROP COLUMN deduplication_key;
diff --git a/backend/resources/migrations/sqlite/090_add_event_deduplication_key.sql b/backend/resources/migrations/sqlite/090_add_event_deduplication_key.sql
new file mode 100644
index 000000000..b96aa9afe
--- /dev/null
+++ b/backend/resources/migrations/sqlite/090_add_event_deduplication_key.sql
@@ -0,0 +1,7 @@
+-- +goose Up
+ALTER TABLE events ADD COLUMN deduplication_key TEXT;
+CREATE UNIQUE INDEX idx_events_deduplication_key ON events (deduplication_key);
+
+-- +goose Down
+DROP INDEX idx_events_deduplication_key;
+ALTER TABLE events DROP COLUMN deduplication_key;
diff --git a/cli/go.mod b/cli/go.mod
index 36af5c563..b5fc6a590 100644
--- a/cli/go.mod
+++ b/cli/go.mod
@@ -21,7 +21,7 @@ require (
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.12.1
go.getarcane.app/acfs v0.6.1
- go.getarcane.app/kit v0.3.1
+ go.getarcane.app/kit v0.3.2
go.getarcane.app/sys/bytes v0.2.1
)
@@ -92,7 +92,7 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
- golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect
+ golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
diff --git a/cli/go.sum b/cli/go.sum
index 08c079d6e..1456f2d1c 100644
--- a/cli/go.sum
+++ b/cli/go.sum
@@ -170,8 +170,8 @@ go.getarcane.app/acfs v0.6.1 h1:xPDfof8Tbr3ZQ6ATcyFM4EtyUh+K657CThpGdg+dz50=
go.getarcane.app/acfs v0.6.1/go.mod h1:Z+0Gs4LGNjvGMLq+TKv/oGKh29FRbAH6mcVPwY5KhFI=
go.getarcane.app/builds v0.4.2 h1:pWrNrJ4pjLm2Dpwii4bkf0ptKrlE9N8j/EL1F6o15mA=
go.getarcane.app/builds v0.4.2/go.mod h1:sOgXeqAcEUeblqdUOSPGZLYz3rlTJpTMXaSy0aQhmlQ=
-go.getarcane.app/kit v0.3.1 h1:fuSl8ZMshuFPUvJl/9QwoI/ds1+zsfpixZYqk/hFLUE=
-go.getarcane.app/kit v0.3.1/go.mod h1:KTjbXTEeC3i9e1QxZowDJWjFyABlD6UF3ISVtoIZs10=
+go.getarcane.app/kit v0.3.2 h1:L7CAvGawDg70K/vVxDeM2xZBCHqwuthc9B5OwQwL0Hc=
+go.getarcane.app/kit v0.3.2/go.mod h1:KTjbXTEeC3i9e1QxZowDJWjFyABlD6UF3ISVtoIZs10=
go.getarcane.app/sys/bytes v0.2.1 h1:Mm/VV1FAde3bVvzmSBxB7HVGkj7ZUw2Gbl1DPcUhlEo=
go.getarcane.app/sys/bytes v0.2.1/go.mod h1:fcMYM7dknLk+KDXYAg6XOfRF22POYxcTnQeypFEjDbE=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
@@ -201,8 +201,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
-golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
-golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
+golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
+golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
diff --git a/go.work b/go.work
index 2a2a4c863..9682eb698 100644
--- a/go.work
+++ b/go.work
@@ -1,4 +1,4 @@
-go 1.27
+go 1.27.1
use (
./backend
diff --git a/go.work.sum b/go.work.sum
index 3d2fc8617..0d92bbde4 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -553,6 +553,8 @@ github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyR
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
github.com/akavel/rsrc v0.10.2 h1:Zxm8V5eI1hW4gGaYsJQUhxpjkENuG91ki8B4zCrvEsw=
github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
+github.com/akutz/memconn v0.1.0 h1:NawI0TORU4hcOMsMr11g7vwlCdkYeLKXBcxWu2W/P8A=
+github.com/akutz/memconn v0.1.0/go.mod h1:Jo8rI7m0NieZyLI5e2CDlRdRqRRB4S7Xp77ukDjH+Fw=
github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
@@ -565,6 +567,8 @@ github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vS
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI=
+github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/alexflint/go-filemutex v1.3.0 h1:LgE+nTUWnQCyRKbpoceKZsPQbs84LivvgwUymZXdOcM=
github.com/alexflint/go-filemutex v1.3.0/go.mod h1:U0+VA/i30mGBlLCrFPGtTe9y6wGQfNAWPBTekHQ+c8A=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
@@ -857,6 +861,8 @@ github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4
github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE=
github.com/brianvoe/gofakeit/v7 v7.12.1 h1:df1tiI4SL1dR5Ix4D/r6a3a+nXBJ/OBGU5jEKRBmmqg=
github.com/brianvoe/gofakeit/v7 v7.12.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
+github.com/brunoga/deep v1.3.1 h1:bSrL6FhAZa6JlVv4vsi7Hg8SLwroDb1kgDERRVipBCo=
+github.com/brunoga/deep v1.3.1/go.mod h1:GDV6dnXqn80ezsLSZ5Wlv1PdKAWAO4L5PnKYtv2dgaI=
github.com/bufbuild/buf v1.56.0 h1:Z0eK+npK01FB924rtDVMOJtvBh9c421mYLo9QhUP3pM=
github.com/bufbuild/buf v1.56.0/go.mod h1:uDNMYshCJIXL99OQc71SDeFiDqOse9sSHXPpZlrqElw=
github.com/bufbuild/buf v1.70.0 h1:rGL4TGoy8F1DbQa4BSlMOVBBR7lWblfnKxUdOxmeFns=
@@ -1090,6 +1096,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/creachadair/msync v0.8.1 h1:QRd8si3qZ2Q4TaDL7tS/MG/lFE3YND7U7J9fy42eAFM=
+github.com/creachadair/msync v0.8.1/go.mod h1:dt0bscS09J8Ie3AdccK9JpCb7LfStaDGlAmDLukOlY4=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 h1:uX1JmpONuD549D73r6cgnxyUu18Zb7yHAy5AYU0Pm4Q=
github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
@@ -1113,6 +1121,8 @@ github.com/data-accelerator/zdfs v0.1.5 h1:F7td8AwicTZ3t618SsEYr8CMyMcxBE1KjkoyY
github.com/data-accelerator/zdfs v0.1.5/go.mod h1:/MyNTsQHHKVLznaRBz+PivhIDwglu+wuXKoZxUNmLKI=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbwwpmHn1J5i43Y0uZP97GqasGCzSRJk=
+github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ=
github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
@@ -1228,6 +1238,10 @@ github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSY
github.com/extism/go-sdk v1.7.1 h1:lWJos6uY+tRFdlIHR+SJjwFDApY7OypS/2nMhiVQ9Sw=
github.com/extism/go-sdk v1.7.1/go.mod h1:IT+Xdg5AZM9hVtpFUA+uZCJMge/hbvshl8bwzLtFyKA=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fchimpan/gomod-age v0.1.1-0.20260405015303-09005169a479 h1:DIiy+URimdWW0YcMOMylyD2nsRUV6SxvCoPdKh5PCb4=
+github.com/fchimpan/gomod-age v0.1.1-0.20260405015303-09005169a479/go.mod h1:3kypUxKFWfX23KnOg6M9pQiiCfb4jeUjm6rfGM4/7Wo=
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@@ -1258,8 +1272,12 @@ github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXE
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
+github.com/gaissmai/bart v0.26.1 h1:+w4rnLGNlA2GDVn382Tfe3jOsK5vOr5n4KmigJ9lbTo=
+github.com/gaissmai/bart v0.26.1/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
github.com/getarcaneapp/arcane/types v0.0.0-20251219045603-8a9960a2bf04/go.mod h1:7Wj+0JnpYq4RQS+otyM6e4zkyJKDC01wxkWPyFCHwZA=
github.com/getarcaneapp/updater v0.3.5/go.mod h1:2ENitZR/lA5MuBn60OZYR5wvU3ZCE+pAgfrebfOlRDA=
+github.com/getkin/kin-openapi v0.149.0 h1:ZbhmVJ4yq5RZDUsyP8lcBcGMsjsaTqXEFt6isdtMDfA=
+github.com/getkin/kin-openapi v0.149.0/go.mod h1:1+BHDzstro+P5CKtPy1X4PfofnFgmRe6uvMy9+r9fKY=
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
github.com/gin-contrib/cors v1.7.7/go.mod h1:K5tW0RkzJtWSiOdikXloy8VEZlgdVNpHNw8FpjUPNrE=
@@ -1297,6 +1315,8 @@ github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyM
github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo=
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 h1:vymEbVwYFP/L05h5TKQxvkXoKxNvTpjxYKdF1Nlwuao=
+github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk=
@@ -1426,6 +1446,8 @@ github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ=
+github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c=
github.com/gofiber/fiber/v2 v2.52.7 h1:6xJpE4sSqErvMiEZo9ZpJLRSVcpkNBvioeqAHKwhTZY=
github.com/gofiber/fiber/v2 v2.52.7/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw=
@@ -1673,6 +1695,8 @@ github.com/hashicorp/terraform-json v0.27.1 h1:zWhEracxJW6lcjt/JvximOYyc12pS/gaK
github.com/hashicorp/terraform-json v0.27.1/go.mod h1:GzPLJ1PLdUG5xL6xn1OXWIjteQRT2CNT9o/6A9mi9hE=
github.com/hashicorp/terraform-json v0.27.2 h1:BwGuzM6iUPqf9JYM/Z4AF1OJ5VVJEEzoKST/tRDBJKU=
github.com/hashicorp/terraform-json v0.27.2/go.mod h1:GzPLJ1PLdUG5xL6xn1OXWIjteQRT2CNT9o/6A9mi9hE=
+github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU=
+github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo=
github.com/hiddeco/sshsig v0.2.0 h1:gMWllgKCITXdydVkDL+Zro0PU96QI55LwUwebSwNTSw=
github.com/hiddeco/sshsig v0.2.0/go.mod h1:nJc98aGgiH6Yql2doqH4CTBVHexQA40Q+hMMLHP4EqE=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -1690,6 +1714,8 @@ github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU
github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
+github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c=
@@ -1726,6 +1752,8 @@ github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU=
github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw=
github.com/jdx/go-netrc v1.0.0 h1:QbLMLyCZGj0NA8glAhxUpf1zDg6cxnWgMBbjq40W0gQ=
github.com/jdx/go-netrc v1.0.0/go.mod h1:Gh9eFQJnoTNIRHXl2j5bJXA1u84hQWJWgGh569zF3v8=
+github.com/jedib0t/go-pretty/v6 v6.7.8 h1:BVYrDy5DPBA3Qn9ICT+PokP9cvCv1KaHv2i+Hc8sr5o=
+github.com/jedib0t/go-pretty/v6 v6.7.8/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU=
github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY=
github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E=
github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
@@ -1760,6 +1788,8 @@ github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtL
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/jsimonetti/rtnetlink v1.4.1 h1:JfD4jthWBqZMEffc5RjgmlzpYttAVw1sdnmiNaPO3hE=
+github.com/jsimonetti/rtnetlink v1.4.1/go.mod h1:xJjT7t59UIZ62GLZbv6PLLo8VFrostJMPBAheR6OM8w=
github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM=
github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -1803,6 +1833,20 @@ github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
+github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
+github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4=
+github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg=
+github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc=
+github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY=
+github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM=
+github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA=
+github.com/knadh/koanf/providers/posflag v1.0.1 h1:EnMxHSrPkYCFnKgBUl5KBgrjed8gVFrcXDzaW4l/C6Y=
+github.com/knadh/koanf/providers/posflag v1.0.1/go.mod h1:3Wn3+YG3f4ljzRyCUgIwH7G0sZ1pMjCOsNBovrbKmAk=
+github.com/knadh/koanf/providers/structs v1.0.0 h1:DznjB7NQykhqCar2LvNug3MuxEQsZ5KvfgMbio+23u4=
+github.com/knadh/koanf/providers/structs v1.0.0/go.mod h1:kjo5TFtgpaZORlpoJqcbeLowM2cINodv8kX+oFAeQ1w=
+github.com/knadh/koanf/v2 v2.3.2 h1:Ee6tuzQYFwcZXQpc2MiVeC6qHMandf5SMUJJNoFp/c4=
+github.com/knadh/koanf/v2 v2.3.2/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI=
github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q=
github.com/knqyf263/go-rpmdb v0.1.1 h1:oh68mTCvp1XzxdU7EfafcWzzfstUZAEa3MW0IJye584=
@@ -1960,6 +2004,8 @@ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvls
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
@@ -2105,6 +2151,10 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU=
github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU=
+github.com/oasdiff/yaml v0.1.1 h1:6nHx+pn9gBRM6YpBlFZFQGCCd1nuvqOBtTD3KKTgGxY=
+github.com/oasdiff/yaml v0.1.1/go.mod h1:EYJNoyktvWMJ0Hmhx+6qTaqMOsalUaRGT8Sj1hNcegU=
+github.com/oasdiff/yaml3 v0.0.14 h1:aLJee3hxBK2H5wdXd9iPcIXb93Nty1Ge0pT171eHtkw=
+github.com/oasdiff/yaml3 v0.0.14/go.mod h1:csto2xfDjYccdUn/yw/bPjj/cYTdp6HtFA0J4TWG+gg=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM=
@@ -2201,6 +2251,8 @@ github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk
github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pires/go-proxyproto v0.8.1 h1:9KEixbdJfhrbtjpz/ZwCdWDD2Xem0NZ38qMYaASJgp0=
+github.com/pires/go-proxyproto v0.8.1/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -2256,8 +2308,6 @@ github.com/quay/claircore/updater/driver v1.0.0 h1:w7dAUjO3GBK6RjNyTZ2Kwz0l/Wuic
github.com/quay/claircore/updater/driver v1.0.0/go.mod h1:My5aY1wBpgxcWaHQZ0VoPmmj/EzuH7fq4ntzJbos4OI=
github.com/quay/goval-parser v0.8.8 h1:Uf+f9iF2GIR5GPUY2pGoa9il2+4cdES44ZlM0mWm4cA=
github.com/quay/goval-parser v0.8.8/go.mod h1:Y0NTNfPYOC7yxsYKzJOrscTWUPq1+QbtHw4XpPXWPMc=
-github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
-github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic=
@@ -2293,6 +2343,8 @@ github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
github.com/rubenv/sql-migrate v1.8.0 h1:dXnYiJk9k3wetp7GfQbKJcPHjVJL6YK19tKj8t2Ns0o=
github.com/rubenv/sql-migrate v1.8.0/go.mod h1:F2bGFBwCU+pnmbtNYDeKvSuvL6lBVtXDXUUv5t+u1qw=
github.com/rubenv/sql-migrate v1.8.1 h1:EPNwCvjAowHI3TnZ+4fQu3a915OpnQoPAjTXCGOy2U0=
@@ -2302,6 +2354,8 @@ github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNl
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c h1:8gOLsYwaY2JwlTMT4brS5/9XJdrdIbmk2obvQ748CC0=
github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c/go.mod h1:kwM/7r/rVluTE8qJbHAffduuqmSv4knVQT2IajGvSiA=
+github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0=
+github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs=
github.com/safchain/ethtool v0.6.2 h1:O3ZPFAKEUEfbtE6J/feEe2Ft7dIJ2Sy8t4SdMRiIMHY=
github.com/safchain/ethtool v0.6.2/go.mod h1:VS7cn+bP3Px3rIq55xImBiZGHVLNyBh5dqG6dDQy8+I=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
@@ -2418,6 +2472,18 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtse
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=
+github.com/tailscale/certstore v0.1.1-0.20260409135935-3638fb84b77d h1:JcGKBZAL7ePLwOhUdN8qGQZlP5GueEiIZwY7R62pejE=
+github.com/tailscale/certstore v0.1.1-0.20260409135935-3638fb84b77d/go.mod h1:XrBNfAFN+pwoWuksbFS9Ccxnopa15zJGgXRFN90l3K4=
+github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 h1:Gzfnfk2TWrk8Jj4P4c1a3CtQyMaTVCznlkLZI++hok4=
+github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55/go.mod h1:4k4QO+dQ3R5FofL+SanAUZe+/QfeK0+OIuwDIRu2vSg=
+github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd h1:Rf9uhF1+VJ7ZHqxrG8pJ6YacmHvVCmByDmGbAWCc/gA=
+github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd/go.mod h1:EbW0wDK/qEUYI0A5bqq0C2kF8JTQwWONmGDBbzsxxHo=
+github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc h1:24heQPtnFR+yfntqhI3oAu9i27nEojcQ4NuBQOo5ZFA=
+github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc/go.mod h1:f93CXfllFsO9ZQVq+Zocb1Gp4G5Fz0b0rXHLOzt/Djc=
+github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:UBPHPtv8+nEAy2PD8RyAhOYvau1ek0HDJqLS/Pysi14=
+github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
+github.com/tailscale/wireguard-go v0.0.0-20260715223240-2e01ba5b00f0 h1:CnIEL2n7Xql6Ux1k+Vu5S5ubDHCT/kxFgkKCY8FjefU=
+github.com/tailscale/wireguard-go v0.0.0-20260715223240-2e01ba5b00f0/go.mod h1:6SerzcvHWQchKO2BfNdmquA77CHSECZuFl+D9fp4RnI=
github.com/tchap/go-patricia/v2 v2.3.2/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k=
github.com/tchap/go-patricia/v2 v2.3.3 h1:xfNEsODumaEcCcY3gI0hYPZ/PcpVv5ju6RMAhgwZDDc=
github.com/tchap/go-patricia/v2 v2.3.3/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k=
@@ -2527,6 +2593,8 @@ github.com/vektah/gqlparser/v2 v2.5.32/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6O
github.com/vektah/gqlparser/v2 v2.5.33 h1:lRp8aIeNUNbimf/axZd7ETg24q06hBtPaas+TcvI/7E=
github.com/vektah/gqlparser/v2 v2.5.33/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts=
github.com/vektah/gqlparser/v2 v2.5.36/go.mod h1:cAJ9qwVgPaUkWv6Gn8vn0mqOE0Ui5Pn56wNy5396XWo=
+github.com/vektra/mockery/v3 v3.7.1 h1:4jZJCTzf5CEXSXHOtgIu3TuHYUuDKwcc/0vurjQEpjk=
+github.com/vektra/mockery/v3 v3.7.1/go.mod h1:fbChccNiUvQaUVaCHS6/7OL5/D65KljJVk31LuPPUjY=
github.com/veraison/go-cose v1.1.0 h1:AalPS4VGiKavpAzIlBjrn7bhqXiXi4jbMYY/2+UC+4o=
github.com/veraison/go-cose v1.1.0/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4JkFEZi/an96Ct4=
github.com/veraison/go-cose v1.3.0 h1:2/H5w8kdSpQJyVtIhx8gmwPJ2uSz1PkyWFx0idbd7rk=
@@ -2541,10 +2609,6 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
-github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
-github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
-github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
-github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/weppos/publicsuffix-go v0.15.1-0.20210511084619-b1f36a2d6c0b h1:FsyNrX12e5BkplJq7wKOLk0+C6LZ+KGXvuEcKUYm5ss=
github.com/weppos/publicsuffix-go v0.15.1-0.20210511084619-b1f36a2d6c0b/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
github.com/weppos/publicsuffix-go v0.50.2 h1:KsJFc8IEKTJovM46SRCnGNsM+rFShxcs6VEHjOJcXzE=
@@ -2699,6 +2763,8 @@ go.getarcane.app/kit v0.1.0 h1:82oO/i1fS4xbzAG4Xc8ALOWOwW4kwQYDdBpVHE19QkI=
go.getarcane.app/kit v0.1.0/go.mod h1:oiH4hLowxfQYFtmldyk/y79XYBHYfzgd86MrSQS9UEg=
go.getarcane.app/kit v0.2.0 h1:3PPHo5nHuns1vjqmCMFPFdnmbH/qQOserONCba/gBi4=
go.getarcane.app/kit v0.2.0/go.mod h1:TGqQCbAJs3ldzkFncXZ93wTVSDhAMQlqyNdVO2CNjIk=
+go.getarcane.app/kit v0.3.2 h1:L7CAvGawDg70K/vVxDeM2xZBCHqwuthc9B5OwQwL0Hc=
+go.getarcane.app/kit v0.3.2/go.mod h1:KTjbXTEeC3i9e1QxZowDJWjFyABlD6UF3ISVtoIZs10=
go.getarcane.app/streams v0.4.0 h1:MtBX1SVgt3nrcRNr5SYAvckGp9a5qeSjMTw/LAGwlzk=
go.getarcane.app/streams v0.4.0/go.mod h1:LHMKblQvsL1nqKh/Y+KgQbwFSMI2I7bs9u8ZwZqzjnQ=
go.getarcane.app/streams v0.4.2 h1:+xrRHiOt25cLbFTDpPLzv7eFkk7biwDz5YjBXkbWUPw=
@@ -2749,6 +2815,10 @@ go.mongodb.org/mongo-driver/v2 v2.8.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzyb
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 h1:5sHc4ToTFjfSZCtGAAM6jPunICAmJX73htv372T4ipc=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.1/go.mod h1:oa6kgvyz/3GYW04dohd0++xJIH4xdQY8PAbpeCMaM8M=
+go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 h1:9qgxsFLskbDMXl8WMqThoF6w8yGJgCumn9qRc67OmnI=
+go.opentelemetry.io/contrib/bridges/prometheus v0.71.0/go.mod h1:2rCjF4F2siiTeLCzJsaGZ3CK0XIoimCSKXEBPdv+Je0=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
@@ -2759,6 +2829,8 @@ go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmf
go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs=
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=
go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=
+go.opentelemetry.io/contrib/exporters/autoexport v0.71.0 h1:VCsJbp0YLyPtx2tu5Vgv2a2/qLoaMCj8hT2uZ34+Mx0=
+go.opentelemetry.io/contrib/exporters/autoexport v0.71.0/go.mod h1:qxZqn7e10f6ajmMCkg/47rMS7qQYfaOl2nj/4aytHUQ=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
@@ -2785,6 +2857,10 @@ go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhO
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4=
go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0 h1:Bu39F5tzJct+f2IZbB8989fwyTps3c8e7EsUQsz+vs8=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.22.0/go.mod h1:dJUwod88EsFgYCqrDHaSPzhiY9pBUpt0d85/qSfua7k=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
@@ -2804,8 +2880,14 @@ go.opentelemetry.io/otel/exporters/prometheus v0.66.0 h1:vkrK8PAznv2NKt2r+kdu252
go.opentelemetry.io/otel/exporters/prometheus v0.66.0/go.mod h1:V/UB6D3vMF/UBOL5igAsAYnk1nG/bzYYTzvsB16cy7o=
go.opentelemetry.io/otel/exporters/prometheus v0.67.0 h1:7IefDa35e6V3NoiqIeLDMDxMFyZDk5qcoC0Ax4cC16E=
go.opentelemetry.io/otel/exporters/prometheus v0.67.0/go.mod h1:nsPI1awTg5Vmg1YrommL2mVarVGlqc4yXOoKAkPRD0c=
+go.opentelemetry.io/otel/exporters/prometheus v0.68.0 h1:QOf2IftqQwITVRJpnn0M7M9ZCbgWfxz4P7i9C9yc2N4=
+go.opentelemetry.io/otel/exporters/prometheus v0.68.0/go.mod h1:bgSvqu2TWGXiz7yr5UTMfObH8oqxJWHTnubQ3ef9BO4=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0 h1:kvMAiLEudKmk+CSG+iYbU8vTUGNNDaf/V09OO5lrTwI=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.22.0/go.mod h1:L9Dlksri+MdT1cb2gIiA1cJJYW3Y92ipvDjNxYEyaDI=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0 h1:PR9eAf7o0dQs3hshZNZpE9aW2dXWX/KdDf6pJilVD3U=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.46.0/go.mod h1:2Z4KyNdH1uuzivdinyfGsxzNNT/Rl45pwtVwfYVI0xk=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE=
@@ -2814,6 +2896,10 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 h1:bl2S7Ubua0Nms+D
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0/go.mod h1:L0hRV50XdVIODHUfWEqGRCXQvj2rV82STVo12FMFBU0=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0 h1:KdRxPiAoMptR3vfWzvjjvutTsSiwbC2uG0496rzZNfo=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.46.0/go.mod h1:K/qSA+3G7Eovxi4K09wzrAgkWRnosS0DAOZeEpve7sM=
+go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0=
+go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc=
go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
@@ -2831,6 +2917,8 @@ go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWm
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk=
+go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g=
go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
@@ -2868,6 +2956,10 @@ go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI=
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
+go4.org/mem v0.0.0-20240501181205-ae6ca9944745 h1:Tl++JLUCe4sxGu8cTpDzRLd3tN7US4hOxG5YpKCzkek=
+go4.org/mem v0.0.0-20240501181205-ae6ca9944745/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y=
@@ -3175,6 +3267,10 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
+golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
+golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
+golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
gonum.org/v1/plot v0.15.2 h1:Tlfh/jBk2tqjLZ4/P8ZIwGrLEWQSPDLRm/SNWKNXiGI=
gonum.org/v1/plot v0.15.2/go.mod h1:DX+x+DWso3LTha+AdkJEv5Txvi+Tql3KAGkehP0/Ubg=
gonum.org/v1/tools v0.0.0-20200318103217-c168b003ce8c h1:cJWOvXtcaFSGXz2F4z2AMM0VV7edDDGrxb5GLQH7ayQ=
@@ -3336,6 +3432,8 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
+gvisor.dev/gvisor v0.0.0-20260224225140-573d5e7127a8 h1:Zy8IV/+FMLxy6j6p87vk/vQGKcdnbprwjTxc8UiUtsA=
+gvisor.dev/gvisor v0.0.0-20260224225140-573d5e7127a8/go.mod h1:QkHjoMIBaYtpVufgwv3keYAbln78mBoCuShZrPrer1Q=
helm.sh/helm/v3 v3.19.2 h1:psQjaM8aIWrSVEly6PgYtLu/y6MRSmok4ERiGhZmtUY=
helm.sh/helm/v3 v3.19.2/go.mod h1:gX10tB5ErM+8fr7bglUUS/UfTOO8UUTYWIBH1IYNnpE=
helm.sh/helm/v4 v4.2.0 h1:J+0TmTtPK2NuS6z9Z2WOcIX0nGGJylokEZLt0fi0X4U=
@@ -3521,3 +3619,5 @@ tags.cncf.io/container-device-interface/specs-go v1.0.0 h1:8gLw29hH1ZQP9K1YtAzpv
tags.cncf.io/container-device-interface/specs-go v1.0.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=
tags.cncf.io/container-device-interface/specs-go v1.1.0 h1:QRZVeAceQM+zTZe12eyfuJuuzp524EKYwhmvLd+h+yQ=
tags.cncf.io/container-device-interface/specs-go v1.1.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=
+tailscale.com v1.102.4 h1:FcAkb7MgfFFUIUASg18Mv5cAiraxb+eMgOQaOKqKyPo=
+tailscale.com v1.102.4/go.mod h1:47bv91Xbg4K1p5wti7F1dmKvUVWV5BXF78d9EWJ+d6c=
diff --git a/types/backup/durable_run.go b/types/backup/durable_run.go
new file mode 100644
index 000000000..fab613f0f
--- /dev/null
+++ b/types/backup/durable_run.go
@@ -0,0 +1,24 @@
+package backup
+
+import schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
+// DurableRunCommand freezes the input accepted by a backup domain.
+type DurableRunCommand struct {
+ UserID string `msgpack:"userID"`
+ EnvironmentID string `msgpack:"environmentID"`
+ Permission string `msgpack:"permission"`
+ RequestedWithKey string `msgpack:"requestedWithKey"`
+ Kind string `msgpack:"kind"`
+ RunID string `msgpack:"runID"`
+ ActivityID string `msgpack:"activityID"`
+ Payload []byte `msgpack:"payload"`
+}
+
+// DurableRunState records dispatch and effect evidence across host restarts.
+type DurableRunState struct {
+ Command DurableRunCommand `msgpack:"command"`
+ Started bool `msgpack:"started"`
+ Status schedulertypes.RunStatus `msgpack:"status"`
+ Targets []schedulertypes.TargetOutcome `msgpack:"targets"`
+ Error string `msgpack:"error"`
+}
diff --git a/types/database/connection.go b/types/database/connection.go
new file mode 100644
index 000000000..43291faf4
--- /dev/null
+++ b/types/database/connection.go
@@ -0,0 +1,10 @@
+package database
+
+import "time"
+
+// SQLiteConnectionOptions supplies pool-specific pragma defaults.
+type SQLiteConnectionOptions struct {
+ IgnoreForeignKeys bool
+ JournalMode string
+ BusyTimeout time.Duration
+}
diff --git a/types/scheduler/durable.go b/types/scheduler/durable.go
new file mode 100644
index 000000000..db1ca2dac
--- /dev/null
+++ b/types/scheduler/durable.go
@@ -0,0 +1,65 @@
+package scheduler
+
+import "time"
+
+// CoordinatorState is the durable state of one environment/job actor.
+type CoordinatorState struct {
+ Revision uint64
+ Record QueueRecord
+ Imported bool
+ ImportHash string
+ Dispatches map[string]DispatchIntent
+ PendingActivitySync map[string]time.Time
+}
+
+type DispatchIntent struct {
+ RunID string
+ Attempt int
+ JobID string
+}
+
+type ExecutionCommand struct {
+ EnvironmentID string
+ JobID string
+ RunID string
+ Attempt int
+}
+
+// ExecutionClaim records whether a delivery acquired its persisted run.
+type ExecutionClaim struct {
+ Run Run
+ Reconcile bool
+ Claimed bool
+ Deferred bool
+}
+
+type ScheduleOccurrence struct {
+ Generation uint64
+ Sequence uint64
+}
+
+type AdmissionKey struct {
+ Scope string
+ ID string
+}
+type AdmissionState struct {
+ Epoch string
+ Token string
+ AcquiredAt time.Time
+}
+type AdmissionCommand struct {
+ Epoch string
+ Token string
+}
+
+// ActivitySyncCommand acknowledges one persisted activity projection version.
+type ActivitySyncCommand struct {
+ RunID string
+ UpdatedAt time.Time
+}
+
+// CoordinatorImport binds imported state to the exact inactive legacy row.
+type CoordinatorImport struct {
+ Record QueueRecord
+ SourceHash string
+}
diff --git a/types/scheduler/job.go b/types/scheduler/job.go
index 049793c67..f5fa02da5 100644
--- a/types/scheduler/job.go
+++ b/types/scheduler/job.go
@@ -64,7 +64,6 @@ type WorkerController interface {
// JobScheduler is the shared public contract for the actor-owned scheduler.
// Concrete cron and actor state remains private to the backend implementation.
type JobScheduler interface {
- SetDispatcher(dispatcher Dispatcher)
ListRegisteredJobs() []Job
WorkerController
DynamicScheduler
diff --git a/types/scheduler/queue.go b/types/scheduler/queue.go
index f496eaeac..8a635d094 100644
--- a/types/scheduler/queue.go
+++ b/types/scheduler/queue.go
@@ -6,7 +6,7 @@ import (
)
// RunObserver projects persisted runs into operator-facing activity records.
-// ActivityID must be deterministic and empty for runs that should stay quiet.
+// ActivityID is persisted before projection and is empty for runs that should stay quiet.
type RunObserver interface {
ActivityID(run Run) string
SyncRunActivity(ctx context.Context, run Run) error
@@ -14,6 +14,8 @@ type RunObserver interface {
// QueueRecord holds atomic admission, claims, and checkpoints for one job and target.
type QueueRecord struct {
+ Generation uint64 `json:"-"`
+ Sequence uint64 `json:"-"`
JobID string `json:"jobId"`
EnvironmentID string `json:"environmentId"`
Schedule string `json:"schedule"`
diff --git a/types/scheduler/run.go b/types/scheduler/run.go
index 46fbbd3e1..f4d91e70c 100644
--- a/types/scheduler/run.go
+++ b/types/scheduler/run.go
@@ -2,6 +2,7 @@ package scheduler
import (
"context"
+ "encoding/json/jsontext"
"time"
)
@@ -26,11 +27,12 @@ func (s RunStatus) Terminal() bool {
}
type TargetOutcome struct {
- ResourceType string `json:"resourceType,omitempty"`
- ID string `json:"id"`
- Status RunStatus `json:"status"`
- Message string `json:"message,omitempty"`
- ActivityID string `json:"activityId,omitempty"`
+ RecoveryData jsontext.Value `json:"-" msgpack:"recoveryData,omitempty"`
+ ResourceType string `json:"resourceType,omitempty"`
+ ID string `json:"id"`
+ Status RunStatus `json:"status"`
+ Message string `json:"message,omitempty"`
+ ActivityID string `json:"activityId,omitempty"`
}
type Outcome struct {
@@ -106,12 +108,6 @@ type WorkerHealth struct {
UpdatedAt time.Time `json:"updatedAt"`
}
-// Dispatcher owns durable admission and execution. Cron only submits work.
-type Dispatcher interface {
- Submit(ctx context.Context, request Request) (Run, error)
- Checkpoint(ctx context.Context, jobID, schedule string, nextRun time.Time) error
-}
-
// Reconciler consults domain state before repeating interrupted work.
type Reconciler interface {
Reconcile(ctx context.Context, previous Run) (Outcome, error)
diff --git a/types/updater/durable.go b/types/updater/durable.go
new file mode 100644
index 000000000..952db96b4
--- /dev/null
+++ b/types/updater/durable.go
@@ -0,0 +1,31 @@
+package updater
+
+// SingleUpdateCommand contains the persisted inputs for an asynchronous update.
+type SingleUpdateCommand struct {
+ ContainerID string
+ ActivityID string
+ UserID string
+ KeyID string
+}
+
+type SingleUpdateState struct {
+ Command SingleUpdateCommand
+ Status string
+ Target *FrozenUpdateTarget
+ Result *Result
+ Failure string
+}
+
+// FrozenUpdateTarget identifies the selected container and immutable image intent.
+type FrozenUpdateTarget struct {
+ ContainerID string
+ ContainerName string
+ ComposeProject string
+ ComposeService string
+ ComposeNumber string
+ BaselineImageID string
+ BaselineStartedAt string
+ BaselineRestartCount int
+ DesiredImageRef string
+ DesiredDigest string
+}